diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index cc41b9c9e4..bf38f76037 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -81,6 +81,7 @@ jobs: crates/*|Cargo.toml|Cargo.lock|pyproject.toml|uv.lock) broad=true performance=true + format_performance=true rules_performance=true ;; src/sqlbuild/compiler/*|src/sqlbuild/adapter/type_system/*) diff --git a/Cargo.lock b/Cargo.lock index dc86e95c01..374bb8b10a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -623,6 +623,7 @@ dependencies = [ "globset", "mimalloc", "polyglot-sql", + "polyglot-sql-function-catalogs", "pyo3", "rayon", "regex", diff --git a/Cargo.toml b/Cargo.toml index 1b058e560b..600870c0f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,9 @@ polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "25ad0 "semantic", "stacker", ] } +polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "25ad011c49f2c7cd48f4427abb8fc0a6af2e4120", default-features = false, features = [ + "dialect-duckdb", +] } quote = "1" rayon = "1" regex = "1" diff --git a/Makefile b/Makefile index 1a4f206025..4b634b5dec 100644 --- a/Makefile +++ b/Makefile @@ -82,6 +82,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_identity.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_propagation.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_remove_column_semantics.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_reusable_model_schemas.py \ @@ -131,6 +132,9 @@ E2E_DUCKDB_CLI_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/databricks \ tests/e2e/src/sqlbuild/cli/commands/main/dbt \ tests/e2e/src/sqlbuild/cli/commands/main/debug \ + tests/e2e/src/sqlbuild/cli/commands/main/format/test_format_example_projects.py \ + tests/e2e/src/sqlbuild/cli/commands/main/format/test_rule_fixes.py \ + tests/e2e/src/sqlbuild/cli/commands/main/format/test_token_preservation.py \ tests/e2e/src/sqlbuild/cli/commands/main/freshness \ tests/e2e/src/sqlbuild/cli/commands/main/init \ tests/e2e/src/sqlbuild/cli/commands/main/rules \ diff --git a/crates/sqlbuild-rules-native/Cargo.toml b/crates/sqlbuild-rules-native/Cargo.toml index 8388367714..f8f260a150 100644 --- a/crates/sqlbuild-rules-native/Cargo.toml +++ b/crates/sqlbuild-rules-native/Cargo.toml @@ -19,6 +19,7 @@ fensu-policy.workspace = true globset.workspace = true mimalloc.workspace = true polyglot-sql.workspace = true +polyglot-sql-function-catalogs.workspace = true pyo3.workspace = true rayon.workspace = true regex.workspace = true diff --git a/crates/sqlbuild-rules-native/function_names/snowflake.txt b/crates/sqlbuild-rules-native/function_names/snowflake.txt new file mode 100644 index 0000000000..6147565f61 --- /dev/null +++ b/crates/sqlbuild-rules-native/function_names/snowflake.txt @@ -0,0 +1,993 @@ +# Snowflake built-in function spellings accepted by SQLBuild's compile-time name check (B101). +# +# Source: Snowflake's public "All functions (alphabetical)" reference, including every listed +# alias (for example SUBSTR and SUBSTRING, DATEDIFF, TIMEDIFF and TIMESTAMPDIFF, TRY_TO_DECIMAL, +# TRY_TO_NUMBER and TRY_TO_NUMERIC, STARTSWITH, ENDSWITH, CONTAINS, IFF), with wildcard families +# such as TO_TIMESTAMP_* and YEAR*/DAY*/WEEK* expanded, plus SYS_CONNECT_BY_PATH from the +# CONNECT BY reference. Names are case-insensitive; one name per line; `#` starts a comment. +# +# A spelling absent from this list is rejected only when the SQL parser maps it to the same +# built-in node as a listed spelling, so user-defined and unknown functions are never rejected. +# Add a missing Snowflake built-in here rather than suppressing the diagnostic. +ABS +ACCEPTED_VALUES +ACOS +ACOSH +ADD_MONTHS +AGENT_RUN +AGG +AI_AGG +AI_CLASSIFY +AI_COMPLETE +AI_COUNT_TOKENS +AI_EMBED +AI_EXTRACT +AI_FILTER +AI_MULTI_EMBED +AI_PARSE_DOCUMENT +AI_REDACT +AI_SENTIMENT +AI_SIMILARITY +AI_SUMMARIZE +AI_SUMMARIZE_AGG +AI_TRANSCRIBE +AI_TRANSLATE +ALERT_HISTORY +ALL_USER_NAMES +ANY_VALUE +APPLICATION_CALLBACK_HISTORY +APPLICATION_CONFIGURATION_VALUE_HISTORY +APPLICATION_JSON +APPLICATION_SPECIFICATION_STATUS_HISTORY +APPROXIMATE_JACCARD_INDEX +APPROXIMATE_SIMILARITY +APPROX_COUNT_DISTINCT +APPROX_PERCENTILE +APPROX_PERCENTILE_ACCUMULATE +APPROX_PERCENTILE_COMBINE +APPROX_PERCENTILE_ESTIMATE +APPROX_TOP_K +APPROX_TOP_K_ACCUMULATE +APPROX_TOP_K_COMBINE +APPROX_TOP_K_ESTIMATE +ARRAYS_OVERLAP +ARRAYS_TO_OBJECT +ARRAYS_ZIP +ARRAY_AGG +ARRAY_APPEND +ARRAY_CAT +ARRAY_COMPACT +ARRAY_CONSTRUCT +ARRAY_CONSTRUCT_COMPACT +ARRAY_CONSTRUCT_STRUCTURED +ARRAY_CONTAINS +ARRAY_DISTINCT +ARRAY_EXCEPT +ARRAY_FLATTEN +ARRAY_GENERATE_RANGE +ARRAY_INSERT +ARRAY_INTERSECTION +ARRAY_MAX +ARRAY_MIN +ARRAY_POSITION +ARRAY_PREPEND +ARRAY_REMOVE +ARRAY_REMOVE_AT +ARRAY_REVERSE +ARRAY_SIZE +ARRAY_SLICE +ARRAY_SORT +ARRAY_TO_STRING +ARRAY_UNION_AGG +ARRAY_UNIQUE_AGG +ASCII +ASIN +ASINH +AS_ARRAY +AS_BINARY +AS_BOOLEAN +AS_CHAR +AS_DATE +AS_DECIMAL +AS_DOUBLE +AS_INTEGER +AS_NUMBER +AS_OBJECT +AS_REAL +AS_TIME +AS_TIMESTAMP_LTZ +AS_TIMESTAMP_NTZ +AS_TIMESTAMP_TZ +AS_VARCHAR +ATAN +ATAN2 +ATANH +AUTOMATIC_CLUSTERING_HISTORY +AUTO_REFRESH_REGISTRATION_HISTORY +AVAILABLE_LISTINGS +AVAILABLE_LISTING_REFRESH_HISTORY +AVG +BASE64_DECODE_BINARY +BASE64_DECODE_STRING +BASE64_ENCODE +BETWEEN +BIND_VALUES +BITAND +BITAND_AGG +BITMAP_ABSOLUTE_POSITION +BITMAP_AND +BITMAP_AND_AGG +BITMAP_BIT_POSITION +BITMAP_BUCKET_NUMBER +BITMAP_CONSTRUCT_AGG +BITMAP_COUNT +BITMAP_OR +BITMAP_OR_AGG +BITMAP_TO_ARRAY +BITNOT +BITOR +BITOR_AGG +BITSHIFTLEFT +BITSHIFTRIGHT +BITXOR +BITXOR_AGG +BIT_LENGTH +BLANK_COUNT +BLANK_PERCENT +BOOLAND +BOOLAND_AGG +BOOLNOT +BOOLOR +BOOLOR_AGG +BOOLXOR +BOOLXOR_AGG +BUILD_SCOPED_FILE_URL +BUILD_STAGE_FILE_URL +CASE +CAST +CBRT +CEIL +CHAR +CHARINDEX +CHECK_JSON +CHECK_XML +CHR +CLASSIFY_TEXT +COALESCE +COLLATE +COLLATION +COMPLETE +COMPLETE_TASK_GRAPHS +COMPRESS +CONCAT +CONCAT_WS +CONDITIONAL_CHANGE_EVENT +CONDITIONAL_TRUE_EVENT +CONTAINS +CONVERT_TIMEZONE +COPY_HISTORY +CORR +CORTEX_SEARCH_BATCH +CORTEX_SEARCH_DATA_SCAN +CORTEX_SEARCH_REFRESH_HISTORY +COS +COSH +COT +COUNT +COUNT_IF +COUNT_TOKENS +COVAR_POP +COVAR_SAMP +CUME_DIST +CUMULATIVE_PRIVACY_LOSSES +CURRENT_ACCOUNT +CURRENT_ACCOUNT_NAME +CURRENT_AVAILABLE_ROLES +CURRENT_CLIENT +CURRENT_DATABASE +CURRENT_DATE +CURRENT_IP_ADDRESS +CURRENT_ORGANIZATION_NAME +CURRENT_ORGANIZATION_USER +CURRENT_REGION +CURRENT_ROLE +CURRENT_ROLE_TYPE +CURRENT_SCHEMA +CURRENT_SCHEMAS +CURRENT_SECONDARY_ROLES +CURRENT_SESSION +CURRENT_STATEMENT +CURRENT_TASK_GRAPHS +CURRENT_TIME +CURRENT_TIMESTAMP +CURRENT_TRANSACTION +CURRENT_USER +CURRENT_VERSION +CURRENT_WAREHOUSE +DATABASE_REFRESH_HISTORY +DATABASE_REFRESH_PROGRESS +DATABASE_REFRESH_PROGRESS_BY_JOB +DATABASE_REPLICATION_USAGE_HISTORY +DATABASE_STORAGE_USAGE_HISTORY +DATASKETCHES_HLL +DATASKETCHES_HLL_ACCUMULATE +DATASKETCHES_HLL_COMBINE +DATASKETCHES_HLL_ESTIMATE +DATA_AGENT_RUN +DATA_METRIC_FUNCTION_EXPECTATIONS +DATA_METRIC_FUNCTION_REFERENCES +DATA_METRIC_SCHEDULED_TIME +DATA_QUALITY_MONITORING_EXPECTATION_STATUS +DATA_QUALITY_MONITORING_RESULTS +DATA_TRANSFER_HISTORY +DATE +DATEADD +DATEDIFF +DATE_FROM_PARTS +DATE_PART +DATE_TRUNC +DAY +DAYNAME +DAYOFMONTH +DAYOFWEEK +DAYOFWEEKISO +DAYOFYEAR +DBT_PROJECT_EXECUTION_HISTORY +DECODE +DECOMPRESS_BINARY +DECOMPRESS_STRING +DECRYPT +DECRYPT_RAW +DEGREES +DENSE_RANK +DIV0 +DIV0NULL +DP_INTERVAL_HIGH +DP_INTERVAL_LOW +DUPLICATE_COUNT +DYNAMIC_TABLES +DYNAMIC_TABLE_GRAPH_HISTORY +DYNAMIC_TABLE_REFRESH_HISTORY +EDITDISTANCE +EMAIL_INTEGRATION_CONFIG +EMBED_TEXT_1024 +EMBED_TEXT_768 +ENCRYPT +ENCRYPT_RAW +ENDSWITH +ENTITY_SENTIMENT +EQUAL_NULL +ESTIMATE_REMAINING_DP_AGGREGATES +EXECUTE_AI_EVALUATION +EXP +EXPLAIN_JSON +EXTERNAL_FUNCTIONS_HISTORY +EXTERNAL_TABLE_FILES +EXTERNAL_TABLE_FILE_REGISTRATION_HISTORY +EXTRACT +EXTRACT_ANSWER +EXTRACT_SEMANTIC_CATEGORIES +EXTREME_OUTLIER_COUNT +EXTREME_OUTLIER_IQR_COUNT +EXTREME_OUTLIER_IQR_PERCENT +EXTREME_OUTLIER_PERCENT +EXTREME_OUTLIER_ZSCORE_COUNT +EXTREME_OUTLIER_ZSCORE_PERCENT +FACTORIAL +FILTER +FINETUNE +FIRST_VALUE +FLATTEN +FLOOR +FL_GET_CONTENT_TYPE +FL_GET_ETAG +FL_GET_FILE_TYPE +FL_GET_LAST_MODIFIED +FL_GET_RELATIVE_PATH +FL_GET_SCOPED_FILE_URL +FL_GET_SIZE +FL_GET_STAGE +FL_GET_STAGE_FILE_URL +FL_IS_AUDIO +FL_IS_COMPRESSED +FL_IS_DOCUMENT +FL_IS_IMAGE +FL_IS_VIDEO +FRESHNESS +GENERATE_COLUMN_DESCRIPTION +GENERATE_POSTGRES_ACCESS_TOKEN_FOR_USER +GENERATOR +GET +GETBIT +GETDATE +GETVARIABLE +GET_ABSOLUTE_PATH +GET_AI_EVALUATION_DATA +GET_AI_OBSERVABILITY_LOGS +GET_AI_RECORD_TRACE +GET_ANACONDA_PACKAGES_REPODATA +GET_CONDITION_QUERY_UUID +GET_CONFIGURATION_VALUE +GET_CONTACTS +GET_DDL +GET_IGNORE_CASE +GET_JOB_HISTORY +GET_LINEAGE +GET_OBJECT_REFERENCES +GET_PATH +GET_PRESIGNED_URL +GET_PYTHON_PROFILER_OUTPUT +GET_QUERY_OPERATOR_STATS +GET_RELATIVE_PATH +GET_STAGE_LOCATION +GREATEST +GREATEST_IGNORE_NULLS +GROUPING +GROUPING_ID +H3_CELL_TO_BOUNDARY +H3_CELL_TO_CHILDREN +H3_CELL_TO_CHILDREN_STRING +H3_CELL_TO_PARENT +H3_CELL_TO_POINT +H3_COMPACT_CELLS +H3_COMPACT_CELLS_STRINGS +H3_COVERAGE +H3_COVERAGE_STRINGS +H3_GET_RESOLUTION +H3_GRID_DISK +H3_GRID_DISTANCE +H3_GRID_PATH +H3_INT_TO_STRING +H3_IS_PENTAGON +H3_IS_VALID_CELL +H3_LATLNG_TO_CELL +H3_LATLNG_TO_CELL_STRING +H3_POINT_TO_CELL +H3_POINT_TO_CELL_STRING +H3_POLYGON_TO_CELLS +H3_POLYGON_TO_CELLS_STRINGS +H3_STRING_TO_INT +H3_TRY_COVERAGE +H3_TRY_COVERAGE_STRINGS +H3_TRY_GRID_DISTANCE +H3_TRY_GRID_PATH +H3_TRY_POLYGON_TO_CELLS +H3_TRY_POLYGON_TO_CELLS_STRINGS +H3_UNCOMPACT_CELLS +H3_UNCOMPACT_CELLS_STRINGS +HASH +HASH_AGG +HAVERSINE +HEX_DECODE_BINARY +HEX_DECODE_STRING +HEX_ENCODE +HLL +HLL_ACCUMULATE +HLL_COMBINE +HLL_ESTIMATE +HLL_EXPORT +HLL_IMPORT +HOUR +ICEBERG_TABLE_FILES +ICEBERG_TABLE_SNAPSHOT_REFRESH_HISTORY +IFF +IFNULL +ILIKE +IN +INFER_SCHEMA +INITCAP +INSERT +INTEGRATION +INTERPOLATE_BFILL +INTERPOLATE_FFILL +INTERPOLATE_LINEAR +INVOKER_ROLE +INVOKER_SHARE +IS_APPLICATION_ROLE_ACTIVATED +IS_APPLICATION_ROLE_IN_SESSION +IS_ARRAY +IS_BINARY +IS_BOOLEAN +IS_CHAR +IS_CONFIGURATION_SET +IS_DATABASE_ROLE_ACTIVATED +IS_DATABASE_ROLE_IN_SESSION +IS_DATE +IS_DATE_VALUE +IS_DECIMAL +IS_DOUBLE +IS_GRANTED_TO_INVOKER_ROLE +IS_GROUP_ACTIVATED +IS_GROUP_IMPORTED +IS_INSTANCE_ROLE_IN_SESSION +IS_INTEGER +IS_NULL_VALUE +IS_OBJECT +IS_ORGANIZATION_USER +IS_ORGANIZATION_USER_GROUP +IS_ORGANIZATION_USER_GROUP_IN_SESSION +IS_REAL +IS_ROLE_ACTIVATED +IS_ROLE_IN_SESSION +IS_TIME +IS_TIMESTAMP_LTZ +IS_TIMESTAMP_NTZ +IS_TIMESTAMP_TZ +IS_USER_IMPORTED +IS_VARCHAR +JAROWINKLER_SIMILARITY +JSON_EXTRACT_PATH_TEXT +KURTOSIS +LAG +LAST_DAY +LAST_QUERY_ID +LAST_SUCCESSFUL_SCHEDULED_TIME +LAST_TRANSACTION +LAST_VALUE +LEAD +LEAST +LEAST_IGNORE_NULLS +LEFT +LEN +LENGTH +LIKE +LISTAGG +LISTING_REFRESH_HISTORY +LN +LOCALTIME +LOCALTIMESTAMP +LOG +LOGIN_HISTORY +LOGIN_HISTORY_BY_USER +LOWER +LPAD +LTRIM +MAP_CAT +MAP_CONSTRUCT +MAP_CONTAINS_KEY +MAP_DELETE +MAP_ENTRIES +MAP_INSERT +MAP_KEYS +MAP_PICK +MAP_SIZE +MATERIALIZED_VIEW_REFRESH_HISTORY +MAX +MAX_BY +MD5 +MD5_BINARY +MD5_HEX +MD5_NUMBER_LOWER64 +MD5_NUMBER_UPPER64 +MEDIAN +MIN +MINHASH +MINHASH_COMBINE +MINUTE +MIN_BY +MOD +MODE +MODEL_MONITOR_DRIFT_METRIC +MODEL_MONITOR_PERFORMANCE_METRIC +MODEL_MONITOR_STAT_METRIC +MONTH +MONTHNAME +MONTHS_BETWEEN +NETWORK_RULE_REFERENCES +NEXT_DAY +NORMAL +NOTIFICATION_HISTORY +NTH_VALUE +NTILE +NULLIF +NULLIFZERO +NULL_COUNT +NULL_PERCENT +NVL +NVL2 +OBJECT_AGG +OBJECT_CONSTRUCT +OBJECT_CONSTRUCT_KEEP_NULL +OBJECT_CONSTRUCT_KEEP_NULL_STRUCTURED +OBJECT_DELETE +OBJECT_INSERT +OBJECT_KEYS +OBJECT_PICK +OCTET_LENGTH +ONLINE_FEATURE_TABLE_REFRESH_HISTORY +OUTLIER_COUNT +OUTLIER_IQR_COUNT +OUTLIER_IQR_PERCENT +OUTLIER_PERCENT +OUTLIER_ZSCORE_COUNT +OUTLIER_ZSCORE_PERCENT +PARSE_DOCUMENT +PARSE_IP +PARSE_JSON +PARSE_URL +PARSE_XML +PERCENTILE_CONT +PERCENTILE_DISC +PERCENT_RANK +PI +PIPE_USAGE_HISTORY +POLICY_CONTEXT +POLICY_REFERENCES +POSITION +POW +POWER +PREVIOUS_DAY +PROMPT +QUARTER +QUERY_ACCELERATION_HISTORY +QUERY_HISTORY +RADIANS +RANDOM +RANDSTR +RANK +RATIO_TO_REPORT +REDUCE +REGEXP +REGEXP_COUNT +REGEXP_INSTR +REGEXP_LIKE +REGEXP_REPLACE +REGEXP_SUBSTR +REGEXP_SUBSTR_ALL +REGR_AVGX +REGR_AVGY +REGR_COUNT +REGR_INTERCEPT +REGR_R2 +REGR_SLOPE +REGR_SXX +REGR_SXY +REGR_SYY +REGR_VALX +REGR_VALY +REPEAT +REPLACE +REPLICATION_GROUP_DANGLING_REFERENCES +REPLICATION_GROUP_REFRESH_HISTORY +REPLICATION_GROUP_REFRESH_HISTORY_ALL +REPLICATION_GROUP_REFRESH_PROGRESS +REPLICATION_GROUP_REFRESH_PROGRESS_ALL +REPLICATION_GROUP_REFRESH_PROGRESS_BY_JOB +REPLICATION_GROUP_USAGE_HISTORY +REPLICATION_USAGE_HISTORY +REST_EVENT_HISTORY +RESULT_SCAN +REVERSE +RIGHT +RLIKE +ROUND +ROW_COUNT +ROW_NUMBER +RPAD +RTRIM +RTRIMMED_LENGTH +SANITIZE_WEBHOOK_CONTENT +SCHEDULED_TIME +SEARCH +SEARCH_IP +SEARCH_OPTIMIZATION_HISTORY +SEARCH_PREVIEW +SECOND +SENTIMENT +SEQ1 +SEQ2 +SEQ4 +SEQ8 +SERVERLESS_ALERT_HISTORY +SERVERLESS_TASK_HISTORY +SET_SYS_CONTEXT +SHA1 +SHA1_BINARY +SHA1_HEX +SHA2 +SHA2_BINARY +SHA2_HEX +SHOW_PYTHON_PACKAGES_DEPENDENCIES +SIGN +SIN +SINH +SKEW +SOUNDEX +SOUNDEX_P123 +SPACE +SPLIT +SPLIT_PART +SPLIT_TEXT_MARKDOWN_HEADER +SPLIT_TEXT_RECURSIVE_CHARACTER +SPLIT_TO_TABLE +SQRT +SQUARE +STAGE_DIRECTORY_FILE_REGISTRATION_HISTORY +STAGE_STORAGE_USAGE_HISTORY +STARTSWITH +STDDEV +STDDEV_POP +STDDEV_SAMP +STORAGE_LIFECYCLE_POLICY_HISTORY +STRIP_NULL_VALUE +STRTOK +STRTOK_SPLIT_TO_TABLE +STRTOK_TO_ARRAY +ST_AREA +ST_ASBINARY +ST_ASEWKB +ST_ASEWKT +ST_ASGEOJSON +ST_ASTEXT +ST_ASWKB +ST_ASWKT +ST_AZIMUTH +ST_BUFFER +ST_CENTROID +ST_COLLECT +ST_CONTAINS +ST_COVEREDBY +ST_COVERS +ST_DIFFERENCE +ST_DIMENSION +ST_DISJOINT +ST_DISTANCE +ST_DWITHIN +ST_ENDPOINT +ST_ENVELOPE +ST_GEOGFROMGEOHASH +ST_GEOGPOINTFROMGEOHASH +ST_GEOGRAPHYFROMWKB +ST_GEOGRAPHYFROMWKT +ST_GEOHASH +ST_GEOMETRYFROMWKB +ST_GEOMETRYFROMWKT +ST_GEOMFROMGEOHASH +ST_GEOMPOINT +ST_GEOMPOINTFROMGEOHASH +ST_HAUSDORFFDISTANCE +ST_INTERPOLATE +ST_INTERSECTION +ST_INTERSECTION_AGG +ST_INTERSECTS +ST_ISVALID +ST_LENGTH +ST_MAKEGEOMPOINT +ST_MAKELINE +ST_MAKEPOINT +ST_MAKEPOLYGON +ST_MAKEPOLYGONORIENTED +ST_NPOINTS +ST_NUMPOINTS +ST_PERIMETER +ST_POINT +ST_POINTN +ST_POLYGON +ST_SETSRID +ST_SIMPLIFY +ST_SRID +ST_STARTPOINT +ST_SYMDIFFERENCE +ST_TRANSFORM +ST_UNION +ST_UNION_AGG +ST_WITHIN +ST_X +ST_XMAX +ST_XMIN +ST_Y +ST_YMAX +ST_YMIN +SUBSTR +SUBSTRING +SUM +SYSDATE +SYSTEM$ABORT_SESSION +SYSTEM$ABORT_TRANSACTION +SYSTEM$ACTIVATE_CMK_INFO +SYSTEM$ACTIVATE_CMK_INFO_POSTGRES +SYSTEM$ADD_EVENT +SYSTEM$ADD_REFERENCE +SYSTEM$ALLOWLIST +SYSTEM$ALLOWLIST_PRIVATELINK +SYSTEM$APPLICATION_GET_LOG_LEVEL +SYSTEM$APPLICATION_GET_METRIC_LEVEL +SYSTEM$APPLICATION_GET_TRACE_LEVEL +SYSTEM$APP_COMPATIBILITY_CHECK +SYSTEM$AUTHORIZE_PRIVATELINK +SYSTEM$AUTHORIZE_STAGE_PRIVATELINK_ACCESS +SYSTEM$AUTO_REFRESH_STATUS +SYSTEM$BEGIN_DEBUG_APPLICATION +SYSTEM$BEHAVIOR_CHANGE_BUNDLE_STATUS +SYSTEM$BLOCK_INTERNAL_STAGES_PUBLIC_ACCESS +SYSTEM$BLOCK_INTERNAL_STAGES_PUBLIC_ACCESS_WITH_EXCEPTION +SYSTEM$CANCEL_ALL_QUERIES +SYSTEM$CANCEL_QUERY +SYSTEM$CATALOG_LINK_STATUS +SYSTEM$CKE_HASH_FUNCTION +SYSTEM$CLEANUP_DATABASE_ROLE_GRANTS +SYSTEM$CLIENT_VERSION_INFO +SYSTEM$CLIENT_VULNERABILITY_INFO +SYSTEM$CLUSTERING_DEPTH +SYSTEM$CLUSTERING_INFORMATION +SYSTEM$COMMIT_MOVE_ORGANIZATION_ACCOUNT +SYSTEM$CONVERT_PIPES_SQS_TO_SNS +SYSTEM$CREATE_BILLING_EVENT +SYSTEM$CREATE_BILLING_EVENTS +SYSTEM$CURRENT_USER_TASK_NAME +SYSTEM$DATABASE_REFRESH_PROGRESS +SYSTEM$DATA_METRIC_SCAN +SYSTEM$DEACTIVATE_CMK_INFO +SYSTEM$DECODE_PAT +SYSTEM$DEPROVISION_PRIVATELINK_ENDPOINT +SYSTEM$DEPROVISION_PRIVATELINK_ENDPOINT_TSS +SYSTEM$DEREGISTER_CMK_INFO +SYSTEM$DEREGISTER_CMK_INFO_POSTGRES +SYSTEM$DESC_ICEBERG_ACCESS_IDENTITY +SYSTEM$DISABLE_BEHAVIOR_CHANGE_BUNDLE +SYSTEM$DISABLE_DATABASE_REPLICATION +SYSTEM$DISABLE_GLOBAL_DATA_SHARING_FOR_ACCOUNT +SYSTEM$DISABLE_PREVIEW_ACCESS +SYSTEM$DISABLE_PRIVATELINK_ACCESS_ONLY +SYSTEM$ENABLE_BEHAVIOR_CHANGE_BUNDLE +SYSTEM$ENABLE_GLOBAL_DATA_SHARING_FOR_ACCOUNT +SYSTEM$ENABLE_PREVIEW_ACCESS +SYSTEM$ENCODE_CKE_PRIMARY_KEY +SYSTEM$END_DEBUG_APPLICATION +SYSTEM$ENFORCE_PRIVATELINK_ACCESS_ONLY +SYSTEM$ESTIMATE_AUTOMATIC_CLUSTERING_COSTS +SYSTEM$ESTIMATE_QUERY_ACCELERATION +SYSTEM$ESTIMATE_SEARCH_OPTIMIZATION_COSTS +SYSTEM$EVALUATE_DATA_QUALITY_EXPECTATIONS +SYSTEM$EVALUATE_DATA_QUALITY_EXPECTATIONS_PERSIST_RESULT +SYSTEM$EXPLAIN_JSON_TO_TEXT +SYSTEM$EXPLAIN_PLAN_JSON +SYSTEM$EXPORT_TDS_FROM_SEMANTIC_VIEW +SYSTEM$EXTERNAL_TABLE_PIPE_STATUS +SYSTEM$FINISH_OAUTH_FLOW +SYSTEM$GENERATE_SAML_CSR +SYSTEM$GENERATE_SCIM_ACCESS_TOKEN +SYSTEM$GET_ALL_DEFAULT_COLUMNS_OVERRIDES +SYSTEM$GET_ALL_REFERENCES +SYSTEM$GET_AWS_SNS_IAM_POLICY +SYSTEM$GET_CATALOG_LINKED_DATABASE_CONFIG +SYSTEM$GET_CLASSIFICATION_RESULT +SYSTEM$GET_CMK_AKV_CONSENT_URL +SYSTEM$GET_CMK_CONFIG +SYSTEM$GET_CMK_CONFIG_POSTGRES +SYSTEM$GET_CMK_INFO +SYSTEM$GET_CMK_INFO_POSTGRES +SYSTEM$GET_CMK_KMS_KEY_POLICY +SYSTEM$GET_COMPUTE_POOL_PENDING_MAINTENANCE +SYSTEM$GET_DBT_LOG +SYSTEM$GET_DEBUG_STATUS +SYSTEM$GET_DEFAULT_COLUMNS_OVERRIDE_FOR_SHOW_COMMAND +SYSTEM$GET_DEFAULT_COLUMNS_OVERRIDE_FOR_SYSTEM_OBJECT +SYSTEM$GET_DIRECTORY_TABLE_STATUS +SYSTEM$GET_GCP_KMS_CMK_GRANT_ACCESS_CMD +SYSTEM$GET_HASH_FOR_APPLICATION +SYSTEM$GET_ICEBERG_TABLE_INFORMATION +SYSTEM$GET_INSTANCE_FAMILY_PLACEMENT_GROUPS +SYSTEM$GET_LOGIN_FAILURE_DETAILS +SYSTEM$GET_PREDECESSOR_RETURN_VALUE +SYSTEM$GET_PREVIEW_ACCESS_STATUS +SYSTEM$GET_PRIVATELINK +SYSTEM$GET_PRIVATELINK_AUTHORIZED_ENDPOINTS +SYSTEM$GET_PRIVATELINK_CONFIG +SYSTEM$GET_PRIVATELINK_ENDPOINTS_INFO +SYSTEM$GET_PRIVATELINK_ENDPOINT_REGISTRATIONS +SYSTEM$GET_PURCHASE_ATTRIBUTES +SYSTEM$GET_REFERENCED_OBJECT_ID_HASH +SYSTEM$GET_RESULTSET_STATUS +SYSTEM$GET_SERVICE_DNS_DOMAIN +SYSTEM$GET_SERVICE_LOGS +SYSTEM$GET_SNOWFLAKE_EGRESS_IP_RANGES +SYSTEM$GET_SNOWFLAKE_PLATFORM_INFO +SYSTEM$GET_STAGE_PRIVATELINK_AUTHORIZED_ENDPOINTS +SYSTEM$GET_TAG +SYSTEM$GET_TAG_ALLOWED_VALUES +SYSTEM$GET_TAG_ON_CURRENT_COLUMN +SYSTEM$GET_TAG_ON_CURRENT_TABLE +SYSTEM$GET_TASK_GRAPH_CONFIG +SYSTEM$GLOBAL_ACCOUNT_SET_PARAMETER +SYSTEM$HOLD_PRIVILEGE_ON_ACCOUNT +SYSTEM$INITIATE_MOVE_ORGANIZATION_ACCOUNT +SYSTEM$INTERNAL_STAGES_PUBLIC_ACCESS_STATUS +SYSTEM$IS_APPLICATION_ALL_MANDATORY_TELEMETRY_EVENT_DEFINITIONS_ENABLED +SYSTEM$IS_APPLICATION_AUTHORIZED_FOR_TELEMETRY_EVENT_SHARING +SYSTEM$IS_APPLICATION_INSTALLED_FROM_SAME_ACCOUNT +SYSTEM$IS_APPLICATION_SHARING_EVENTS_WITH_PROVIDER +SYSTEM$IS_GLOBAL_DATA_SHARING_ENABLED_FOR_ACCOUNT +SYSTEM$IS_LISTING_PURCHASED +SYSTEM$IS_LISTING_TRIAL +SYSTEM$LAST_CHANGE_COMMIT_TIME +SYSTEM$LINK_ACCOUNT_OBJECTS_BY_NAME +SYSTEM$LINK_ORGANIZATION_USER +SYSTEM$LINK_ORGANIZATION_USER_GROUP +SYSTEM$LIST_APPLICATION_RESTRICTED_FEATURES +SYSTEM$LIST_ICEBERG_TABLES_FROM_CATALOG +SYSTEM$LIST_NAMESPACES_FROM_CATALOG +SYSTEM$LOCATE_DBT_ARCHIVE +SYSTEM$LOCATE_DBT_ARTIFACTS +SYSTEM$LOG +SYSTEM$MIGRATE_SAML_IDP_REGISTRATION +SYSTEM$OPT_IN_INTERNAL_STAGE_NETWORK_LOGS +SYSTEM$OPT_OUT_INTERNAL_STAGE_NETWORK_LOGS +SYSTEM$OPT_OUT_MALICIOUS_IP_PROTECTION_BY_CATEGORY +SYSTEM$PIPE_FORCE_RESUME +SYSTEM$PIPE_REBINDING_WITH_NOTIFICATION_CHANNEL +SYSTEM$PIPE_STATUS +SYSTEM$PROVISION_PRIVATELINK_ENDPOINT +SYSTEM$PROVISION_PRIVATELINK_ENDPOINT_TSS +SYSTEM$QUERY_REFERENCE +SYSTEM$READ_OSI_YAML_FROM_SEMANTIC_VIEW +SYSTEM$READ_YAML_FROM_SEMANTIC_VIEW +SYSTEM$REFERENCE +SYSTEM$REGISTER_CMK_INFO +SYSTEM$REGISTER_CMK_INFO_POSTGRES +SYSTEM$REGISTER_PRIVATELINK_ENDPOINT +SYSTEM$REMOVE_ALL_REFERENCES +SYSTEM$REMOVE_REFERENCE +SYSTEM$REPORT_HEALTH_STATUS +SYSTEM$RESOLVE_PYTHON_PACKAGES +SYSTEM$RESTORE_PRIVATELINK_ENDPOINT +SYSTEM$RESTORE_PRIVATELINK_ENDPOINT_TSS +SYSTEM$REVOKE_PRIVATELINK +SYSTEM$REVOKE_STAGE_PRIVATELINK_ACCESS +SYSTEM$SCHEDULE_ASYNC_REPLICATION_GROUP_REFRESH +SYSTEM$SEND_NOTIFICATIONS_TO_CATALOG +SYSTEM$SET_APPLICATION_RESTRICTED_FEATURE_ACCESS +SYSTEM$SET_CATALOG_INTEGRATION +SYSTEM$SET_DEFAULT_COLUMNS_OVERRIDE_FOR_SHOW_COMMAND +SYSTEM$SET_DEFAULT_COLUMNS_OVERRIDE_FOR_SYSTEM_OBJECT +SYSTEM$SET_EVENT_SHARING_ACCOUNT_FOR_REGION +SYSTEM$SET_PRIVATELINK_ENDPOINT_HOSTNAME +SYSTEM$SET_REFERENCE +SYSTEM$SET_RETURN_VALUE +SYSTEM$SET_ROW_TIMESTAMP_ON_ALL_SUPPORTED_TABLES +SYSTEM$SET_SPAN_ATTRIBUTES +SYSTEM$SHOW_ACTIVE_BEHAVIOR_CHANGE_BUNDLES +SYSTEM$SHOW_BUDGETS_FOR_RESOURCE +SYSTEM$SHOW_BUDGETS_IN_ACCOUNT +SYSTEM$SHOW_EVENT_SHARING_ACCOUNTS +SYSTEM$SHOW_MOVE_ORGANIZATION_ACCOUNT_STATUS +SYSTEM$SHOW_OAUTH_CLIENT_SECRETS +SYSTEM$SHOW_SENSITIVE_DATA_MONITORED_ENTITIES +SYSTEM$SNOWPIPE_STREAMING_UPDATE_CHANNEL_OFFSET_TOKEN +SYSTEM$START_OAUTH_FLOW +SYSTEM$START_USER_EMAIL_VERIFICATION +SYSTEM$STREAM_BACKLOG +SYSTEM$STREAM_GET_TABLE_TIMESTAMP +SYSTEM$STREAM_HAS_DATA +SYSTEM$SUPPORTED_DBT_VERSIONS +SYSTEM$TASK_DEPENDENTS_ENABLE +SYSTEM$TASK_RUNTIME_INFO +SYSTEM$TRIGGER_LISTING_REFRESH +SYSTEM$TYPEOF +SYSTEM$UNBLOCK_INTERNAL_STAGES_PUBLIC_ACCESS +SYSTEM$UNLINK_ORGANIZATION_USER +SYSTEM$UNLINK_ORGANIZATION_USER_GROUP +SYSTEM$UNREGISTER_PRIVATELINK_ENDPOINT +SYSTEM$UNSET_DEFAULT_COLUMNS_OVERRIDE_FOR_SHOW_COMMAND +SYSTEM$UNSET_DEFAULT_COLUMNS_OVERRIDE_FOR_SYSTEM_OBJECT +SYSTEM$UNSET_EVENT_SHARING_ACCOUNT_FOR_REGION +SYSTEM$USER_TASK_CANCEL_ONGOING_EXECUTIONS +SYSTEM$VALIDATE_STORAGE_INTEGRATION +SYSTEM$VERIFY_CATALOG_INTEGRATION +SYSTEM$VERIFY_CMK_INFO +SYSTEM$VERIFY_CMK_INFO_POSTGRES +SYSTEM$VERIFY_EXTERNAL_OAUTH_TOKEN +SYSTEM$VERIFY_EXTERNAL_VOLUME +SYSTEM$WAIT +SYSTEM$WAIT_FOR_SERVICES +SYSTIMESTAMP +SYS_CONNECT_BY_PATH +SYS_CONTEXT +TAG_REFERENCES +TAG_REFERENCES_ALL_COLUMNS +TAG_REFERENCES_WITH_LINEAGE +TAN +TANH +TASK_DEPENDENTS +TASK_HISTORY +TEXT_HTML +TEXT_PLAIN +TIME +TIMEADD +TIMEDIFF +TIMESTAMPADD +TIMESTAMPDIFF +TIMESTAMP_FROM_PARTS +TIME_FROM_PARTS +TIME_SLICE +TO_ARRAY +TO_BINARY +TO_BOOLEAN +TO_CHAR +TO_DATE +TO_DECFLOAT +TO_DECIMAL +TO_DOUBLE +TO_FILE +TO_GEOGRAPHY +TO_GEOMETRY +TO_JSON +TO_NUMBER +TO_NUMERIC +TO_OBJECT +TO_QUERY +TO_TIME +TO_TIMESTAMP +TO_TIMESTAMP_LTZ +TO_TIMESTAMP_NTZ +TO_TIMESTAMP_TZ +TO_UUID +TO_VARCHAR +TO_VARIANT +TO_XML +TRANSFORM +TRANSLATE +TRIM +TRUNC +TRUNCATE +TRY_BASE64_DECODE_BINARY +TRY_BASE64_DECODE_STRING +TRY_CAST +TRY_COMPLETE +TRY_DECRYPT +TRY_DECRYPT_RAW +TRY_HEX_DECODE_BINARY +TRY_HEX_DECODE_STRING +TRY_PARSE_JSON +TRY_TO_BINARY +TRY_TO_BOOLEAN +TRY_TO_DATE +TRY_TO_DECFLOAT +TRY_TO_DECIMAL +TRY_TO_DOUBLE +TRY_TO_FILE +TRY_TO_GEOGRAPHY +TRY_TO_GEOMETRY +TRY_TO_NUMBER +TRY_TO_NUMERIC +TRY_TO_TIME +TRY_TO_TIMESTAMP +TRY_TO_TIMESTAMP_LTZ +TRY_TO_TIMESTAMP_NTZ +TRY_TO_TIMESTAMP_TZ +TRY_TO_UUID +TYPEOF +UNICODE +UNIFORM +UNIQUE_COUNT +UPPER +UUID_STRING +VALIDATE +VALIDATE_PIPE_LOAD +VARIANCE +VARIANCE_POP +VARIANCE_SAMP +VAR_POP +VAR_SAMP +VECTOR_AVG +VECTOR_COSINE_SIMILARITY +VECTOR_INNER_PRODUCT +VECTOR_L1_DISTANCE +VECTOR_L2_DISTANCE +VECTOR_MAX +VECTOR_MIN +VECTOR_NORMALIZE +VECTOR_SUM +VECTOR_TRUNCATE +WAREHOUSE_LOAD_HISTORY +WAREHOUSE_METERING_HISTORY +WEEK +WEEKISO +WEEKOFYEAR +WIDTH_BUCKET +XMLGET +YEAR +YEAROFWEEK +YEAROFWEEKISO +ZEROIFNULL +ZIPF diff --git a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs index a61408c6cc..28598b4d26 100644 --- a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs +++ b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs @@ -136,6 +136,14 @@ fn format_sql_batch_json(py: Python<'_>, request_json: &str) -> PyResult .map_err(value_error) } +#[pyfunction] +fn query_fingerprint(py: Python<'_>, sql: &str, dialect: &str) -> PyResult { + py.compiler_detach(|| { + crate::sql_tokens::main::query_fingerprint::query_fingerprint(sql, dialect) + }) + .map_err(value_error) +} + #[pyfunction(name = "validate_sql_with_schema_json")] fn schema_validation_json(py: Python<'_>, request_json: &str) -> PyResult { py.compiler_detach(|| crate::semantic_validation::main::validation_json(request_json)) @@ -431,6 +439,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(lint_backtick_identifiers, module)?)?; module.add_function(wrap_pyfunction!(lint_sql_batch_json, module)?)?; module.add_function(wrap_pyfunction!(format_sql_json, module)?)?; + module.add_function(wrap_pyfunction!(query_fingerprint, module)?)?; module.add_function(wrap_pyfunction!(format_sql_batch_json, module)?)?; module.add_function(wrap_pyfunction!(schema_validation_json, module)?)?; module.add_function(wrap_pyfunction!(schema_validations_json, module)?)?; diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_sql_scanning.rs b/crates/sqlbuild-rules-native/src/engine/tests/test_sql_scanning.rs index 5d4607b0b3..afbd58d470 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/test_sql_scanning.rs +++ b/crates/sqlbuild-rules-native/src/engine/tests/test_sql_scanning.rs @@ -1,5 +1,7 @@ use crate::compiler::_helpers::sql_references::extraction::extract; -use crate::engine::tests::test_types::{LintDialectSiteTestCase, SqlScannerTestCase}; +use crate::engine::tests::test_types::{ + DollarQuoteSiteTestCase, LintDialectSiteTestCase, SqlScannerTestCase, +}; use crate::rules::_helpers::evaluation::{normalize_rules_sql, rules_quote_policy}; use crate::rules::_helpers::numeric_decisions::compact_sql; use crate::sql_lint::_helpers::preparation::prepare; @@ -177,7 +179,7 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann expected_snowflake_comma: "SELECT $$ ) $$ a FROM t", expected_compact: "a=$$)$$andb", expected_snowflake_compact: "a=$$)$$andb", - expected_lint_site: Some("@m($$ )"), + expected_lint_site: Some("@m($$ ) $$ x)"), expected_reference_fast_path: true, }, SqlScannerTestCase { @@ -339,6 +341,18 @@ fn given_lint_dialect_when_preparing_macro_sites_then_backticks_follow_rules_quo fragment: "`a)b`", expected_lint_site: "@m(`a)", }, + LintDialectSiteTestCase { + description: "duckdb dollar quote hides an apostrophe and close paren", + dialect: "duckdb", + fragment: "$$Customer's order)$$", + expected_lint_site: "@m($$Customer's order)$$ x)", + }, + LintDialectSiteTestCase { + description: "snowflake tagged dollar quote hides a nested dollar quote", + dialect: "snowflake", + fragment: "$note$ $$ ) $note$", + expected_lint_site: "@m($note$ $$ ) $note$ x)", + }, LintDialectSiteTestCase { description: "sqlserver backtick remains code", dialect: "tsql", @@ -359,3 +373,33 @@ fn given_lint_dialect_when_preparing_macro_sites_then_backticks_follow_rules_quo } Ok(()) } + +#[test] +fn given_dollar_quoted_apostrophe_when_preparing_lint_sites_then_later_references_are_sites() +-> Result<(), String> { + let test_cases = [ + DollarQuoteSiteTestCase { + description: "duckdb untagged and tagged quotes", + dialect: "duckdb", + expected_sites: &["__ref(\"orders\")"], + }, + DollarQuoteSiteTestCase { + description: "postgres untagged and tagged quotes", + dialect: "postgres", + expected_sites: &["__ref(\"orders\")"], + }, + DollarQuoteSiteTestCase { + description: "snowflake untagged and tagged quotes", + dialect: "snowflake", + expected_sites: &["__ref(\"orders\")"], + }, + ]; + let sql = "SELECT $$Customer's order$$ AS note, $tag$it's$tag$ AS tag FROM __ref(\"orders\") o"; + for test_case in test_cases { + let sites: Vec = prepare(sql, sql, &[], test_case.dialect)? + .map(|prepared| prepared.1.into_iter().map(|site| site.5).collect()) + .unwrap_or_default(); + assert_eq!(sites, test_case.expected_sites, "{}", test_case.description); + } + Ok(()) +} diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs b/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs index ab954a9371..bf1a75ccb9 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs @@ -169,6 +169,12 @@ pub(crate) struct EmptyInputMinimumTestsTestCase { } /// One SQL fragment scanned by every native quote- and comment-aware scanner. +pub(crate) struct DollarQuoteSiteTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static str, + pub(crate) expected_sites: &'static [&'static str], +} + pub(crate) struct LintDialectSiteTestCase { pub(crate) description: &'static str, pub(crate) dialect: &'static str, diff --git a/crates/sqlbuild-rules-native/src/lib.rs b/crates/sqlbuild-rules-native/src/lib.rs index 056f1dd5f9..2b36ba76d5 100644 --- a/crates/sqlbuild-rules-native/src/lib.rs +++ b/crates/sqlbuild-rules-native/src/lib.rs @@ -14,3 +14,4 @@ mod semantic_usage; mod semantic_validation; mod sql_lint; mod sql_scan; +mod sql_tokens; diff --git a/crates/sqlbuild-rules-native/src/query_analysis/_helpers/engine.rs b/crates/sqlbuild-rules-native/src/query_analysis/_helpers/engine.rs index 2169948894..98955b002b 100644 --- a/crates/sqlbuild-rules-native/src/query_analysis/_helpers/engine.rs +++ b/crates/sqlbuild-rules-native/src/query_analysis/_helpers/engine.rs @@ -496,8 +496,12 @@ fn analyze_compact_query_work( if let (Some(sql), Ok(dialect)) = (diagnostic_sql, dialect) && let Some(validation) = result.validation.take() { + let local_probes = crate::semantic_validation::models::FunctionProbes::default(); + let probes = catalog.map_or(&local_probes, |catalog| &catalog.function_probes); result.validation = Some(validation.and_then(|value| { - crate::semantic_validation::main::map_diagnostics::map_diagnostics(&sql, dialect, value) + crate::semantic_validation::main::map_diagnostics::map_diagnostics( + &sql, dialect, value, probes, + ) })); } result diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/catalog.rs b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/catalog.rs index 5b88c77840..3ece2a48b9 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/catalog.rs +++ b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/catalog.rs @@ -4,7 +4,7 @@ use crate::bindings::main::compiler_error::compiler_error; use crate::bindings::types::CompilerDetach; use std::collections::HashMap; -use crate::semantic_validation::models::{CatalogInput, Columns, ProjectCatalog}; +use crate::semantic_validation::models::{CatalogInput, Columns, FunctionProbes, ProjectCatalog}; use crate::semantic_validation::types::{BindingRequest, DiagnosticRow, Relations}; use crate::semantic_validation::{ _helpers::{diagnostics, identifiers}, @@ -129,6 +129,7 @@ impl ProjectCatalog { tables: HashMap::new(), overrides: Vec::new(), analysis_tables: HashMap::new(), + function_probes: FunctionProbes::default(), }; catalog.update_relations(relations); Ok(catalog) @@ -148,6 +149,7 @@ impl ProjectCatalog { tables: self.tables.clone(), overrides: Vec::new(), analysis_tables: self.analysis_tables.clone(), + function_probes: FunctionProbes::default(), }; catalog.update_relations(relations); catalog @@ -229,7 +231,12 @@ impl ProjectCatalog { .map(|(sql, references, overrides)| { let schema = self.schema(&references, overrides)?; let result = self.validate(&sql, &schema)?; - let result = diagnostics::map_diagnostics(&sql, self.dialect, result)?; + let result = diagnostics::map_diagnostics( + &sql, + self.dialect, + result, + &self.function_probes, + )?; Ok(result.errors.into_iter().map(diagnostic_row).collect()) }) .collect::, String>>() @@ -252,6 +259,7 @@ impl ProjectCatalog { tables: HashMap::new(), overrides: Vec::new(), analysis_tables: HashMap::new(), + function_probes: FunctionProbes::default(), } } pub(crate) fn analysis_schema( diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/diagnostics.rs b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/diagnostics.rs index 2dae13a1f8..b4945081ab 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/diagnostics.rs +++ b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/diagnostics.rs @@ -1,6 +1,8 @@ //! SQLBuild codes and dialect-specific diagnostic evidence. use crate::semantic_validation::_helpers::catalog::diagnostic_row; +use crate::semantic_validation::_helpers::function_names; +use crate::semantic_validation::models::FunctionProbes; use crate::semantic_validation::types::DiagnosticRow; use polyglot_sql::expressions::Select; use polyglot_sql::{ @@ -144,6 +146,7 @@ pub(crate) fn map_diagnostics( sql: &str, dialect: DialectType, result: ValidationResult, + probes: &FunctionProbes, ) -> Result { let mut context = DiagnosticContext { sql, @@ -156,6 +159,9 @@ pub(crate) fn map_diagnostics( errors.push(error); } } + errors.extend(function_names::unsupported_function_errors( + sql, dialect, probes, + )); let valid = !errors .iter() .any(|error| error.severity == ValidationSeverity::Error); @@ -405,6 +411,7 @@ pub(crate) fn binding_diagnostics( valid: false, errors, }, + &FunctionProbes::default(), )?; Ok(result.errors.into_iter().map(diagnostic_row).collect()) } diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/function_names.rs b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/function_names.rs new file mode 100644 index 0000000000..858cf88e53 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/function_names.rs @@ -0,0 +1,537 @@ +//! Built-in function spellings checked against the spellings the target dialect accepts. + +use crate::semantic_validation::models::FunctionProbes; +use crate::semantic_validation::types::ProbeKey; +use crate::sql_scan::main::comment_end::comment_end; +use crate::sql_scan::main::non_code_end::non_code_end; +use crate::sql_scan::models::{QuotePolicy, Unclosed}; +use polyglot_sql::{Dialect, DialectType, Expression, ValidationError}; +use polyglot_sql_function_catalogs::{CatalogSink, FunctionNameCase, FunctionSignature}; +use std::collections::{HashMap, HashSet}; +use std::mem::Discriminant; +use std::sync::{LazyLock, OnceLock, PoisonError}; + +const UNKNOWN_FUNCTION: &str = "B101"; +const SNOWFLAKE_FUNCTIONS: &str = include_str!("../../../function_names/snowflake.txt"); +const DUCKDB_CATALOG: &str = "duckdb"; +const PROBE_PREFIX: &str = "SELECT "; +const TYPE_POSITION_KEYWORD: &str = "AS"; +const PROBE_ARITIES: usize = 9; +/// Keywords that may precede a parenthesis without being a function call. +const NON_CALL_KEYWORDS: &str = "ALL AND ANY AS ASOF AT BEFORE BETWEEN BY CASE CHANGES CONNECT CUBE \ + DEFINE DISTINCT DISTINCTROW ELSE END EXCEPT EXCLUDE EXISTS FILTER FROM HAVING ILIKE IN \ + INTERSECT INTERVAL IS JOIN LATERAL LIKE MATCH_CONDITION MATCH_RECOGNIZE MEASURES MINUS NOT ON \ + OR OVER PARTITION PATTERN PERCENT PIVOT PRIOR QUALIFY RANGE RENAME ROLLUP ROW ROWS SAMPLE \ + SELECT SETS SOME TABLE TABLESAMPLE THEN UNION UNPIVOT USING VALUES WHEN WHERE WITH WITHIN"; + +/// Words after which an expression, and so a function call, may start. +const EXPRESSION_KEYWORDS: &str = "ALL AND ANY BETWEEN BY CASE DISTINCT ELSE ELSEIF ESCAPE HAVING \ + ILIKE INTERVAL IS LATERAL LIKE LIMIT MEASURES NOT OFFSET ON OR PRIOR QUALIFY REGEXP RETURN \ + RLIKE SELECT SET SOME THEN TOP WHEN WHERE WITH"; +/// Clause keywords that end a `FROM` list or a `DEFINE` list at the same nesting depth. +const CLAUSE_KEYWORD_LIST: &str = "CONNECT EXCEPT FETCH GROUP HAVING INTERSECT LIMIT \ + MATCH_RECOGNIZE MEASURES MINUS OFFSET ON ORDER PATTERN PIVOT QUALIFY SAMPLE START TABLESAMPLE \ + UNION UNPIVOT USING WHERE WINDOW"; +const FROM_KEYWORD: &str = "FROM"; +const NOT_KEYWORD: &str = "NOT"; +const MATERIALIZED_KEYWORD: &str = "MATERIALIZED"; +const DISTINCT_KEYWORD: &str = "DISTINCT"; + +type SpellingIndex = HashMap>; + +static KEYWORDS: LazyLock> = + LazyLock::new(|| NON_CALL_KEYWORDS.split_whitespace().collect()); +static EXPRESSION_STARTS: LazyLock> = + LazyLock::new(|| EXPRESSION_KEYWORDS.split_whitespace().collect()); +static CLAUSE_KEYWORDS: LazyLock> = + LazyLock::new(|| CLAUSE_KEYWORD_LIST.split_whitespace().collect()); +static ROOT_FRAME: Frame = Frame { + select_seen: false, + clause: Clause::Other, +}; +static SNOWFLAKE: LazyLock> = LazyLock::new(|| accepted_names(SNOWFLAKE_FUNCTIONS)); +static DUCKDB: LazyLock> = LazyLock::new(|| { + let mut sink: CatalogNames = CatalogNames::default(); + polyglot_sql_function_catalogs::register_enabled_catalogs(&mut sink); + sink.names +}); +static SNOWFLAKE_INDEX: [OnceLock; PROBE_ARITIES] = + [const { OnceLock::new() }; PROBE_ARITIES]; +static DUCKDB_INDEX: [OnceLock; PROBE_ARITIES] = + [const { OnceLock::new() }; PROBE_ARITIES]; + +/// One call site whose authored built-in spelling the dialect does not accept. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct UnsupportedCall { + pub(crate) name: String, + pub(crate) start: usize, + pub(crate) end: usize, + pub(crate) suggestion: String, +} + +/// The clause a scan is inside at one parenthesis depth. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +enum Clause { + #[default] + Other, + From, + Define, +} + +/// Lexical state at one parenthesis depth. +#[derive(Debug, Default)] +struct Frame { + select_seen: bool, + clause: Clause, +} + +/// Lexical context of the previous token, used to tell calls from relation and column lists. +struct Scan { + frames: Vec, + previous_word: Option, + word_before: Option, + previous_byte: Option, + path_colon: bool, +} + +/// How the parser represents one probe call. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +enum Signature { + /// A generic function node with this upper-case name. + Named(String), + /// A typed built-in node, identified by its expression variant. + Typed(Discriminant), +} + +/// Upper-case DuckDB names from the SQL library's introspected function catalogue. +#[derive(Default)] +struct CatalogNames { + names: HashSet, +} + +/// Incremental byte-to-character position, so locating every call stays linear in the SQL. +struct Cursor { + byte: usize, + characters: usize, + line: usize, + column: usize, +} + +impl CatalogSink for CatalogNames { + fn set_dialect_name_case(&mut self, _dialect: &'static str, _name_case: FunctionNameCase) {} + + fn set_function_name_case( + &mut self, + _dialect: &'static str, + _function_name: &str, + _name_case: FunctionNameCase, + ) { + } + + fn register( + &mut self, + dialect: &'static str, + function_name: &str, + _signatures: Vec, + ) { + if dialect == DUCKDB_CATALOG && is_identifier(function_name) { + self.names.insert(function_name.to_ascii_uppercase()); + } + } +} + +impl Cursor { + fn new() -> Self { + Self { + byte: 0, + characters: 0, + line: 1, + column: 1, + } + } + + fn advance(&mut self, sql: &str, byte: usize) { + for character in sql[self.byte..byte].chars() { + self.characters += 1; + if character == '\n' { + self.line += 1; + self.column = 1; + } else { + self.column += 1; + } + } + self.byte = byte; + } +} + +impl FunctionProbes { + /// Return the accepted spelling for an unaccepted call-site name the parser treats as built-in. + fn suggestion(&self, dialect: DialectType, upper: &str, arity: usize) -> Option { + let key: ProbeKey = (dialect, upper.to_owned(), arity.min(PROBE_ARITIES - 1)); + if let Some(cached) = self + .suggestions + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(&key) + { + return cached.clone(); + } + let found: Option = closest_spelling(dialect, upper, key.2); + self.suggestions + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(key, found.clone()); + found + } +} + +fn accepted_names(source: &str) -> HashSet { + source + .lines() + .map(str::trim) + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .map(str::to_ascii_uppercase) + .collect() +} + +fn accepted(dialect: DialectType) -> Option<&'static HashSet> { + match dialect { + DialectType::Snowflake => Some(&SNOWFLAKE), + DialectType::DuckDB => Some(&DUCKDB), + _ => None, + } +} + +/// Return located B101 diagnostics, with character spans, for unsupported built-in spellings. +pub(crate) fn unsupported_function_errors( + sql: &str, + dialect: DialectType, + probes: &FunctionProbes, +) -> Vec { + let mut cursor: Cursor = Cursor::new(); + unsupported_calls(sql, dialect, probes) + .into_iter() + .map(|call| { + let message: String = format!( + "Unknown function '{}' for dialect {dialect:?}; did you mean {}?", + call.name, call.suggestion + ); + cursor.advance(sql, call.start); + let start: usize = cursor.characters; + let end: usize = start + sql[call.start..call.end].chars().count(); + ValidationError::error(message, UNKNOWN_FUNCTION) + .with_location(cursor.line, cursor.column) + .with_span(Some(start), Some(end)) + }) + .collect() +} + +/// Return call sites in `sql` whose built-in spelling `dialect` does not accept. +pub(crate) fn unsupported_calls( + sql: &str, + dialect: DialectType, + probes: &FunctionProbes, +) -> Vec { + let Some(names) = accepted(dialect) else { + return Vec::new(); + }; + let bytes: &[u8] = sql.as_bytes(); + let mut calls: Vec = Vec::new(); + let mut scan: Scan = Scan::new(); + let mut index: usize = 0; + while index < bytes.len() { + match comment_end(bytes, index) { + Ok(Some(end)) => { + index = end; + continue; + } + Ok(None) => {} + Err(Unclosed::BlockComment | Unclosed::Quote | Unclosed::Parenthesis) => break, + } + match non_code_end(bytes, index, QuotePolicy::COMPILER) { + Ok(Some(end)) => { + scan.punctuation(bytes[index]); + index = end; + continue; + } + Ok(None) => {} + Err(Unclosed::BlockComment | Unclosed::Quote | Unclosed::Parenthesis) => break, + } + let byte: u8 = bytes[index]; + if byte.is_ascii_whitespace() { + index += 1; + continue; + } + if !(byte.is_ascii_alphabetic() || byte == b'_') { + scan.punctuation(byte); + index += 1; + continue; + } + let start: usize = index; + while index < bytes.len() && is_word_byte(bytes[index]) { + index += 1; + } + let name: &str = &sql[start..index]; + let upper: String = name.to_ascii_uppercase(); + let open: usize = skip_trivia(bytes, index); + if bytes.get(open) == Some(&b'(') + && scan.call_position() + && !name.starts_with("__") + && !names.contains(&upper) + && !KEYWORDS.contains(upper.as_str()) + && let Some((arity, close)) = arguments(bytes, open) + && !opens_cte_body(bytes, close + 1) + && let Some(suggestion) = probes.suggestion(dialect, &upper, arity) + { + calls.push(UnsupportedCall { + name: name.to_owned(), + start, + end: index, + suggestion, + }); + } + scan.word(upper); + } + calls +} + +impl Scan { + fn new() -> Self { + Self { + frames: vec![Frame::default()], + previous_word: None, + word_before: None, + previous_byte: None, + path_colon: false, + } + } + + fn frame(&self) -> &Frame { + self.frames.last().unwrap_or(&ROOT_FRAME) + } + + /// Return whether a name followed by `(` here is a call rather than a relation or column list. + fn call_position(&self) -> bool { + match self.previous_byte { + Some(b'.' | b'@' | b')' | b'"' | b'`') => return false, + Some(b':') if self.path_colon => return false, + Some(b',') if self.frame().clause == Clause::From => return false, + _ => {} + } + let Some(word) = self.previous_word.as_deref() else { + return true; + }; + match word { + TYPE_POSITION_KEYWORD => self.frame().clause == Clause::Define, + FROM_KEYWORD => !self.relation_from(), + _ => EXPRESSION_STARTS.contains(word), + } + } + + /// Return whether a `FROM` just seen introduces relations rather than an operand. + fn relation_from(&self) -> bool { + self.frame().select_seen && self.word_before.as_deref() != Some(DISTINCT_KEYWORD) + } + + fn punctuation(&mut self, byte: u8) { + self.path_colon = byte == b':' && self.previous_byte != Some(b'\''); + match byte { + b'(' => self.frames.push(Frame::default()), + b')' if self.frames.len() > 1 => { + self.frames.pop(); + } + _ => {} + } + self.previous_word = None; + self.word_before = None; + self.previous_byte = Some(byte); + } + + fn word(&mut self, upper: String) { + let relation_from: bool = upper == FROM_KEYWORD + && self.frame().select_seen + && self.previous_word.as_deref() != Some(DISTINCT_KEYWORD); + if let Some(frame) = self.frames.last_mut() { + match upper.as_str() { + "SELECT" => { + frame.select_seen = true; + frame.clause = Clause::Other; + } + "JOIN" => frame.clause = Clause::From, + FROM_KEYWORD if relation_from => frame.clause = Clause::From, + "DEFINE" => frame.clause = Clause::Define, + keyword if CLAUSE_KEYWORDS.contains(keyword) => frame.clause = Clause::Other, + _ => {} + } + } + self.word_before = self.previous_word.replace(upper); + self.previous_byte = None; + self.path_colon = false; + } +} + +/// Skip whitespace and comments between a name and its opening parenthesis. +fn skip_trivia(sql: &[u8], mut index: usize) -> usize { + loop { + while sql.get(index).is_some_and(u8::is_ascii_whitespace) { + index += 1; + } + match comment_end(sql, index) { + Ok(Some(end)) => index = end, + Ok(None) | Err(Unclosed::BlockComment | Unclosed::Quote | Unclosed::Parenthesis) => { + return index; + } + } + } +} + +/// Return whether `AS [[NOT] MATERIALIZED] (` follows, so the parenthesis was a CTE column list. +fn opens_cte_body(sql: &[u8], index: usize) -> bool { + let Some(mut index) = keyword_end(sql, index, TYPE_POSITION_KEYWORD) else { + return false; + }; + if let Some(end) = keyword_end(sql, index, NOT_KEYWORD) { + let Some(end) = keyword_end(sql, end, MATERIALIZED_KEYWORD) else { + return false; + }; + index = end; + } else if let Some(end) = keyword_end(sql, index, MATERIALIZED_KEYWORD) { + index = end; + } + sql.get(skip_trivia(sql, index)) == Some(&b'(') +} + +/// Return the end of `keyword` when it is the next token after trivia at `index`. +fn keyword_end(sql: &[u8], index: usize, keyword: &str) -> Option { + let start: usize = skip_trivia(sql, index); + let end: usize = start + keyword.len(); + (sql.get(start..end)? + .eq_ignore_ascii_case(keyword.as_bytes()) + && !sql.get(end).copied().is_some_and(is_word_byte)) + .then_some(end) +} + +fn is_word_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'$' +} + +fn is_identifier(name: &str) -> bool { + name.bytes() + .next() + .is_some_and(|byte| byte.is_ascii_alphabetic() || byte == b'_') + && name.bytes().all(is_word_byte) +} + +/// Return the top-level argument count and closing index of the call opening at `open`. +fn arguments(sql: &[u8], open: usize) -> Option<(usize, usize)> { + let mut depth: usize = 0; + let mut commas: usize = 0; + let mut empty: bool = true; + let mut index: usize = open; + while index < sql.len() { + match comment_end(sql, index) { + Ok(Some(end)) => { + index = end; + continue; + } + Ok(None) => {} + Err(Unclosed::BlockComment | Unclosed::Quote | Unclosed::Parenthesis) => return None, + } + match non_code_end(sql, index, QuotePolicy::COMPILER) { + Ok(Some(end)) => { + empty = false; + index = end; + continue; + } + Ok(None) => {} + Err(Unclosed::BlockComment | Unclosed::Quote | Unclosed::Parenthesis) => return None, + } + match sql[index] { + b'(' | b'[' | b'{' => depth += 1, + b')' | b']' | b'}' => { + depth = depth.checked_sub(1)?; + if depth == 0 { + return Some((if empty { 0 } else { commas + 1 }, index)); + } + } + b',' if depth == 1 => commas += 1, + byte if !byte.is_ascii_whitespace() => empty = false, + _ => {} + } + index += 1; + } + None +} + +/// Return the closest accepted spelling the parser maps to the same built-in node as `upper`. +fn closest_spelling(dialect: DialectType, upper: &str, arity: usize) -> Option { + let signature: Signature = parsed_signature(dialect, upper, arity)?; + if signature == Signature::Named(upper.to_owned()) { + return None; + } + spelling_index(dialect, arity)? + .get(&signature)? + .iter() + .min_by_key(|accepted| (edit_distance(upper, accepted), (*accepted).clone())) + .cloned() +} + +fn spelling_index(dialect: DialectType, arity: usize) -> Option<&'static SpellingIndex> { + let (names, indexes) = match dialect { + DialectType::Snowflake => (&*SNOWFLAKE, &SNOWFLAKE_INDEX), + DialectType::DuckDB => (&*DUCKDB, &DUCKDB_INDEX), + _ => return None, + }; + Some(indexes.get(arity)?.get_or_init(|| { + let mut index: SpellingIndex = HashMap::new(); + for name in names { + if let Some(signature) = parsed_signature(dialect, name, arity) { + index.entry(signature).or_default().push(name.clone()); + } + } + index + })) +} + +fn parsed_signature(dialect: DialectType, upper: &str, arity: usize) -> Option { + let arguments: Vec = (1..=arity).map(|index| format!("a{index}")).collect(); + let sql: String = format!("{PROBE_PREFIX}{upper}({})", arguments.join(", ")); + let Ok(statements) = Dialect::get(dialect).parse(&sql) else { + return None; + }; + let [Expression::Select(select)] = statements.as_slice() else { + return None; + }; + let [expression] = select.expressions.as_slice() else { + return None; + }; + match expression { + Expression::Function(function) => { + Some(Signature::Named(function.name.to_ascii_uppercase())) + } + Expression::AggregateFunction(function) => { + Some(Signature::Named(function.name.to_ascii_uppercase())) + } + Expression::Column(_) + | Expression::Identifier(_) + | Expression::Tuple(_) + | Expression::Paren(_) => None, + other => Some(Signature::Typed(std::mem::discriminant(other))), + } +} + +fn edit_distance(left: &str, right: &str) -> usize { + let right: Vec = right.chars().collect(); + let mut previous: Vec = (0..=right.len()).collect(); + for (row, left_char) in left.chars().enumerate() { + let mut current: Vec = vec![row + 1; right.len() + 1]; + for (column, right_char) in right.iter().enumerate() { + let substitution: usize = previous[column] + usize::from(left_char != *right_char); + current[column + 1] = substitution + .min(previous[column + 1] + 1) + .min(current[column] + 1); + } + previous = current; + } + previous[right.len()] +} diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/mod.rs b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/mod.rs index a524e55b94..a0cffe880f 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/mod.rs +++ b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod catalog; pub(crate) mod diagnostics; +pub(crate) mod function_names; pub(crate) mod identifiers; pub(crate) mod normalization; pub(crate) mod positions; diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/main/map_diagnostics.rs b/crates/sqlbuild-rules-native/src/semantic_validation/main/map_diagnostics.rs index 75bfd83774..73e179774e 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/main/map_diagnostics.rs +++ b/crates/sqlbuild-rules-native/src/semantic_validation/main/map_diagnostics.rs @@ -1,9 +1,11 @@ +use crate::semantic_validation::models::FunctionProbes; use polyglot_sql::{DialectType, ValidationResult}; pub(crate) fn map_diagnostics( sql: &str, dialect: DialectType, result: ValidationResult, + probes: &FunctionProbes, ) -> Result { - crate::semantic_validation::_helpers::diagnostics::map_diagnostics(sql, dialect, result) + crate::semantic_validation::_helpers::diagnostics::map_diagnostics(sql, dialect, result, probes) } diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/models.rs b/crates/sqlbuild-rules-native/src/semantic_validation/models.rs index 07e407c815..7fd72ededa 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/models.rs +++ b/crates/sqlbuild-rules-native/src/semantic_validation/models.rs @@ -1,10 +1,11 @@ //! Compile-owned native catalog, mapping state and Python-boundary requests. -use crate::semantic_validation::types::{Expansion, Relations}; +use crate::semantic_validation::types::{Expansion, ProbeKey, Relations}; use polyglot_sql::validation::SchemaTable; use polyglot_sql::{DialectType, SchemaValidationOptions}; use pyo3::{FromPyObject, pyclass}; use std::collections::HashMap; +use std::sync::Mutex; #[derive(Default, Debug)] pub(crate) struct Columns(pub(crate) Vec<(String, Option)>); @@ -38,6 +39,12 @@ pub(crate) struct PositionInput { pub(crate) passes: Vec>, } +/// Memoized call-site function-name probes, owned by the catalog that validates a compile. +#[derive(Debug, Default)] +pub(crate) struct FunctionProbes { + pub(crate) suggestions: Mutex>>, +} + #[pyclass(module = "sqlbuild._native")] pub(crate) struct ProjectCatalog { pub(crate) dialect: DialectType, @@ -46,6 +53,7 @@ pub(crate) struct ProjectCatalog { pub(super) tables: HashMap, pub(super) overrides: Vec>, pub(super) analysis_tables: HashMap, + pub(crate) function_probes: FunctionProbes, } #[pyclass(module = "sqlbuild._native")] diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/tests.rs b/crates/sqlbuild-rules-native/src/semantic_validation/tests.rs index 343726858a..b4df2321eb 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/tests.rs +++ b/crates/sqlbuild-rules-native/src/semantic_validation/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_function_names.rs"] +mod function_names; #[path = "tests/test_semantic_validation.rs"] mod semantic_validation; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/tests/test_function_names.rs b/crates/sqlbuild-rules-native/src/semantic_validation/tests/test_function_names.rs new file mode 100644 index 0000000000..6761b1fbd3 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/semantic_validation/tests/test_function_names.rs @@ -0,0 +1,255 @@ +use crate::semantic_validation::_helpers::diagnostics::map_diagnostics; +use crate::semantic_validation::_helpers::function_names::unsupported_calls; +use crate::semantic_validation::models::FunctionProbes; +use crate::semantic_validation::tests::test_types::{ + LocatedFunctionDiagnosticTestCase, UnsupportedFunctionTestCase, +}; +use polyglot_sql::{DialectType, ValidationResult}; + +#[test] +fn given_unsupported_builtin_spelling_when_scanning_then_reports_dialect_spelling() { + let test_cases = [ + UnsupportedFunctionTestCase { + description: "Snowflake STARTS_WITH suggests STARTSWITH", + dialect: DialectType::Snowflake, + sql: "SELECT STARTS_WITH(product_name, 'a') AS flagged FROM products", + expected_calls: &[("STARTS_WITH", "STARTSWITH")], + }, + UnsupportedFunctionTestCase { + description: "matching is case-insensitive and keeps the authored spelling", + dialect: DialectType::Snowflake, + sql: "select ends_with(product_name, 'a'), lcase(product_name) from products", + expected_calls: &[("ends_with", "ENDSWITH"), ("lcase", "LOWER")], + }, + UnsupportedFunctionTestCase { + description: "conditional and length aliases suggest the Snowflake spelling", + dialect: DialectType::Snowflake, + sql: "SELECT IF(quantity > 0, 1, 0), CHAR_LENGTH(product_name) FROM products", + expected_calls: &[("IF", "IFF"), ("CHAR_LENGTH", "LENGTH")], + }, + UnsupportedFunctionTestCase { + description: "calls inside windows, lambdas and table functions are checked", + dialect: DialectType::Snowflake, + sql: "SELECT SUM(quantity) OVER (PARTITION BY UCASE(category)), \ + FILTER(tags, tag -> STARTS_WITH(tag, 'a')) \ + FROM products, TABLE(FLATTEN(input => ARRAY_CONSTRUCT(LCASE(sku))))", + expected_calls: &[ + ("UCASE", "UPPER"), + ("STARTS_WITH", "STARTSWITH"), + ("LCASE", "LOWER"), + ], + }, + UnsupportedFunctionTestCase { + description: "an object literal value after a string key is checked", + dialect: DialectType::Snowflake, + sql: "SELECT {'featured': STARTS_WITH(product_name, 'a')} FROM products", + expected_calls: &[("STARTS_WITH", "STARTSWITH")], + }, + UnsupportedFunctionTestCase { + description: "a MATCH_RECOGNIZE DEFINE condition after AS is checked", + dialect: DialectType::Snowflake, + sql: "SELECT * FROM products MATCH_RECOGNIZE (ORDER BY sku PATTERN (featured) \ + DEFINE featured AS STARTS_WITH(product_name, 'a'))", + expected_calls: &[("STARTS_WITH", "STARTSWITH")], + }, + UnsupportedFunctionTestCase { + description: "comments between the name and its parenthesis are skipped", + dialect: DialectType::Snowflake, + sql: "SELECT LCASE /* lower */ (sku), UCASE -- upper\n (sku) FROM products", + expected_calls: &[("LCASE", "LOWER"), ("UCASE", "UPPER")], + }, + UnsupportedFunctionTestCase { + description: "operand FROM and select lists inside a FROM subquery are checked", + dialect: DialectType::Snowflake, + sql: "SELECT TRIM(BOTH 'x' FROM LCASE(sku)), sku IS DISTINCT FROM UCASE(sku) \ + FROM (SELECT sku, LCASE(sku) AS lowered FROM products) AS listed, catalog \ + WHERE sku IN (1, 2) AND LCASE(sku) = 'a'", + expected_calls: &[ + ("LCASE", "LOWER"), + ("UCASE", "UPPER"), + ("LCASE", "LOWER"), + ("LCASE", "LOWER"), + ], + }, + UnsupportedFunctionTestCase { + description: "a call followed by AS NOT without MATERIALIZED is still checked", + dialect: DialectType::DuckDB, + sql: "SELECT STARTSWITH(product_name, 'a') AS not_featured FROM products", + expected_calls: &[("STARTSWITH", "STARTS_WITH")], + }, + UnsupportedFunctionTestCase { + description: "DuckDB STARTSWITH suggests the catalogue spelling", + dialect: DialectType::DuckDB, + sql: "SELECT STARTSWITH(product_name, 'a') FROM products", + expected_calls: &[("STARTSWITH", "STARTS_WITH")], + }, + ]; + for test_case in &test_cases { + let calls = unsupported_calls(test_case.sql, test_case.dialect, &FunctionProbes::default()); + let observed: Vec<(&str, &str)> = calls + .iter() + .map(|call| { + ( + &test_case.sql[call.start..call.end], + call.suggestion.as_str(), + ) + }) + .collect(); + + assert_eq!( + observed, test_case.expected_calls, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_supported_or_non_builtin_call_when_scanning_then_reports_nothing() { + let test_cases = [ + UnsupportedFunctionTestCase { + description: "listed Snowflake synonyms are accepted", + dialect: DialectType::Snowflake, + sql: "SELECT STARTSWITH(a, 'x'), ENDSWITH(a, 'x'), CONTAINS(a, 'x'), SUBSTR(a, 1, 2), \ + SUBSTRING(a, 1, 2), TIMESTAMPDIFF(day, b, c), DATEDIFF(day, b, c), \ + TRY_TO_DECIMAL(a, 10, 2), TRY_TO_NUMBER(a), IFF(d, 1, 0), LEN(a), \ + LENGTH(a), CEIL(e), DATE_PART(year, b), COUNT_IF(d), POSITION('x', a), \ + CHARINDEX('x', a) FROM products", + expected_calls: &[], + }, + UnsupportedFunctionTestCase { + description: "quoted identifiers, literals and comments are not call sites", + dialect: DialectType::Snowflake, + sql: "SELECT \"STARTS_WITH\"(a, 'x'), 'STARTS_WITH(a, b)', $$LCASE(a)$$ \ + -- STARTS_WITH(a, 'x')\n /* LCASE(a) */ FROM products", + expected_calls: &[], + }, + UnsupportedFunctionTestCase { + description: "methods, qualified functions, sentinels and macros are not checked", + dialect: DialectType::Snowflake, + sql: "SELECT tools.starts_with(a, 'x'), analytics.helpers.lcase(a), \ + __sqlbuild_udf_starts_with(a, 'x'), @starts_with(a), payload:lcase(a) \ + FROM __ref(\"products\")", + expected_calls: &[], + }, + UnsupportedFunctionTestCase { + description: "unknown and user-defined function names are never flagged", + dialect: DialectType::Snowflake, + sql: "SELECT normalize_sku(a), DATE_ADD(day, 1, b), my_udf() FROM products", + expected_calls: &[], + }, + UnsupportedFunctionTestCase { + description: "type positions and keywords before parentheses are not calls", + dialect: DialectType::Snowflake, + sql: "WITH ranked (sku, position_rank) AS (SELECT DISTINCT (a), \ + ROW_NUMBER() OVER (ORDER BY b) FROM products WHERE NOT (a IN (1, 2)) \ + AND EXISTS (SELECT 1)) \ + SELECT CAST(sku AS VARCHAR(10)), position_rank::NUMBER(10, 2) FROM ranked", + expected_calls: &[], + }, + UnsupportedFunctionTestCase { + description: "CTE column lists are not calls", + dialect: DialectType::Snowflake, + sql: "WITH listed AS (SELECT 1 AS dow), day_of_week (dow) AS (SELECT 1), \ + lcase(x) AS (SELECT 2) SELECT dow FROM day_of_week", + expected_calls: &[], + }, + UnsupportedFunctionTestCase { + description: "materialized CTE column lists are not calls", + dialect: DialectType::DuckDB, + sql: "WITH day_of_week (d) AS MATERIALIZED (SELECT 1), \ + day_of_month (d) AS /* hint */ NOT -- keep\n MATERIALIZED (SELECT 2) \ + SELECT * FROM day_of_week", + expected_calls: &[], + }, + UnsupportedFunctionTestCase { + description: "derived-table, table-function and table aliases are not calls", + dialect: DialectType::Snowflake, + sql: "SELECT x FROM (VALUES (1)) day_of_week (x), products lcase (y), \ + TABLE(FLATTEN(input => tags)) ucase (z), day_of_month (w) \ + JOIN char_length (v) ON TRUE", + expected_calls: &[], + }, + UnsupportedFunctionTestCase { + description: "INSERT and CREATE column lists are not calls", + dialect: DialectType::Snowflake, + sql: "CREATE TABLE day_of_week (dow INTEGER); CREATE VIEW lcase (x) AS SELECT 1; \ + INSERT INTO ucase (x) SELECT 1; CREATE TABLE IF NOT EXISTS char_length (x INT)", + expected_calls: &[], + }, + UnsupportedFunctionTestCase { + description: "dialects without a spelling table are not checked", + dialect: DialectType::PostgreSQL, + sql: "SELECT STARTS_WITH(a, 'x'), LCASE(a) FROM products", + expected_calls: &[], + }, + UnsupportedFunctionTestCase { + description: "DuckDB catalogue spellings are accepted", + dialect: DialectType::DuckDB, + sql: "SELECT starts_with(a, 'x'), prefix(a, 'x'), lcase(a), substr(a, 1, 2) \ + FROM products", + expected_calls: &[], + }, + ]; + for test_case in &test_cases { + let calls = unsupported_calls(test_case.sql, test_case.dialect, &FunctionProbes::default()); + let observed: Vec<(&str, &str)> = calls + .iter() + .map(|call| (call.name.as_str(), call.suggestion.as_str())) + .collect(); + + assert_eq!( + observed, test_case.expected_calls, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_unsupported_spelling_when_mapping_diagnostics_then_returns_located_b101() +-> Result<(), String> { + let test_cases = [LocatedFunctionDiagnosticTestCase { + description: "the diagnostic points at the authored name after a multibyte literal", + sql: "SELECT\n 'é' AS label, STARTS_WITH(product_name, 'a')\nFROM products", + expected_message: "Unknown function 'STARTS_WITH' for dialect Snowflake; \ + did you mean STARTSWITH?", + expected_location: (Some(2), Some(19)), + expected_span: (Some(25), Some(36)), + }]; + for test_case in &test_cases { + let result = map_diagnostics( + test_case.sql, + DialectType::Snowflake, + ValidationResult { + valid: true, + errors: Vec::new(), + }, + &FunctionProbes::default(), + )?; + let [error] = result.errors.as_slice() else { + return Err(format!("{}: {:?}", test_case.description, result.errors)); + }; + + assert!(!result.valid, "{}", test_case.description); + assert_eq!(error.code, "B101", "{}", test_case.description); + assert_eq!( + error.message, test_case.expected_message, + "{}", + test_case.description + ); + assert_eq!( + (error.line, error.column), + test_case.expected_location, + "{}", + test_case.description + ); + assert_eq!( + (error.start, error.end), + test_case.expected_span, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/tests/test_types.rs b/crates/sqlbuild-rules-native/src/semantic_validation/tests/test_types.rs index 8145f970d6..5228220a9a 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/tests/test_types.rs +++ b/crates/sqlbuild-rules-native/src/semantic_validation/tests/test_types.rs @@ -9,3 +9,18 @@ pub(crate) struct SemanticValidationBatchTestCase { pub(crate) request: &'static str, pub(crate) expected_validity: [bool; 2], } + +pub(crate) struct UnsupportedFunctionTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: polyglot_sql::DialectType, + pub(crate) sql: &'static str, + pub(crate) expected_calls: &'static [(&'static str, &'static str)], +} + +pub(crate) struct LocatedFunctionDiagnosticTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_message: &'static str, + pub(crate) expected_location: (Option, Option), + pub(crate) expected_span: (Option, Option), +} diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/types.rs b/crates/sqlbuild-rules-native/src/semantic_validation/types.rs index d3c9798d08..4a105c3cd7 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/types.rs +++ b/crates/sqlbuild-rules-native/src/semantic_validation/types.rs @@ -13,3 +13,4 @@ pub(crate) type DiagnosticRow = ( String, ); pub(crate) type Expansion = (usize, usize, usize, usize); +pub(crate) type ProbeKey = (polyglot_sql::DialectType, String, usize); diff --git a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/formatter.rs b/crates/sqlbuild-rules-native/src/sql_lint/_helpers/formatter.rs index 48f4f7beab..8b8183ecd4 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/formatter.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/_helpers/formatter.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use std::str::FromStr; use polyglot_sql::tokens::{Token, TokenType}; @@ -6,16 +6,17 @@ use polyglot_sql::{ComplexityGuardOptions, Dialect, DialectType, ParseOptions}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use crate::sql_lint::_helpers::formatter_syntax::{cte_macro_indices, protect_syntax}; +use crate::sql_lint::_helpers::printed_layout::{PrintedSql, WrapOptions, wrap_lines}; +use crate::sql_lint::_helpers::token_layout::{ + AuthoredSql, KeywordRecase, comments_in, neutralize_comments, print_authored_tokens, + verify_token_invariant, +}; use crate::sql_lint::constants::{ CAST_TYPE_SEPARATOR_KEYWORD, CLOSE_PARENTHESIS, LINT_API_VERSION, OPEN_PARENTHESIS, - VALUES_RELATION_PREFIX_TOKEN_COUNT, }; use crate::sql_lint::models::{FormatRequest, FormatResponse}; -const COMMENT_ATTACHMENT_FAILURE: &str = - "native formatter could not preserve comment token attachments"; -const TOKEN_PRESERVATION_FAILURE: &str = - "native formatter would change authored SQL tokens, not only layout"; +const NOT_IDEMPOTENT_FAILURE: &str = "native formatter output is not idempotent"; const UNSUPPORTED_SQL_FAILURE: &str = "native formatter could not safely restore literal or cast spelling"; @@ -65,283 +66,134 @@ fn try_format_sql(request: FormatRequest) -> Result { ..Default::default() }), }; + let wrap = request.line_width.map(|line_width| WrapOptions { + line_width, + token_widths: &request.token_widths, + }); + let layout = Layout { + dialect: &dialect, + wrap: wrap.as_ref(), + }; + let (formatted, oracle) = format_layout(&original, &layout, &parse_options)?; + if relayout(&formatted, &oracle, &layout)? != formatted { + return Err(NOT_IDEMPOTENT_FAILURE.to_string()); + } + let changed = formatted != original; + Ok(FormatResponse { + version: LINT_API_VERSION, + changed, + sql: formatted, + formatted: true, + reason: None, + }) +} + +/// Print the tokens of `sql` with parse-tree whitespace; return the output and its oracle. +fn format_layout( + sql: &str, + layout: &Layout<'_>, + parse_options: &ParseOptions, +) -> Result<(String, String), String> { + let dialect = layout.dialect; let tokens = dialect - .tokenize(&original) + .tokenize(sql) .map_err(|error| format!("native formatter tokenization failed: {error}"))?; - let comments = comments_in(&original, &tokens); - let neutral = neutralize_comments(&original, &comments); + if tokens.is_empty() { + return Ok((sql.to_string(), sql.to_string())); + } + let comments = comments_in(sql, &tokens); + let neutral = neutralize_comments(sql, &comments); let validation_sql = protect_syntax(&neutral, &tokens, &[], false)?; dialect - .parse_with_options(&validation_sql.sql, &parse_options) + .parse_with_options(&validation_sql.sql, parse_options) .map_err(|error| format!("native formatter could not parse SQL: {error}"))?; let semantic_tokens = without_statement_terminators(&tokens); - let formatted_context = FormatOnceContext { - parse_options: &parse_options, - dialect: &dialect, - original_sql: &neutral, - original_tokens: &semantic_tokens, + let oracle = layout_oracle(&neutral, &semantic_tokens, dialect, parse_options)?; + let authored = AuthoredSql { + sql, + tokens: &tokens, comments: &comments, }; - let formatted = preserve_authored_tokens( - &neutral, - format_once(&neutral, &formatted_context)?, - &dialect, - )?; - let formatted_tokens = dialect - .tokenize(&formatted) - .map_err(|error| error.to_string())?; - let formatted_neutral = - neutralize_comments(&formatted, &comments_in(&formatted, &formatted_tokens)); - let validation_sql = protect_syntax(&formatted_neutral, &formatted_tokens, &[], false)?; - let _ = dialect - .parse_with_options(&validation_sql.sql, &parse_options) - .map_err(|error| error.to_string())?; - let second_tokens = dialect - .tokenize(&formatted) - .map_err(|error| error.to_string())?; - let second_comments = comments_in(&formatted, &second_tokens); - let second_neutral = neutralize_comments(&formatted, &second_comments); - let second_semantic_tokens = without_statement_terminators(&second_tokens); - let second_context = FormatOnceContext { - parse_options: &parse_options, - dialect: &dialect, - original_sql: &second_neutral, - original_tokens: &second_semantic_tokens, - comments: &second_comments, - }; - let second_pass = preserve_authored_tokens( - &second_neutral, - format_once(&second_neutral, &second_context)?, - &dialect, - )?; - if second_pass != formatted { - return Err("native formatter output is not idempotent".to_string()); - } - let changed = formatted != original; - response(formatted, changed, true, None) + let (printed, recases) = layout.print(&authored, &oracle)?; + verify_token_invariant(sql, &printed, dialect, &recases)?; + Ok((printed, oracle)) } -/// Restores authored token text into generated layout, refusing any structural token change. -pub(crate) fn preserve_authored_tokens( - authored_sql: &str, - mut formatted: String, - dialect: &Dialect, -) -> Result { - let authored = dialect - .tokenize(authored_sql) - .map_err(|error| error.to_string())?; - let generated = dialect - .tokenize(&formatted) - .map_err(|error| error.to_string())?; - let mut replacements: Vec<(usize, usize, String)> = Vec::new(); - let mut authored_index = 0; - let mut generated_index = 0; - while authored_index < authored.len() || generated_index < generated.len() { - let before = authored.get(authored_index); - let after = generated.get(generated_index); - match (before, after) { - (before, Some(after)) - if after.token_type == TokenType::As - && before.is_none_or(|token| token.token_type != TokenType::As) => - { - generated_index += 1; - } - (Some(before), Some(after)) if tokens_align(before, after) => { - let authored_text = raw_token_text(authored_sql, before)?; - let generated_text = raw_token_text(&formatted, after)?; - if authored_text != generated_text - && !(is_unquoted_word(&authored_text) - && authored_text.eq_ignore_ascii_case(&generated_text)) - { - replacements.push((after.span.start, after.span.end, authored_text)); - } - authored_index += 1; - generated_index += 1; - } - (Some(before), after) if is_optional_terminator(&authored, authored_index, before) => { - if after.is_some_and(|token| token.token_type == before.token_type) { - generated_index += 1; - } - authored_index += 1; - } - (None, Some(after)) if after.token_type == TokenType::Semicolon => { - generated_index += 1; - } - _ => return Err(TOKEN_PRESERVATION_FAILURE.to_string()), - } - } - for (start, end, text) in replacements.into_iter().rev() { - let start = char_to_byte(&formatted, start)?; - let end = char_to_byte(&formatted, end)?; - formatted.replace_range(start..end, &text); +/// Re-print formatted SQL against its oracle; unchanged tokens imply an unchanged layout. +fn relayout(formatted: &str, oracle: &str, layout: &Layout<'_>) -> Result { + let tokens = layout + .dialect + .tokenize(formatted) + .map_err(|error| format!("native formatter tokenization failed: {error}"))?; + if tokens.is_empty() { + return Ok(formatted.to_string()); } - Ok(formatted) -} - -fn tokens_align(before: &Token, after: &Token) -> bool { - before.token_type == after.token_type || (is_word_token(before) && is_word_token(after)) -} - -fn is_word_token(token: &Token) -> bool { - is_unquoted_word(&token.text) -} - -fn is_unquoted_word(text: &str) -> bool { - let mut characters = text.chars(); - characters - .next() - .is_some_and(|first| first.is_ascii_alphabetic() || first == '_') - && characters - .all(|character| character.is_ascii_alphanumeric() || matches!(character, '_' | '$')) -} - -fn is_optional_terminator(tokens: &[Token], index: usize, token: &Token) -> bool { - token.token_type == TokenType::Semicolon - || (token.token_type == TokenType::Comma - && tokens - .get(index + 1) - .is_some_and(|next| next.token_type == TokenType::From)) -} - -fn raw_token_text(sql: &str, token: &Token) -> Result { - char_slice(sql, token.span.start, token.span.end) - .ok_or_else(|| TOKEN_PRESERVATION_FAILURE.to_string()) + let comments = comments_in(formatted, &tokens); + let authored = AuthoredSql { + sql: formatted, + tokens: &tokens, + comments: &comments, + }; + Ok(layout.print(&authored, oracle)?.0) } -#[derive(Debug, Clone)] -struct Comment { - start: usize, - end: usize, - text: String, - line: bool, - leading: bool, +/// The dialect and optional line wrapping every printed layout uses. +struct Layout<'a> { + dialect: &'a Dialect, + wrap: Option<&'a WrapOptions<'a>>, } -struct FormatOnceContext<'a> { - parse_options: &'a ParseOptions, - dialect: &'a Dialect, - original_sql: &'a str, - original_tokens: &'a [Token], - comments: &'a [Comment], +impl Layout<'_> { + fn print( + &self, + authored: &AuthoredSql<'_>, + oracle: &str, + ) -> Result<(String, BTreeMap), String> { + let (printed, recases) = print_authored_tokens(authored, oracle, self.dialect)?; + let no_widths: HashMap = HashMap::new(); + let token_widths = self.wrap.map_or(&no_widths, |wrap| wrap.token_widths); + let Some(mut sql) = PrintedSql::parse(&printed, self.dialect, token_widths)? else { + return Ok((printed, recases)); + }; + sql.arrange_clauses(self.wrap.map_or(usize::MAX, |wrap| wrap.line_width)); + if let Some(wrap) = self.wrap { + wrap_lines(&mut sql, wrap.line_width); + } + sql.lead_operators_of_multiline_operands(); + Ok((sql.render(), recases)) + } } -fn format_once(neutral_sql: &str, context: &FormatOnceContext<'_>) -> Result { +/// Return the parse-tree layout of `neutral_sql`; only its whitespace is ever printed. +fn layout_oracle( + neutral_sql: &str, + semantic_tokens: &[Token], + dialect: &Dialect, + parse_options: &ParseOptions, +) -> Result { let protected = protect_syntax( neutral_sql, - context.original_tokens, - &cast_type_token_ranges(context)?, + semantic_tokens, + &cast_type_token_ranges(neutral_sql, semantic_tokens)?, true, )?; - let expressions = context - .dialect - .parse_with_options(&protected.sql, context.parse_options) + let expressions = dialect + .parse_with_options(&protected.sql, parse_options) .map_err(|error| error.to_string())?; - let formatted_statements: Result, String> = expressions + let statements: Result, String> = expressions .iter() .map(|expression| { - context - .dialect + dialect .generate_pretty(expression) .map_err(|error| error.to_string()) }) .collect(); - let mut formatted = protected.restore(formatted_statements?.join(";\n"), context.dialect)?; - formatted = restore_unparenthesized_from_values(formatted, context)?; - formatted = restore_null_treatment(formatted, context)?; - formatted = layout_cte_macros(formatted, context)?; - if context.comments.is_empty() { - return Ok(formatted); - } - formatted = restore_implicit_aliases(formatted, context)?; - let formatted_tokens = context - .dialect - .tokenize(&formatted) - .map_err(|error| error.to_string())?; - if context.original_tokens.len() != formatted_tokens.len() - || context - .original_tokens - .iter() - .zip(&formatted_tokens) - .any(|(before, after)| before.token_type != after.token_type) - { - return Err(COMMENT_ATTACHMENT_FAILURE.to_string()); - } - let mut insertions: Vec = Vec::new(); - let mut leading_groups: BTreeMap> = BTreeMap::new(); - for comment in context.comments { - if let Some(target) = leading_comment_target(context, comment) { - leading_groups.entry(target).or_default().push(comment); - } else { - insertions.push(trailing_comment_insertion( - &formatted, - &formatted_tokens, - context, - comment, - )); - } - } - for (target, comments) in &leading_groups { - insertions.push(leading_comment_insertion( - &formatted, - &formatted_tokens[*target], - comments, - )); - } - insertions.sort_by_key(|insertion| insertion.start); - for insertion in insertions.into_iter().rev() { - let start = char_to_byte(&formatted, insertion.start)?; - let end = char_to_byte(&formatted, insertion.end)?; - formatted.replace_range(start..end, &insertion.text); - } - Ok(formatted) + let restored = protected.restore(statements?.join(";\n"), dialect)?; + layout_cte_macros(restored, dialect) } -fn restore_implicit_aliases( - mut formatted: String, - context: &FormatOnceContext<'_>, -) -> Result { - let tokens = context - .dialect - .tokenize(&formatted) - .map_err(|error| error.to_string())?; - let mut original = context.original_tokens.iter().peekable(); - let mut removals: Vec<(usize, usize)> = Vec::new(); - for token in &tokens { - if original - .peek() - .is_some_and(|before| before.token_type == token.token_type) - { - original.next(); - } else if token.token_type == TokenType::As { - removals.push((token.span.start, token.span.end)); - } else { - return Err(format!( - "{COMMENT_ATTACHMENT_FAILURE}: expected {:?}, generated {:?}", - original.peek().map(|token| token.token_type), - token.token_type - )); - } - } - if original.next().is_some() { - return Err(COMMENT_ATTACHMENT_FAILURE.to_string()); - } - for (start, end) in removals.into_iter().rev() { - let start = char_to_byte(&formatted, start)?; - let end = char_to_byte(&formatted, end)?; - formatted.replace_range( - start..end + usize::from(formatted[end..].starts_with(' ')), - "", - ); - } - Ok(formatted) -} - -fn layout_cte_macros( - mut formatted: String, - context: &FormatOnceContext<'_>, -) -> Result { - let tokens = context - .dialect +fn layout_cte_macros(mut formatted: String, dialect: &Dialect) -> Result { + let tokens = dialect .tokenize(&formatted) .map_err(|error| error.to_string())?; let macros = cte_macro_indices(&tokens); @@ -380,193 +232,6 @@ fn layout_cte_macros( Ok(formatted) } -fn restore_null_treatment( - mut formatted: String, - context: &FormatOnceContext<'_>, -) -> Result { - let authored: Vec = context - .original_tokens - .windows(3) - .filter(|tokens| { - matches!( - tokens[0].text.to_ascii_uppercase().as_str(), - "IGNORE" | "RESPECT" - ) && tokens[1].text.eq_ignore_ascii_case("NULLS") - }) - .map(|tokens| tokens[2].token_type == TokenType::RParen) - .collect(); - if authored.is_empty() { - return Ok(formatted); - } - let tokens = context - .dialect - .tokenize(&formatted) - .map_err(|error| error.to_string())?; - let generated: Vec = (1..tokens.len().saturating_sub(1)) - .filter(|&index| { - matches!( - tokens[index].text.to_ascii_uppercase().as_str(), - "IGNORE" | "RESPECT" - ) && tokens[index + 1].text.eq_ignore_ascii_case("NULLS") - }) - .collect(); - if authored.len() != generated.len() { - return Err("native formatter changed null-treatment clauses".to_string()); - } - for (inside, index) in authored.into_iter().zip(generated).rev() { - if inside && tokens[index - 1].token_type == TokenType::RParen { - let start = char_to_byte(&formatted, tokens[index - 1].span.start)?; - let end = char_to_byte(&formatted, tokens[index + 1].span.end)?; - formatted.replace_range( - start..end, - &format!(" {} NULLS)", tokens[index].text.to_ascii_uppercase()), - ); - } - } - Ok(formatted) -} - -fn restore_unparenthesized_from_values( - mut formatted: String, - context: &FormatOnceContext<'_>, -) -> Result { - let authored_parenthesized: Vec = context - .original_tokens - .iter() - .enumerate() - .filter(|(_, token)| token_text(context.original_sql, token).as_deref() == Some("VALUES")) - .map(|(index, _)| { - index > 0 - && token_text(context.original_sql, &context.original_tokens[index - 1]).as_deref() - == Some(OPEN_PARENTHESIS) - }) - .collect(); - if authored_parenthesized.is_empty() { - return Ok(formatted); - } - let formatted_tokens = context - .dialect - .tokenize(&formatted) - .map_err(|error| error.to_string())?; - let formatted_values: Vec = formatted_tokens - .iter() - .enumerate() - .filter_map(|(index, token)| { - (token_text(&formatted, token).as_deref() == Some("VALUES")).then_some(index) - }) - .collect(); - if authored_parenthesized.len() != formatted_values.len() { - return Err("native formatter could not preserve VALUES relation count".to_string()); - } - let mut removals: Vec<(usize, usize)> = Vec::new(); - for (was_parenthesized, values_index) in - authored_parenthesized.into_iter().zip(formatted_values) - { - if was_parenthesized || values_index < VALUES_RELATION_PREFIX_TOKEN_COUNT { - continue; - } - let wrapper_index = values_index - 1; - let relation_index = values_index - 2; - let relation_keyword = token_text(&formatted, &formatted_tokens[relation_index]); - if token_text(&formatted, &formatted_tokens[wrapper_index]).as_deref() - != Some(OPEN_PARENTHESIS) - || !matches!(relation_keyword.as_deref(), Some("FROM" | "JOIN")) - { - continue; - } - let mut depth = 0_i32; - let mut close_index: Option = None; - for (index, token) in formatted_tokens.iter().enumerate().skip(wrapper_index) { - match token_text(&formatted, token).as_deref() { - Some(OPEN_PARENTHESIS) => depth += 1, - Some(CLOSE_PARENTHESIS) => { - depth -= 1; - if depth == 0 { - close_index = Some(index); - break; - } - } - _ => {} - } - } - let Some(close_index) = close_index else { - return Err("native formatter could not locate the VALUES wrapper end".to_string()); - }; - let open = &formatted_tokens[wrapper_index]; - let close = &formatted_tokens[close_index]; - removals.push((open.span.start, open.span.end)); - removals.push((close.span.start, close.span.end)); - } - removals.sort_unstable(); - for (start, end) in removals.into_iter().rev() { - let start = char_to_byte(&formatted, start)?; - let end = char_to_byte(&formatted, end)?; - formatted.replace_range(start..end, ""); - } - Ok(formatted) -} - -fn token_text(sql: &str, token: &Token) -> Option { - char_slice(sql, token.span.start, token.span.end).map(|value| value.to_ascii_uppercase()) -} - -struct CommentInsertion { - start: usize, - end: usize, - text: String, -} - -fn leading_comment_target(context: &FormatOnceContext<'_>, comment: &Comment) -> Option { - if !comment.leading { - return None; - } - context - .original_tokens - .iter() - .position(|token| token.span.start >= comment.end) -} - -/// Place a block of leading comments on their own lines directly above the token they explain. -fn leading_comment_insertion( - formatted: &str, - target: &Token, - comments: &[&Comment], -) -> CommentInsertion { - let characters: Vec = formatted.chars().collect(); - let target_start = target.span.start.min(characters.len()); - let mut whitespace_start = target_start; - while whitespace_start > 0 - && characters[whitespace_start - 1].is_whitespace() - && characters[whitespace_start - 1] != '\n' - { - whitespace_start -= 1; - } - let at_line_start = whitespace_start == 0 || characters[whitespace_start - 1] == '\n'; - if at_line_start { - let indentation = " ".repeat(target_start - whitespace_start); - let text: String = comments - .iter() - .map(|comment| format!("{}\n{indentation}", comment.text)) - .collect(); - return CommentInsertion { - start: target_start, - end: target_start, - text, - }; - } - let indentation = " ".repeat(line_indentation(&characters, whitespace_start)); - let text: String = comments - .iter() - .map(|comment| format!("\n{indentation}{}", comment.text)) - .chain(std::iter::once(format!("\n{indentation}"))) - .collect(); - CommentInsertion { - start: whitespace_start, - end: target_start, - text, - } -} - fn line_indentation(characters: &[char], offset: usize) -> usize { let line_start = characters[..offset] .iter() @@ -578,38 +243,17 @@ fn line_indentation(characters: &[char], offset: usize) -> usize { .count() } -fn trailing_comment_insertion( - formatted: &str, - formatted_tokens: &[Token], - context: &FormatOnceContext<'_>, - comment: &Comment, -) -> CommentInsertion { - let previous = context - .original_tokens - .iter() - .rposition(|token| token.span.end <= comment.start); - let char_offset = previous.map_or(0, |index| formatted_tokens[index].span.end); - let separator = if char_offset == 0 { "" } else { " " }; - let followed_by_newline = formatted.chars().nth(char_offset) == Some('\n'); - let terminator = if (comment.line || char_offset == 0) && !followed_by_newline { - "\n" - } else { - "" - }; - CommentInsertion { - start: char_offset, - end: char_offset, - text: format!("{separator}{}{terminator}", comment.text), - } -} - -fn cast_type_token_ranges(context: &FormatOnceContext<'_>) -> Result, String> { +fn cast_type_token_ranges(sql: &str, tokens: &[Token]) -> Result, String> { + let characters: Vec = sql.chars().collect(); let mut ranges: Vec<(usize, usize)> = Vec::new(); let mut cast_type_starts: Vec> = Vec::new(); let mut previous: Option = None; - for (index, token) in context.original_tokens.iter().enumerate() { - let text = char_slice(context.original_sql, token.span.start, token.span.end) - .ok_or_else(|| UNSUPPORTED_SQL_FAILURE.to_string())?; + for (index, token) in tokens.iter().enumerate() { + let text: String = characters + .get(token.span.start..token.span.end) + .ok_or_else(|| UNSUPPORTED_SQL_FAILURE.to_string())? + .iter() + .collect(); let normalized = text.to_ascii_uppercase(); if normalized == OPEN_PARENTHESIS { let opens_cast = previous @@ -635,10 +279,6 @@ fn cast_type_token_ranges(context: &FormatOnceContext<'_>) -> Result Option { - (start <= end).then(|| value.chars().skip(start).take(end - start).collect()) -} - fn without_statement_terminators(tokens: &[Token]) -> Vec { tokens .iter() @@ -654,70 +294,6 @@ fn without_statement_terminators(tokens: &[Token]) -> Vec { .collect() } -fn comments_in(sql: &str, tokens: &[Token]) -> Vec { - let characters: Vec = sql.chars().collect(); - let mut comments: Vec = Vec::new(); - let mut gap_start = 0_usize; - for token in tokens { - comments.extend(comments_in_gap(&characters, gap_start, token.span.start)); - gap_start = token.span.end; - } - comments.extend(comments_in_gap(&characters, gap_start, characters.len())); - comments -} - -fn comments_in_gap(characters: &[char], start: usize, end: usize) -> Vec { - let mut comments: Vec = Vec::new(); - let mut index = start; - while index < end { - let line = characters[index] == '-' && characters.get(index + 1) == Some(&'-'); - let block = characters[index] == '/' && characters.get(index + 1) == Some(&'*'); - if !line && !block { - index += 1; - continue; - } - let start = index; - let line_start = (0..start) - .rev() - .find(|&position| characters[position] == '\n') - .map_or(0, |position| position + 1); - let leading = characters[line_start..start] - .iter() - .all(|character| character.is_whitespace()); - index += 2; - if line { - while index < end && characters[index] != '\n' { - index += 1; - } - } else { - while index + 1 < end && !(characters[index] == '*' && characters[index + 1] == '/') { - index += 1; - } - index = (index + 2).min(end); - } - comments.push(Comment { - start, - end: index, - text: characters[start..index].iter().collect(), - line, - leading, - }); - } - comments -} - -fn neutralize_comments(sql: &str, comments: &[Comment]) -> String { - let mut characters: Vec = sql.chars().collect(); - for comment in comments { - for character in &mut characters[comment.start..comment.end] { - if *character != '\n' && *character != '\r' { - *character = ' '; - } - } - } - characters.into_iter().collect() -} - fn char_to_byte(value: &str, char_offset: usize) -> Result { if char_offset == value.chars().count() { return Ok(value.len()); @@ -728,18 +304,3 @@ fn char_to_byte(value: &str, char_offset: usize) -> Result { .map(|(offset, _)| offset) .ok_or_else(|| "native formatter produced an invalid character offset".to_string()) } - -fn response( - sql: String, - changed: bool, - formatted: bool, - reason: Option<&'static str>, -) -> Result { - Ok(FormatResponse { - version: LINT_API_VERSION, - changed, - sql, - formatted, - reason: reason.map(str::to_owned), - }) -} diff --git a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/mod.rs b/crates/sqlbuild-rules-native/src/sql_lint/_helpers/mod.rs index 18467af6db..c180f8d4ac 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/mod.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/_helpers/mod.rs @@ -5,4 +5,6 @@ pub(crate) mod formatter_syntax; pub(crate) mod inline_relations; pub(crate) mod join_predicates; pub(crate) mod preparation; +pub(crate) mod printed_layout; pub(crate) mod terminal_shape; +pub(crate) mod token_layout; diff --git a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/preparation.rs b/crates/sqlbuild-rules-native/src/sql_lint/_helpers/preparation.rs index e0d6f65181..6d520c10c5 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/preparation.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/_helpers/preparation.rs @@ -8,19 +8,11 @@ use regex::Regex; use crate::rules::main::quote_policy::quote_policy; use crate::sql_lint::types::{InterpolationSite, PreparedSql}; use crate::sql_scan::main::matching_paren::matching_paren as scan_matching_paren; +use crate::sql_scan::main::non_code_end::non_code_end; use crate::sql_scan::models::QuotePolicy; -const SITE_PATTERN: &str = - r#"(?P@@|\$\{|@|(?:__dbt_ref|__ref|__seed|__source|__table_fn|__udf)\s*\()"#; -const NON_CODE_PATTERN: &str = r#"--[^\n]*(?:\n|\z)|/\*[\s\S]*?(?:\*/|\z)|'(?:\\.|''|[^'\\])*(?:'|\z)|"(?:\\.|""|[^"\\])*(?:"|\z)"#; -const BACKTICK_PATTERN: &str = r#"`(?:``|[^`])*(?:`|\z)"#; - -static SITES: LazyLock> = - LazyLock::new(|| Regex::new(&format!("{NON_CODE_PATTERN}|{SITE_PATTERN}"))); -static BACKTICK_SITES: LazyLock> = LazyLock::new(|| { - Regex::new(&format!( - "{NON_CODE_PATTERN}|{BACKTICK_PATTERN}|{SITE_PATTERN}" - )) +static SITE_START: LazyLock> = LazyLock::new(|| { + Regex::new(r"^(?:@@|\$\{|@|(?:__dbt_ref|__ref|__seed|__source|__table_fn|__udf)\s*\()") }); static CTES: LazyLock> = LazyLock::new(|| { Regex::new( @@ -47,23 +39,14 @@ pub(crate) fn prepare( return Ok(None); } let policy = QuotePolicy::SQL_LINT.with_backtick_identifiers(backtick_identifiers(dialect)); - let sites_regex = if policy.backtick_identifiers { - &BACKTICK_SITES - } else { - &SITES - }; - let site_pattern = sites_regex.as_ref().map_err(|error| error.to_string())?; + let site_pattern = SITE_START.as_ref().map_err(|error| error.to_string())?; let cte_pattern = CTES.as_ref().map_err(|error| error.to_string())?; let word_pattern = WORDS.as_ref().map_err(|error| error.to_string())?; let opaque_pattern = OPAQUE_CTE.as_ref().map_err(|error| error.to_string())?; let mut text = String::with_capacity(expanded.len()); let mut sites: Vec = Vec::new(); let mut copied_to = 0; - for captures in site_pattern.captures_iter(expanded) { - let Some(site) = captures.name("site") else { - continue; - }; - let start = site.start(); + for start in site_starts(expanded, site_pattern, policy) { if start < copied_to { continue; } @@ -116,6 +99,30 @@ pub(crate) fn prepare( Ok(Some((text, sites, referenced))) } +/// Offsets of candidate interpolation sites outside comments and quoted text. +fn site_starts(text: &str, site_pattern: &Regex, policy: QuotePolicy) -> Vec { + let bytes = text.as_bytes(); + let mut starts: Vec = Vec::new(); + let mut index = 0; + while index < bytes.len() { + match non_code_end(bytes, index, policy) { + Ok(Some(end)) => { + index = end; + continue; + } + Err(_) => break, + Ok(None) => {} + } + if let Some(found) = site_pattern.find(&text[index..]) { + starts.push(index); + index += found.end(); + } else { + index += 1; + } + } + starts +} + fn site_end(text: &str, start: usize, policy: QuotePolicy) -> Option { let bytes = text.as_bytes(); match bytes[start] { diff --git a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/printed_layout.rs b/crates/sqlbuild-rules-native/src/sql_lint/_helpers/printed_layout.rs new file mode 100644 index 0000000000..5aa8522702 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_lint/_helpers/printed_layout.rs @@ -0,0 +1,931 @@ +//! Printed SQL as tokens and gaps, with clause layout rules and line-width wrapping. + +use std::collections::HashMap; + +use polyglot_sql::Dialect; +use polyglot_sql::tokens::TokenType; + +use crate::sql_lint::constants::OPEN_PARENTHESIS; + +const CONTINUATION_INDENT: usize = 2; +const COMMA: &str = ","; +const SINGLE_SPACE: &str = " "; +const BETWEEN_KEYWORD: &str = "BETWEEN"; +const BETWEEN_SEPARATOR: &str = "AND"; +const VALUES_KEYWORD: &str = "VALUES"; +const DECODE_FUNCTION: &str = "DECODE"; +const CASE_KEYWORD: &str = "CASE"; +const SENTINEL_PREFIX: &str = "__SQB_LINT_"; +const SINGLE_ITEM_CLAUSES: [&str; 4] = ["SELECT", "WHERE", "QUALIFY", "HAVING"]; +const SELECT_MODIFIERS: [&str; 10] = [ + "DISTINCT", "ALL", "TOP", "PERCENT", "WITH", "TIES", "ON", "AS", "STRUCT", "VALUE", +]; +const LEADING_OPERATORS: [&str; 8] = ["+", "-", "*", "/", "%", "||", "AND", "OR"]; +const CHAIN_KEYWORDS: [&str; 2] = ["AND", "OR"]; +const BETWEEN_BOUNDARIES: [&str; 12] = [ + "AND", "OR", "WHEN", "THEN", "ELSE", "WHERE", "ON", "HAVING", "QUALIFY", "SELECT", ",", "NOT", +]; +const ARITHMETIC_OPERATORS: [&str; 5] = ["+", "-", "||", "/", "%"]; +const WINDOW_CLAUSE_KEYWORDS: [&str; 4] = ["ORDER", "ROWS", "RANGE", "GROUPS"]; +const CASE_BRANCH_KEYWORDS: [&str; 3] = ["WHEN", "THEN", "ELSE"]; +const TYPE_PARAMETER_WORDS: [&str; 1] = ["MAX"]; +const TYPE_NAMES: [&str; 30] = [ + "BIGNUMERIC", + "BINARY", + "BIT", + "CHAR", + "CHARACTER", + "DATETIME", + "DATETIME2", + "DATETIMEOFFSET", + "DEC", + "DECIMAL", + "DOUBLE", + "FLOAT", + "INTERVAL", + "NCHAR", + "NUMBER", + "NUMERIC", + "NVARCHAR", + "STRING", + "TEXT", + "TIME", + "TIMESTAMP", + "TIMESTAMPTZ", + "TIMESTAMP_LTZ", + "TIMESTAMP_NTZ", + "TIMESTAMP_TZ", + "TIMETZ", + "VARBINARY", + "VARCHAR", + "BYTES", + "BPCHAR", +]; + +/// Printed SQL whose layout passes only ever rewrite the gaps between tokens. +pub(crate) struct PrintedSql<'a> { + tokens: Vec, + kinds: Vec, + /// The text before each token, plus the text after the last token. + gaps: Vec, + depths: Vec, + partners: Vec>, + /// Gaps that line wrapping must keep: inside VALUES rows and type parameter lists. + locked: Vec, + token_widths: &'a HashMap, +} + +/// One printed line; end columns are absent when comments or multi-line tokens pin it. +struct Line { + first: usize, + last: usize, + indentation: usize, + end_columns: Option>, +} + +impl<'a> PrintedSql<'a> { + pub(crate) fn parse( + sql: &str, + dialect: &Dialect, + token_widths: &'a HashMap, + ) -> Result, String> { + let tokens = dialect + .tokenize(sql) + .map_err(|error| format!("native formatter tokenization failed: {error}"))?; + if tokens.is_empty() { + return Ok(None); + } + let characters: Vec = sql.chars().collect(); + let mut texts: Vec = Vec::with_capacity(tokens.len()); + let mut gaps: Vec = Vec::with_capacity(tokens.len() + 1); + let mut previous_end = 0_usize; + for token in &tokens { + gaps.push(characters[previous_end..token.span.start].iter().collect()); + texts.push( + characters[token.span.start..token.span.end] + .iter() + .collect(), + ); + previous_end = token.span.end; + } + gaps.push(characters[previous_end..].iter().collect()); + let (depths, partners) = bracket_structure(&texts); + let locked = vec![false; texts.len() + 1]; + Ok(Some(Self { + kinds: tokens.iter().map(|token| token.token_type).collect(), + tokens: texts, + gaps, + depths, + partners, + locked, + token_widths, + })) + } + + fn len(&self) -> usize { + self.tokens.len() + } + + fn upper(&self, index: usize) -> String { + self.tokens[index].to_ascii_uppercase() + } + + fn is(&self, index: usize, text: &str) -> bool { + self.tokens + .get(index) + .is_some_and(|token| token.eq_ignore_ascii_case(text)) + } + + fn line_ranges(&self, first: usize, last: usize) -> Vec<(usize, usize)> { + let mut ranges: Vec<(usize, usize)> = Vec::new(); + let mut start = first; + for index in first + 1..=last { + if self.gaps[index].contains('\n') { + ranges.push((start, index - 1)); + start = index; + } + } + ranges.push((start, last)); + ranges + } + + /// The first and last token of the printed line holding `index`. + fn line_of(&self, index: usize) -> (usize, usize) { + let first = (1..=index) + .rev() + .find(|&position| self.gaps[position].contains('\n')) + .unwrap_or(0); + let last = (index + 1..self.len()) + .find(|&position| self.gaps[position].contains('\n')) + .map_or(self.len() - 1, |position| position - 1); + (first, last) + } + + fn line(&self, first: usize, last: usize) -> Line { + let indentation = self.indentation(first); + let lead = self.gaps[first].rsplit('\n').next().unwrap_or_default(); + let trailing = self.gaps[last + 1].split('\n').next().unwrap_or_default(); + let commented = !lead.trim().is_empty() + || !trailing.trim().is_empty() + || (first + 1..=last).any(|index| !self.gaps[index].trim().is_empty()); + let multiline = (first..=last).any(|index| self.tokens[index].contains('\n')); + let end_columns = (!commented && !multiline).then(|| { + let mut column = indentation; + let mut ends: Vec = Vec::with_capacity(last - first + 1); + for index in first..=last { + if index > first { + column += self.gaps[index].chars().count(); + } + column += self.width(index); + ends.push(column); + } + ends + }); + Line { + first, + last, + indentation, + end_columns, + } + } + + /// Indentation of the line that starts at token `first`. + fn indentation(&self, first: usize) -> usize { + self.gaps[first] + .rsplit('\n') + .next() + .unwrap_or_default() + .chars() + .count() + } + + fn width(&self, index: usize) -> usize { + let token = &self.tokens[index]; + self.token_widths + .get(&token.to_ascii_uppercase()) + .copied() + .unwrap_or_else(|| token.chars().count()) + } + + /// Whether any gap of the tokens after `first` up to `last` holds a line break. + fn spans_lines(&self, first: usize, last: usize) -> bool { + (first + 1..=last).any(|index| self.gaps[index].contains('\n')) + } + + pub(crate) fn render(&self) -> String { + let mut output = String::new(); + for (gap, token) in self.gaps.iter().zip(&self.tokens) { + output.push_str(gap); + output.push_str(token); + } + output.push_str(&self.gaps[self.tokens.len()]); + output + } +} + +fn bracket_structure(tokens: &[String]) -> (Vec, Vec>) { + let mut depths: Vec = Vec::with_capacity(tokens.len()); + let mut partners: Vec> = vec![None; tokens.len()]; + let mut open: Vec = Vec::new(); + for (index, token) in tokens.iter().enumerate() { + match token.as_str() { + "(" | "[" | "{" => { + depths.push(open.len()); + open.push(index); + } + ")" | "]" | "}" => { + let opener = open.pop(); + depths.push(open.len()); + if let Some(opener) = opener { + partners[opener] = Some(index); + partners[index] = Some(opener); + } + } + _ => depths.push(open.len()), + } + } + (depths, partners) +} + +fn newline(indentation: usize) -> String { + format!("\n{}", " ".repeat(indentation)) +} + +impl PrintedSql<'_> { + /// Apply the layout rules that only move existing line breaks before wrapping. + pub(crate) fn arrange_clauses(&mut self, line_width: usize) { + for index in 0..self.len() { + if self.is(index, DECODE_FUNCTION) { + self.pair_decode_arguments(index); + } + } + for index in 1..self.len() { + if CHAIN_KEYWORDS.contains(&self.upper(index).as_str()) { + self.join_chain_break_in_group(index); + } + } + for index in 0..self.len() { + if self.is(index, VALUES_KEYWORD) { + self.break_values_rows_unless_fitting(index, line_width); + } + } + for index in 0..self.len() { + if self.kinds[index] == TokenType::With { + self.separate_common_table_expressions(index); + } + } + for index in 0..self.len() { + if SINGLE_ITEM_CLAUSES.contains(&self.upper(index).as_str()) { + self.join_single_line_body(index, line_width); + } + } + } + + /// Start the next line with any binary operator whose operand spans several lines. + pub(crate) fn lead_operators_of_multiline_operands(&mut self) { + for index in 1..self.len().saturating_sub(1) { + if !LEADING_OPERATORS.contains(&self.upper(index).as_str()) + || self.gaps[index] != SINGLE_SPACE + || self.gaps[index + 1].contains('\n') + || !self.ends_operand(index - 1) + || (self.is(index, "AND") && self.follows_between(index)) + { + continue; + } + let multiline = self + .left_operand_start(index - 1) + .is_some_and(|start| self.spans_lines(start, index - 1)) + || self + .right_operand_end(index + 1) + .is_some_and(|end| self.spans_lines(index + 1, end)); + if multiline { + let (first, _) = self.line_of(index); + self.gaps[index] = newline(self.indentation(first)); + } + } + } + + /// Put the first DECODE argument on its own line and each search/result pair on one line. + fn pair_decode_arguments(&mut self, name: usize) { + let opener = name + 1; + if !self.is(opener, OPEN_PARENTHESIS) || (name > 0 && self.is(name - 1, ".")) { + return; + } + let Some(closer) = self.partners[opener] else { + return; + }; + if !self.gaps[opener + 1].contains('\n') { + return; + } + let mut argument = 0_usize; + for index in opener + 2..closer { + if self.depths[index] != self.depths[opener] + 1 || self.tokens[index - 1] != COMMA { + continue; + } + argument += 1; + if argument.is_multiple_of(2) && self.gaps[index].trim().is_empty() { + self.gaps[index] = SINGLE_SPACE.to_string(); + } + } + } + + /// Join an AND or OR line break inside a bracket group whose arguments share its line. + fn join_chain_break_in_group(&mut self, index: usize) { + if !self.gaps[index].contains('\n') || !self.gaps[index].trim().is_empty() { + return; + } + let depth = self.depths[index]; + let Some(opener) = (0..index) + .rev() + .find(|&previous| self.depths[previous] < depth) + else { + return; + }; + if self.tokens[opener] == OPEN_PARENTHESIS && !self.spans_lines(opener, index - 1) { + self.gaps[index] = SINGLE_SPACE.to_string(); + } + } + + /// Start every row of a VALUES list on its own line unless the whole list fits on its line. + fn break_values_rows_unless_fitting(&mut self, values: usize, line_width: usize) { + let rows = self.values_rows(values); + let Some(&(_, last_closer)) = rows.last() else { + return; + }; + let (first, last) = self.line_of(values); + let fits = !self.spans_lines(values, last_closer) + && self + .line(first, last) + .end_columns + .is_some_and(|ends| ends.last().is_some_and(|end| *end <= line_width)); + if fits { + return; + } + let mut indentation = self.indentation(first); + if self.gaps[values].contains('\n') { + indentation = self.indentation(values); + } else if values > 0 + && self.tokens[values - 1] == OPEN_PARENTHESIS + && let Some(closer) = self.partners[values - 1] + { + self.set_break(values, &newline(indentation + CONTINUATION_INDENT)); + self.set_break(closer, &newline(indentation)); + indentation += CONTINUATION_INDENT; + } + let row_line = newline(indentation + CONTINUATION_INDENT); + for (opener, _) in rows { + self.set_break(opener, &row_line); + } + } + + /// Separate the CTEs of the WITH at `with`, and the statement after them, by one blank line. + fn separate_common_table_expressions(&mut self, with: usize) { + let (first, _) = self.line_of(with); + let indentation = self.indentation(first); + let mut item = with + 1; + if self.is(item, "RECURSIVE") { + item += 1; + } + while let Some(end) = self.common_table_expression_end(item) { + let next = end + 1; + if self.is(next, ",") { + self.blank_line_before(next + 1, indentation); + item = next + 1; + continue; + } + if next < self.len() && !self.is(next, ";") && !self.is(next, ")") { + self.blank_line_before(next, indentation); + } + break; + } + } + + /// The last token of the CTE, or CTE macro placeholder, that starts at `item`. + fn common_table_expression_end(&self, item: usize) -> Option { + let name = self.tokens.get(item)?; + if name.to_ascii_uppercase().starts_with(SENTINEL_PREFIX) + && !self.is(item + 1, "AS") + && !self.is(item + 1, OPEN_PARENTHESIS) + { + return Some(item); + } + if !matches!( + self.kinds[item], + TokenType::Var | TokenType::Identifier | TokenType::QuotedIdentifier + ) && !name.chars().all(|c| c.is_alphanumeric() || c == '_') + { + return None; + } + let mut index = item + 1; + if self.is(index, OPEN_PARENTHESIS) { + index = self.partners[index]? + 1; + } + if !self.is(index, "AS") { + return None; + } + index += 1; + if self.is(index, "NOT") { + index += 1; + } + if self.is(index, "MATERIALIZED") { + index += 1; + } + if !self.is(index, OPEN_PARENTHESIS) { + return None; + } + self.partners[index] + } + + /// Replace the line break before `index` with exactly one blank line, keeping comments. + fn blank_line_before(&mut self, index: usize, indentation: usize) { + let gap = &self.gaps[index]; + let (trailing, rest) = gap.split_once('\n').unwrap_or((gap.as_str(), "")); + let rest = rest.trim_start(); + let margin = " ".repeat(indentation); + self.gaps[index] = if rest.is_empty() { + format!("{}\n\n{margin}", trailing.trim_end()) + } else { + format!("{}\n\n{margin}{rest}", trailing.trim_end()) + }; + } + + /// Move a clause body that is one line under its clause keyword onto the keyword line. + fn join_single_line_body(&mut self, clause: usize, line_width: usize) { + let (first, last) = self.line_of(clause); + let body = last + 1; + let is_select = self.is(clause, "SELECT"); + let modifiers_only = (clause + 1..=last).all(|index| { + SELECT_MODIFIERS.contains(&self.upper(index).as_str()) + || self.kinds[index] == TokenType::Number + || matches!(self.tokens[index].as_str(), "(" | ")" | ",") + || self.depths[index] > self.depths[clause] + }); + if body >= self.len() + || clause != first + || (!is_select && clause != last) + || (is_select && !modifiers_only) + || !self.gaps[body].contains('\n') + || !self.gaps[body].trim().is_empty() + { + return; + } + let clause_indentation = self.indentation(first); + if self.indentation(body) <= clause_indentation { + return; + } + let (_, body_last) = self.line_of(body); + if body_last + 1 < self.len() && self.indentation(body_last + 1) > clause_indentation { + return; + } + let pinned = (body + 1..=body_last).any(|index| !self.gaps[index].trim().is_empty()) + || (first..=body_last).any(|index| self.tokens[index].contains('\n')); + if pinned || (!is_select && self.has_top_level_chain(body, body_last)) { + return; + } + let mut width = clause_indentation; + for index in first..=body_last { + if index > first { + width += if index == body { + 1 + } else { + self.gaps[index].chars().count() + }; + } + width += self.width(index); + } + width += self.gaps[body_last + 1] + .split('\n') + .next() + .unwrap_or_default() + .chars() + .count(); + if width <= line_width { + self.gaps[body] = SINGLE_SPACE.to_string(); + } + } + + fn has_top_level_chain(&self, first: usize, last: usize) -> bool { + let depth = (first..=last).map(|index| self.depths[index]).min(); + (first..=last).any(|index| { + Some(self.depths[index]) == depth + && CHAIN_KEYWORDS.contains(&self.upper(index).as_str()) + && !(self.is(index, "AND") && self.follows_between(index)) + }) + } + + /// Whether the AND at `index` separates the bounds of a BETWEEN. + fn follows_between(&self, index: usize) -> bool { + let depth = self.depths[index]; + for previous in (0..index).rev() { + if self.depths[previous] < depth { + return false; + } + if self.depths[previous] > depth { + continue; + } + let upper = self.upper(previous); + if upper == BETWEEN_KEYWORD { + return true; + } + if BETWEEN_BOUNDARIES.contains(&upper.as_str()) { + return false; + } + } + false + } + + fn ends_operand(&self, index: usize) -> bool { + matches!(self.tokens[index].as_str(), ")" | "]") + || self.is(index, "END") + || matches!( + self.kinds[index], + TokenType::Var + | TokenType::Identifier + | TokenType::QuotedIdentifier + | TokenType::Number + | TokenType::String + ) + } + + /// The first token of a bracket group or CASE expression that ends at `last`. + fn left_operand_start(&self, last: usize) -> Option { + if matches!(self.tokens[last].as_str(), ")" | "]") { + return self.partners[last]; + } + if !self.is(last, "END") { + return None; + } + let depth = self.depths[last]; + let mut nested = 0_usize; + for previous in (0..last).rev() { + if self.depths[previous] != depth { + continue; + } + if self.is(previous, "END") { + nested += 1; + } else if self.is(previous, "CASE") { + if nested == 0 { + return Some(previous); + } + nested -= 1; + } + } + None + } + + /// The last token of a bracket group, call, or CASE expression that starts at `first`. + fn right_operand_end(&self, first: usize) -> Option { + if self.tokens[first] == OPEN_PARENTHESIS { + return self.partners[first]; + } + let word = self.tokens[first] + .chars() + .all(|character| character.is_alphanumeric() || character == '_'); + if word && self.is(first + 1, OPEN_PARENTHESIS) { + return self.partners[first + 1]; + } + if !self.is(first, CASE_KEYWORD) { + return None; + } + self.case_end(first) + } +} + +/// Line width and the printed widths of tokens that stand in for longer authored text. +pub(crate) struct WrapOptions<'a> { + pub(crate) line_width: usize, + pub(crate) token_widths: &'a HashMap, +} + +/// Re-break every line of `printed` that is longer than the line width and can be shortened. +pub(crate) fn wrap_lines(printed: &mut PrintedSql<'_>, line_width: usize) { + printed.lock_unbreakable_groups(); + let mut pending: Vec<(usize, usize)> = printed.line_ranges(0, printed.len() - 1); + pending.reverse(); + while let Some((first, last)) = pending.pop() { + let line = printed.line(first, last); + if !overflows(&line, line_width) || !printed.break_line(&line) { + continue; + } + let mut lines = printed.line_ranges(first, last); + lines.reverse(); + pending.extend(lines); + } +} + +fn overflows(line: &Line, line_width: usize) -> bool { + line.end_columns + .as_ref() + .is_some_and(|ends| ends.last().is_some_and(|end| *end > line_width)) +} + +impl PrintedSql<'_> { + /// Lock the inside of VALUES rows and of type parameter lists such as `NUMBER(38, 2)`. + fn lock_unbreakable_groups(&mut self) { + for index in 0..self.len() { + if self.is(index, VALUES_KEYWORD) { + for (opener, closer) in self.values_rows(index) { + self.lock(opener, closer); + } + } else if let Some(closer) = self.type_parameters_closer(index) { + self.lock(index - 1, closer); + } + } + } + + fn lock(&mut self, first: usize, last: usize) { + for gap in first + 1..=last { + self.locked[gap] = true; + } + } + + /// The bracketed rows that directly follow the VALUES keyword at `values`. + fn values_rows(&self, values: usize) -> Vec<(usize, usize)> { + let mut rows: Vec<(usize, usize)> = Vec::new(); + let mut opener = values + 1; + while opener < self.len() && self.tokens[opener] == OPEN_PARENTHESIS { + let Some(closer) = self.partners[opener] else { + break; + }; + rows.push((opener, closer)); + if !(self.is(closer + 1, COMMA) && self.is(closer + 2, OPEN_PARENTHESIS)) { + break; + } + opener = closer + 2; + } + rows + } + + /// The closer of a type parameter list opened at `opener`, such as `(38, 2)` of `NUMBER`. + fn type_parameters_closer(&self, opener: usize) -> Option { + if opener < 1 || self.tokens[opener] != OPEN_PARENTHESIS { + return None; + } + let closer = self.partners[opener]?; + let name = &self.tokens[opener - 1]; + if closer == opener + 1 || !name.chars().all(|c| c.is_alphanumeric() || c == '_') { + return None; + } + let type_position = opener > 1 && (self.is(opener - 2, "::") || self.is(opener - 2, "AS")); + if !type_position && !TYPE_NAMES.contains(&name.to_ascii_uppercase().as_str()) { + return None; + } + let parameters = (opener + 1..closer).all(|index| { + let token = &self.tokens[index]; + token == COMMA + || is_digits(token) + || TYPE_PARAMETER_WORDS.contains(&token.to_ascii_uppercase().as_str()) + }); + parameters.then_some(closer) + } + + /// Apply the outermost applicable break: chains, VALUES rows, bracket groups, then commas. + fn break_line(&mut self, line: &Line) -> bool { + let values_row = self.tokens[line.first] == OPEN_PARENTHESIS && self.locked[line.first + 1]; + !values_row + && (self.break_before(line, &CHAIN_KEYWORDS, 0) + || self.break_before(line, &ARITHMETIC_OPERATORS, CONTINUATION_INDENT) + || self.break_values_rows(line) + || self.explode_group(line) + || self.break_after_commas(line)) + } + + /// Put each row of a VALUES list that starts this line on its own line. + fn break_values_rows(&mut self, line: &Line) -> bool { + let depth = self.line_depth(line); + let Some(values) = (line.first..=line.last).find(|&index| { + self.is(index, VALUES_KEYWORD) + && (self.depths[index] == depth || index == line.first) + && !self.locked[index + 1] + }) else { + return false; + }; + let rows = self.values_rows(values); + if rows.is_empty() + || rows + .iter() + .all(|(opener, _)| self.gaps[*opener].contains('\n')) + { + return false; + } + let row_line = newline(line.indentation + CONTINUATION_INDENT); + for (opener, _) in rows { + self.gaps[opener] = row_line.clone(); + } + true + } + + /// Put the arguments or CASE branches of the outermost, widest group on their own lines. + fn explode_group(&mut self, line: &Line) -> bool { + let Some(ends) = line.end_columns.as_ref() else { + return false; + }; + let start_column = |index: usize| { + if index == line.first { + line.indentation + } else { + ends[index - 1 - line.first] + } + }; + let line_end = ends[ends.len() - 1]; + let mut chosen: Option<(usize, usize, usize)> = None; + for opener in line.first..=line.last { + let Some(closer) = self.group_closer(opener, line) else { + continue; + }; + let end_column = if closer > line.last { + line_end + } else { + ends[closer - line.first] + }; + let width = end_column - start_column(opener); + let better = chosen.is_none_or(|(best, _, best_width)| { + self.depths[opener] < self.depths[best] + || (self.depths[opener] == self.depths[best] && width >= best_width) + }); + if better { + chosen = Some((opener, closer, width)); + } + } + let Some((opener, closer, _)) = chosen else { + return false; + }; + if closer > line.last { + self.indent_lines(opener + 1, closer - 1, CONTINUATION_INDENT); + } + let inner = newline(line.indentation + CONTINUATION_INDENT); + if self.is(opener, CASE_KEYWORD) { + self.explode_case(opener, closer, line.indentation); + return true; + } + self.set_break(opener + 1, &inner); + self.set_break(closer, &newline(line.indentation)); + if self.is(opener + 1, VALUES_KEYWORD) { + return true; + } + let window = opener > 0 && self.is(opener - 1, "OVER"); + let decode = opener > 0 && self.is(opener - 1, DECODE_FUNCTION); + let mut argument = 0_usize; + for index in opener + 2..closer { + if self.depths[index] != self.depths[opener] + 1 { + continue; + } + if window { + if WINDOW_CLAUSE_KEYWORDS.contains(&self.upper(index).as_str()) { + self.set_break(index, &inner); + } + } else if self.tokens[index - 1] == COMMA { + argument += 1; + if !decode || !argument.is_multiple_of(2) { + self.set_break(index, &inner); + } + } + } + true + } + + /// The closer of an explodable group opened at `opener` on `line`. + fn group_closer(&self, opener: usize, line: &Line) -> Option { + if self.is(opener, CASE_KEYWORD) { + let end = self.case_end(opener)?; + return (end <= line.last && end > opener + 2).then_some(end); + } + let closer = self.partners[opener]?; + let explodable = closer > opener + 2 + && matches!(self.tokens[opener].as_str(), "(" | "[" | "{") + && !self.locked[opener + 1] + && (closer <= line.last || opener < line.last); + explodable.then_some(closer) + } + + /// The END of the CASE expression at `case`. + fn case_end(&self, case: usize) -> Option { + let depth = self.depths[case]; + let mut nested = 0_usize; + for index in case + 1..self.len() { + if self.depths[index] < depth { + return None; + } + if self.depths[index] != depth { + continue; + } + if self.is(index, CASE_KEYWORD) { + nested += 1; + } else if self.is(index, "END") { + if nested == 0 { + return Some(index); + } + nested -= 1; + } + } + None + } + + /// Put each WHEN, THEN and ELSE of a one-line CASE on its own line and END under CASE. + fn explode_case(&mut self, case: usize, end: usize, indentation: usize) { + let inner = newline(indentation + CONTINUATION_INDENT); + let depth = self.depths[case]; + let mut nested = 0_usize; + for index in case + 1..end { + if self.depths[index] != depth { + continue; + } + if self.is(index, CASE_KEYWORD) { + nested += 1; + } else if self.is(index, "END") { + nested -= 1; + } else if nested == 0 && CASE_BRANCH_KEYWORDS.contains(&self.upper(index).as_str()) { + self.set_break(index, &inner); + } + } + self.set_break(end, &newline(indentation)); + } + + /// Replace a whitespace-only gap; comments keep their gaps. + fn set_break(&mut self, index: usize, gap: &str) { + if self.gaps[index].trim().is_empty() { + self.gaps[index] = gap.to_string(); + } + } + + /// Indent lines starting within `first..=last` by `extra`, except gaps with block comments. + fn indent_lines(&mut self, first: usize, last: usize, extra: usize) { + let margin = format!("\n{}", " ".repeat(extra)); + for index in first..=last { + if self.gaps[index].contains('\n') && !self.gaps[index].contains("/*") { + self.gaps[index] = self.gaps[index].replace('\n', &margin); + } + } + } + + /// Break before every operator of `operators` at the line's outermost depth. + fn break_before(&mut self, line: &Line, operators: &[&str], extra_indent: usize) -> bool { + let depth = self.line_depth(line); + let mut between = false; + let mut breaks: Vec = Vec::new(); + for index in line.first..=line.last { + if self.depths[index] != depth || self.locked[index] { + continue; + } + let upper = self.upper(index); + if upper == BETWEEN_KEYWORD { + between = true; + continue; + } + if upper == BETWEEN_SEPARATOR && between { + between = false; + continue; + } + if index > line.first + && operators.contains(&upper.as_str()) + && self.gaps[index] == SINGLE_SPACE + && ends_operand(&self.tokens[index - 1]) + { + breaks.push(index); + } + } + let indentation = newline(line.indentation + extra_indent); + for index in &breaks { + self.gaps[*index] = indentation.clone(); + } + !breaks.is_empty() + } + + /// Break after every comma at the outermost comma depth of the line. + fn break_after_commas(&mut self, line: &Line) -> bool { + let Some(depth) = (line.first..line.last) + .filter(|&index| self.tokens[index] == COMMA && !self.locked[index + 1]) + .map(|index| self.depths[index]) + .min() + else { + return false; + }; + let indentation = newline(line.indentation + CONTINUATION_INDENT); + let mut changed = false; + for index in line.first..line.last { + if self.depths[index] == depth && self.tokens[index] == COMMA && !self.locked[index + 1] + { + self.gaps[index + 1] = indentation.clone(); + changed = true; + } + } + changed + } + + fn line_depth(&self, line: &Line) -> usize { + (line.first..=line.last) + .map(|index| self.depths[index]) + .min() + .unwrap_or_default() + } +} + +fn is_digits(token: &str) -> bool { + token.chars().all(|character| character.is_ascii_digit()) +} + +fn ends_operand(token: &str) -> bool { + token.chars().last().is_some_and(|character| { + character.is_alphanumeric() || matches!(character, '_' | '\'' | '"' | '`' | ')' | ']') + }) +} diff --git a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/token_layout.rs b/crates/sqlbuild-rules-native/src/sql_lint/_helpers/token_layout.rs new file mode 100644 index 0000000000..488b057c41 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_lint/_helpers/token_layout.rs @@ -0,0 +1,694 @@ +//! Print authored SQL tokens with whitespace taken from a parse-tree layout oracle. + +use std::collections::{BTreeMap, BTreeSet}; + +use polyglot_sql::Dialect; +use polyglot_sql::tokens::{Token, TokenType}; + +use crate::sql_lint::constants::{CLOSE_PARENTHESIS, OPEN_PARENTHESIS}; +use crate::sql_tokens::main::canonical_tokens::canonical_tokens; +use crate::sql_tokens::main::case_folding::foldable_tokens; +use crate::sql_tokens::main::is_unquoted_word::is_unquoted_word; +use crate::sql_tokens::main::name_positions::name_positions; +use crate::sql_tokens::main::token_texts::token_texts; + +const TOKEN_PRESERVATION_FAILURE: &str = + "native formatter would change authored SQL tokens, not only layout"; +const COMMENT_ATTACHMENT_FAILURE: &str = + "native formatter could not preserve comment token attachments"; +const STRING_BREAK_FAILURE: &str = + "native formatter would join adjacent string literals across a line break"; +const ALIGNMENT_FAILURE: &str = "native formatter could not align its layout with authored tokens"; +const RESYNC_WINDOW: usize = 48; +const RESYNC_CONFIRMATION: usize = 1; +const NO_SPACE_BEFORE: [&str; 7] = [",", ")", "]", ".", ";", "::", ":"]; +const NO_SPACE_AFTER: [&str; 5] = ["(", "[", ".", "::", ":"]; +const STRUCTURAL_PUNCTUATION: [&str; 6] = ["(", ")", "[", "]", ",", ";"]; + +#[derive(Debug, Clone)] +pub(crate) struct Comment { + pub(crate) start: usize, + pub(crate) end: usize, + pub(crate) text: String, + pub(crate) line: bool, + pub(crate) leading: bool, +} + +/// One authored or oracle token with its raw source text and alignment key. +struct Lexeme { + start: usize, + end: usize, + raw: String, + key: String, + /// In the reserved-keyword and built-in fold set shared with the change fingerprint. + foldable: bool, + /// An unquoted authored word outside the name positions that keep reserved words exact. + recasable: bool, + /// An unquoted word the dialect tokenizer types as a keyword. + keyword: bool, +} + +/// An authored token the printer recased to the layout guide's keyword spelling. +pub(crate) struct KeywordRecase { + pub(crate) generated: String, + pub(crate) keyword: bool, +} + +/// Comments attached to the authored token they precede or follow. +#[derive(Default)] +struct Attached<'a> { + leading: Vec<&'a Comment>, + trailing: Vec<&'a Comment>, +} + +pub(crate) fn comments_in(sql: &str, tokens: &[Token]) -> Vec { + let characters: Vec = sql.chars().collect(); + let mut comments: Vec = Vec::new(); + let mut gap_start = 0_usize; + for token in tokens { + comments.extend(comments_in_gap(&characters, gap_start, token.span.start)); + gap_start = token.span.end; + } + comments.extend(comments_in_gap(&characters, gap_start, characters.len())); + comments +} + +/// Split the non-whitespace text the tokenizer skipped between two tokens into comments. +fn comments_in_gap(characters: &[char], start: usize, end: usize) -> Vec { + let mut comments: Vec = Vec::new(); + let mut index = start; + while index < end { + if characters[index].is_whitespace() { + index += 1; + continue; + } + let start = index; + let line_start = (0..start) + .rev() + .find(|&position| characters[position] == '\n') + .map_or(0, |position| position + 1); + let leading = characters[line_start..start] + .iter() + .all(|character| character.is_whitespace()); + let block = characters[index] == '/' && characters.get(index + 1) == Some(&'*'); + index = if block { + block_comment_end(characters, start, end) + } else { + (start..end) + .find(|&position| characters[position] == '\n') + .unwrap_or(end) + }; + comments.push(Comment { + start, + end: index, + text: characters[start..index].iter().collect(), + line: !block, + leading, + }); + } + comments +} + +/// End of a block comment: nested comments when they close inside the gap, else the first `*/`. +fn block_comment_end(characters: &[char], start: usize, end: usize) -> usize { + let closes_at = |position: usize| { + position + 1 < end && characters[position] == '*' && characters[position + 1] == '/' + }; + let opens_at = |position: usize| { + position + 1 < end && characters[position] == '/' && characters[position + 1] == '*' + }; + let mut depth = 0_usize; + let mut position = start; + while position < end { + if opens_at(position) { + depth += 1; + position += 2; + } else if closes_at(position) { + depth -= 1; + position += 2; + if depth == 0 { + return position; + } + } else { + position += 1; + } + } + (start + 2..end) + .find(|&position| closes_at(position)) + .map_or(end, |position| position + 2) +} + +pub(crate) fn neutralize_comments(sql: &str, comments: &[Comment]) -> String { + let mut characters: Vec = sql.chars().collect(); + for comment in comments { + for character in &mut characters[comment.start..comment.end] { + if *character != '\n' && *character != '\r' { + *character = ' '; + } + } + } + characters.into_iter().collect() +} + +/// Authored SQL text with its dialect tokens and the comments between them. +pub(crate) struct AuthoredSql<'a> { + pub(crate) sql: &'a str, + pub(crate) tokens: &'a [Token], + pub(crate) comments: &'a [Comment], +} + +/// Print every authored token in order, taking only whitespace and keyword case from `oracle`. +pub(crate) fn print_authored_tokens( + source: &AuthoredSql<'_>, + oracle: &str, + dialect: &Dialect, +) -> Result<(String, BTreeMap), String> { + let AuthoredSql { + sql, + tokens, + comments, + } = *source; + let authored = lexemes(sql, tokens, Some(dialect))?; + let oracle_tokens = dialect + .tokenize(oracle) + .map_err(|error| error.to_string())?; + let generated = lexemes(oracle, &oracle_tokens, None)?; + let alignment = align(&authored, &generated)?; + let oracle_characters: Vec = oracle.chars().collect(); + let separators = separators(&authored, &generated, &alignment, &oracle_characters)?; + let attached = attach_comments(tokens, comments); + let mut output = String::with_capacity(sql.len() + sql.len() / 4); + let string_breaks = string_breaks(sql, tokens); + let mut recases: BTreeMap = BTreeMap::new(); + for (index, lexeme) in authored.iter().enumerate() { + let forced_break = string_breaks.contains(&index) && !separators[index].contains('\n'); + let separator = &if forced_break { + format!("\n{}", current_indentation(&output)) + } else { + separators[index].clone() + }; + let leading = &attached[index].leading; + if leading.is_empty() { + output.push_str(separator); + } else if index == 0 || separator.contains('\n') { + output.push_str(separator); + let indentation = current_indentation(&output); + for comment in leading { + output.push_str(&comment.text); + output.push('\n'); + output.push_str(&indentation); + } + } else { + let indentation = current_indentation(&output); + for comment in leading { + output.push('\n'); + output.push_str(&indentation); + output.push_str(&comment.text); + } + output.push('\n'); + output.push_str(&indentation); + } + let guide = alignment[index].map(|at| &generated[at]); + if let Some(guide) = guide.filter(|guide| confirms_keyword(lexeme, guide)) { + recases.insert( + index, + KeywordRecase { + generated: guide.raw.clone(), + keyword: guide.keyword, + }, + ); + } + output.push_str(&printed_text(lexeme, guide)); + let trailing = &attached[index].trailing; + for (position, comment) in trailing.iter().enumerate() { + output.push(' '); + output.push_str(&comment.text); + if !comment.line { + continue; + } + let next_starts_line = if position + 1 < trailing.len() { + false + } else { + separators + .get(index + 1) + .is_some_and(|next| next.starts_with('\n')) + }; + if !next_starts_line { + output.push('\n'); + if position + 1 < trailing.len() { + output.push_str(¤t_indentation(&output)); + } + } + } + } + Ok((output, recases)) +} + +/// Refuse output that changes anything but layout, fold-set case, and recorded keyword recases. +pub(crate) fn verify_token_invariant( + before: &str, + after: &str, + dialect: &Dialect, + recases: &BTreeMap, +) -> Result<(), String> { + let before_tokens = tokenize(before, dialect)?; + let after_tokens = tokenize(after, dialect)?; + let before_canonical = canonical_tokens(before, &before_tokens, dialect)?; + let after_canonical = canonical_tokens(after, &after_tokens, dialect)?; + if before_canonical.len() != after_canonical.len() { + return Err(TOKEN_PRESERVATION_FAILURE.to_string()); + } + let before_texts = token_texts(before, &before_tokens)?; + let after_texts = token_texts(after, &after_tokens)?; + for (index, (left, right)) in before_canonical.iter().zip(&after_canonical).enumerate() { + if left != right + && !recases + .get(&index) + .is_some_and(|recase| recase.allows(&before_texts[index], &after_texts[index])) + { + return Err(TOKEN_PRESERVATION_FAILURE.to_string()); + } + } + if comment_positions(before, &before_tokens) != comment_positions(after, &after_tokens) { + return Err(COMMENT_ATTACHMENT_FAILURE.to_string()); + } + if !string_breaks(before, &before_tokens).is_subset(&string_breaks(after, &after_tokens)) { + return Err(STRING_BREAK_FAILURE.to_string()); + } + Ok(()) +} + +impl KeywordRecase { + /// Whether `authored` became `printed` only by taking the guide's upper-case keyword spelling. + fn allows(&self, authored: &str, printed: &str) -> bool { + self.keyword + && printed == self.generated + && printed == printed.to_ascii_uppercase() + && authored.eq_ignore_ascii_case(printed) + } +} + +fn tokenize(sql: &str, dialect: &Dialect) -> Result, String> { + dialect.tokenize(sql).map_err(|error| error.to_string()) +} + +fn comment_positions(sql: &str, tokens: &[Token]) -> Vec<(String, usize)> { + let mut positions: Vec<(String, usize)> = Vec::new(); + for comment in comments_in(sql, tokens) { + let preceding = tokens.partition_point(|token| token.span.end <= comment.start); + positions.push((comment.text, preceding)); + } + positions +} + +/// Lexemes of `sql`; only authored lexemes carry the recase classification. +fn lexemes(sql: &str, tokens: &[Token], dialect: Option<&Dialect>) -> Result, String> { + let raws = token_texts(sql, tokens)?; + let (foldable, recasable) = match dialect { + Some(dialect) => { + let uppers: Vec = raws.iter().map(|raw| raw.to_ascii_uppercase()).collect(); + let names = name_positions(&uppers); + let foldable = foldable_tokens(&raws, tokens, dialect); + let recasable: Vec = raws + .iter() + .zip(names) + .map(|(raw, name)| !name && is_unquoted_word(raw)) + .collect(); + (foldable, recasable) + } + None => (vec![false; tokens.len()], vec![false; tokens.len()]), + }; + let mut lexemes = Vec::with_capacity(tokens.len()); + for (index, (raw, token)) in raws.into_iter().zip(tokens).enumerate() { + lexemes.push(lexeme(raw, token, foldable[index], recasable[index])); + } + Ok(lexemes) +} + +fn lexeme(raw: String, token: &Token, foldable: bool, recasable: bool) -> Lexeme { + let key = if is_unquoted_word(&raw) { + format!("w:{}", raw.to_ascii_uppercase()) + } else if raw + .chars() + .all(|character| character.is_ascii_punctuation()) + { + format!("p:{:?}", token.token_type) + } else { + format!("r:{raw}") + }; + let keyword = is_unquoted_word(&raw) + && !matches!( + token.token_type, + TokenType::Var | TokenType::Identifier | TokenType::QuotedIdentifier + ); + Lexeme { + start: token.span.start, + end: token.span.end, + raw, + key, + foldable, + recasable, + keyword, + } +} + +fn is_string_literal(token: &Token) -> bool { + matches!( + token.token_type, + TokenType::String + | TokenType::NationalString + | TokenType::EscapeString + | TokenType::RawString + | TokenType::ByteString + | TokenType::UnicodeString + | TokenType::HexString + | TokenType::BitString + | TokenType::DollarString + | TokenType::TripleSingleQuotedString + | TokenType::TripleDoubleQuotedString + ) +} + +/// Indices of string literals that follow another string literal across a line break. +fn string_breaks(sql: &str, tokens: &[Token]) -> BTreeSet { + let characters: Vec = sql.chars().collect(); + (1..tokens.len()) + .filter(|&index| { + is_string_literal(&tokens[index - 1]) + && is_string_literal(&tokens[index]) + && characters[tokens[index - 1].span.end..tokens[index].span.start].contains(&'\n') + }) + .collect() +} + +/// Map authored tokens to matching oracle tokens; a closer only matches its opener's partner. +fn align(authored: &[Lexeme], generated: &[Lexeme]) -> Result>, String> { + let mut state = Alignment { + authored, + generated, + authored_partners: bracket_partners(authored), + generated_partners: bracket_partners(generated), + forward: vec![None; authored.len()], + backward: vec![None; generated.len()], + }; + let (mut left, mut right) = (0_usize, 0_usize); + while left < authored.len() && right < generated.len() { + if state.matches(left, right) { + state.forward[left] = Some(right); + state.backward[right] = Some(left); + left += 1; + right += 1; + continue; + } + let (skip_left, skip_right) = state + .resync(left, right) + .ok_or_else(|| ALIGNMENT_FAILURE.to_string())?; + left += skip_left; + right += skip_right; + } + Ok(state.forward) +} + +struct Alignment<'a> { + authored: &'a [Lexeme], + generated: &'a [Lexeme], + authored_partners: Vec>, + generated_partners: Vec>, + forward: Vec>, + backward: Vec>, +} + +impl Alignment<'_> { + fn matches(&self, left: usize, right: usize) -> bool { + if self.authored[left].key != self.generated[right].key { + return false; + } + if !is_closer(&self.authored[left].raw) { + return true; + } + match (self.authored_partners[left], self.generated_partners[right]) { + (Some(opener), Some(generated_opener)) => match self.forward[opener] { + Some(aligned) => aligned == generated_opener, + None => self.backward[generated_opener].is_none(), + }, + (None, None) => true, + _ => false, + } + } + + fn resync(&self, left: usize, right: usize) -> Option<(usize, usize)> { + let remaining_left = self.authored.len() - left; + let remaining_right = self.generated.len() - right; + for cost in 1..=(2 * RESYNC_WINDOW) { + for skip_left in 0..=cost.min(RESYNC_WINDOW) { + let skip_right = cost - skip_left; + if skip_right > RESYNC_WINDOW { + continue; + } + if skip_left >= remaining_left || skip_right >= remaining_right { + if skip_left >= remaining_left && skip_right >= remaining_right { + return Some((remaining_left, remaining_right)); + } + continue; + } + if !self.matches(left + skip_left, right + skip_right) { + continue; + } + let anchor = &self.authored[left + skip_left]; + let confirmations = if anchor.key.starts_with("p:") { + RESYNC_CONFIRMATION + } else { + 0 + }; + let confirmed = (1..=confirmations).all(|offset| { + match ( + self.authored.get(left + skip_left + offset), + self.generated.get(right + skip_right + offset), + ) { + (Some(before), Some(after)) => before.key == after.key, + _ => true, + } + }); + if confirmed { + return Some((skip_left, skip_right)); + } + } + } + None + } +} + +fn is_closer(raw: &str) -> bool { + matches!(raw, ")" | "]" | "}") +} + +fn bracket_partners(lexemes: &[Lexeme]) -> Vec> { + let mut partners: Vec> = vec![None; lexemes.len()]; + let mut open: Vec = Vec::new(); + for (index, lexeme) in lexemes.iter().enumerate() { + match lexeme.raw.as_str() { + "(" | "[" | "{" => open.push(index), + ")" | "]" | "}" => { + if let Some(opener) = open.pop() { + partners[opener] = Some(index); + partners[index] = Some(opener); + } + } + _ => {} + } + } + partners +} + +fn separators( + authored: &[Lexeme], + generated: &[Lexeme], + alignment: &[Option], + oracle: &[char], +) -> Result, String> { + let mut separators: Vec = vec![String::new(); authored.len()]; + let anchors: Vec = (0..authored.len()) + .filter(|&index| alignment[index].is_some()) + .collect(); + if anchors.is_empty() { + return Err(ALIGNMENT_FAILURE.to_string()); + } + let gap = |left: usize, right: usize| -> String { + oracle[generated[left].end..generated[right].start] + .iter() + .collect() + }; + for index in 1..anchors[0] { + separators[index] = default_space(&authored[index - 1], &authored[index]).to_string(); + } + if anchors[0] > 0 { + separators[anchors[0]] = + default_space(&authored[anchors[0] - 1], &authored[anchors[0]]).to_string(); + } + for pair in anchors.windows(2) { + let (previous, next) = (pair[0], pair[1]); + let (previous_at, next_at) = ( + alignment[previous].ok_or(ALIGNMENT_FAILURE)?, + alignment[next].ok_or(ALIGNMENT_FAILURE)?, + ); + let merged = merged_gap( + &(previous_at..next_at) + .map(|at| gap(at, at + 1)) + .collect::>(), + ); + if previous + 1 == next { + separators[next] = merged; + continue; + } + for index in previous + 2..next { + separators[index] = default_space(&authored[index - 1], &authored[index]).to_string(); + } + let first = previous + 1; + let last = next - 1; + let substitution = next_at > previous_at + 1; + separators[first] = if default_space(&authored[previous], &authored[first]).is_empty() { + String::new() + } else if substitution { + gap(previous_at, previous_at + 1) + } else { + " ".to_string() + }; + separators[next] = if substitution { + let oracle_gap = gap(next_at - 1, next_at); + let unspaced = default_space(&authored[last], &authored[next]).is_empty(); + if unspaced && !oracle_gap.contains('\n') { + String::new() + } else { + oracle_gap + } + } else if merged.contains('\n') { + merged + } else { + default_space(&authored[last], &authored[next]).to_string() + }; + } + let last_anchor = anchors[anchors.len() - 1]; + for index in last_anchor + 1..authored.len() { + separators[index] = default_space(&authored[index - 1], &authored[index]).to_string(); + } + let last_at = alignment[last_anchor].ok_or(ALIGNMENT_FAILURE)?; + if last_anchor + 1 < authored.len() + && last_at + 1 < generated.len() + && !separators[last_anchor + 1].is_empty() + { + separators[last_anchor + 1] = gap(last_at, last_at + 1); + } + for index in 1..authored.len() { + let adjacent_in_source = authored[index - 1].end == authored[index].start; + let oracle_pair = matches!( + (alignment[index - 1], alignment[index]), + (Some(previous), Some(next)) if previous + 1 == next + ); + let structural = [&authored[index - 1], &authored[index]] + .iter() + .any(|lexeme| STRUCTURAL_PUNCTUATION.contains(&lexeme.raw.as_str())); + let call = + is_unquoted_word(&authored[index - 1].raw) && authored[index].raw == OPEN_PARENTHESIS; + let inside_parentheses = + authored[index - 1].raw == OPEN_PARENTHESIS || authored[index].raw == CLOSE_PARENTHESIS; + if (adjacent_in_source && !oracle_pair && (!structural || call)) + || (inside_parentheses && !separators[index].contains('\n')) + { + separators[index].clear(); + } else if separators[index].is_empty() + && !adjacent_in_source + && would_fuse(&authored[index - 1], &authored[index]) + { + separators[index] = " ".to_string(); + } + } + Ok(separators) +} + +fn merged_gap(gaps: &[String]) -> String { + if let [only] = gaps { + return only.clone(); + } + if let Some(newline) = gaps.iter().rev().find(|gap| gap.contains('\n')) { + return newline.clone(); + } + if gaps.iter().any(|gap| !gap.is_empty()) { + " ".to_string() + } else { + String::new() + } +} + +fn default_space(previous: &Lexeme, next: &Lexeme) -> &'static str { + if NO_SPACE_BEFORE.contains(&next.raw.as_str()) + || NO_SPACE_AFTER.contains(&previous.raw.as_str()) + { + "" + } else { + " " + } +} + +/// Return whether printing two separated tokens without a space could lex them differently. +fn would_fuse(previous: &Lexeme, next: &Lexeme) -> bool { + let (Some(left), Some(right)) = (previous.raw.chars().last(), next.raw.chars().next()) else { + return false; + }; + let word = |character: char| { + character.is_alphanumeric() || matches!(character, '_' | '$' | '\'' | '"' | '`' | '@') + }; + let operator = |character: char| "+-*/<>=!|&^%~:#?".contains(character); + (word(left) && word(right)) || (operator(left) && operator(right)) +} + +/// The authored text, recased only for keywords and built-in function names. +fn printed_text(authored: &Lexeme, generated: Option<&Lexeme>) -> String { + if let Some(generated) = generated.filter(|generated| confirms_keyword(authored, generated)) { + return generated.raw.clone(); + } + if !authored.foldable { + return authored.raw.clone(); + } + match generated { + Some(generated) if generated.raw.eq_ignore_ascii_case(&authored.raw) => { + generated.raw.clone() + } + Some(_) => authored.raw.clone(), + None => authored.raw.to_ascii_uppercase(), + } +} + +/// Whether the layout guide prints the aligned authored word as an upper-case keyword. +fn confirms_keyword(authored: &Lexeme, generated: &Lexeme) -> bool { + !authored.foldable + && authored.recasable + && generated.keyword + && generated.raw != authored.raw + && generated.raw == generated.raw.to_ascii_uppercase() + && generated.raw.eq_ignore_ascii_case(&authored.raw) +} + +fn attach_comments<'a>(tokens: &[Token], comments: &'a [Comment]) -> Vec> { + let mut attached: Vec> = (0..tokens.len()).map(|_| Attached::default()).collect(); + for comment in comments { + let next = tokens.partition_point(|token| token.span.start < comment.end); + let preceding = tokens.partition_point(|token| token.span.end <= comment.start); + let next = (next < tokens.len()).then_some(next); + let previous = preceding.checked_sub(1); + match (comment.leading, next, previous) { + (true, Some(next), _) | (false, Some(next), None) => { + attached[next].leading.push(comment); + } + (_, _, Some(previous)) => attached[previous].trailing.push(comment), + (_, None, None) => {} + } + } + attached +} + +fn current_indentation(output: &str) -> String { + let line = output.rsplit('\n').next().unwrap_or_default(); + line.chars() + .take_while(|character| *character == ' ' || *character == '\t') + .collect() +} diff --git a/crates/sqlbuild-rules-native/src/sql_lint/constants.rs b/crates/sqlbuild-rules-native/src/sql_lint/constants.rs index 7d1409979e..ebb457fe57 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/constants.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/constants.rs @@ -2,7 +2,6 @@ pub(crate) const CAST_TYPE_SEPARATOR_KEYWORD: &str = "AS"; pub(crate) const CLOSE_PARENTHESIS: &str = ")"; pub(crate) const LINT_API_VERSION: u32 = 1; pub(crate) const OPEN_PARENTHESIS: &str = "("; -pub(crate) const VALUES_RELATION_PREFIX_TOKEN_COUNT: usize = 2; pub(crate) const POSITION_ARGUMENT_SEPARATOR: &str = ","; pub(crate) const CTE_MACRO_PLACEHOLDER_LITERAL: &str = "1"; pub(crate) const POSTFIX_CAST_PREFIX_TOKEN_COUNT: usize = 2; diff --git a/crates/sqlbuild-rules-native/src/sql_lint/models.rs b/crates/sqlbuild-rules-native/src/sql_lint/models.rs index b2848a5f73..b1acb86dbd 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/models.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/models.rs @@ -1,4 +1,4 @@ -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use polyglot_sql::tokens::Span; use serde::{Deserialize, Serialize}; @@ -125,6 +125,10 @@ pub(crate) struct FormatRequest { pub dialect: String, #[serde(default)] pub max_function_call_depth: Option, + #[serde(default)] + pub line_width: Option, + #[serde(default)] + pub token_widths: HashMap, } #[derive(Debug, Serialize)] diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests.rs b/crates/sqlbuild-rules-native/src/sql_lint/tests.rs index 61539eb14a..7b2b923819 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/tests.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/tests.rs @@ -6,8 +6,16 @@ mod batch_lint; mod comment_formatter; #[path = "tests/test_empty_fixture_lint.rs"] mod empty_fixture_lint; +#[path = "tests/test_format_corpus.rs"] +mod format_corpus; +#[path = "tests/test_function_call_spacing.rs"] +mod function_call_spacing; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_layout_rules.rs"] +mod layout_rules; +#[path = "tests/test_line_wrap.rs"] +mod line_wrap; #[path = "tests/test_sql_lint.rs"] mod sql_lint; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/fixtures/FORMAT_CORPUS_NOTICE.md b/crates/sqlbuild-rules-native/src/sql_lint/tests/fixtures/FORMAT_CORPUS_NOTICE.md new file mode 100644 index 0000000000..39e0868f78 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_lint/tests/fixtures/FORMAT_CORPUS_NOTICE.md @@ -0,0 +1,32 @@ +# Formatter corpus notice + +`format_corpus.json` contains SELECT and WITH queries taken from the SQLGlot dialect identity test +fixtures for BigQuery, Databricks, DuckDB, PostgreSQL, Snowflake and T-SQL, as extracted by the +Polyglot project. One literal was reworded. The expected `formatted` and `previously_formatted` +values are SQLBuild formatter output. + +The queries are used under the following license: + +```text +MIT License + +Copyright (c) 2026 Toby Mao + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/fixtures/format_corpus.json b/crates/sqlbuild-rules-native/src/sql_lint/tests/fixtures/format_corpus.json new file mode 100644 index 0000000000..b45dd71a3f --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_lint/tests/fixtures/format_corpus.json @@ -0,0 +1,5354 @@ +[ + { + "dialect": "bigquery", + "sql": "SELECT 'foo' 'bar'", + "formatted": "SELECT 'foo' 'bar'" + }, + { + "dialect": "bigquery", + "sql": "SELECT 'foo'/* c */'bar'", + "formatted": "SELECT\n 'foo' /* c */ 'bar'" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM `domain.com:project-id.mydataset.mytable`", + "formatted": "SELECT *\nFROM `domain.com:project-id.mydataset.mytable`" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM `domain.com:project-id.region-us.INFORMATION_SCHEMA`.JOBS", + "formatted": "SELECT *\nFROM `domain.com:project-id.region-us.INFORMATION_SCHEMA`.JOBS" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM `domain.com:project-id.region-us.INFORMATION_SCHEMA.JOBS`", + "formatted": "SELECT *\nFROM `domain.com:project-id.region-us.INFORMATION_SCHEMA.JOBS`" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM `a.b.com:project-id.mydataset.mytable`", + "formatted": "SELECT *\nFROM `a.b.com:project-id.mydataset.mytable`" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM `a.b.com:project-id.region-us.INFORMATION_SCHEMA.JOBS`", + "formatted": "SELECT *\nFROM `a.b.com:project-id.region-us.INFORMATION_SCHEMA.JOBS`" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM x-0.y", + "formatted": "SELECT *\nFROM x-0.y" + }, + { + "dialect": "bigquery", + "sql": "SELECT `p.d.UdF`(data) FROM `p.d.t`", + "formatted": "SELECT `p.d.UdF`(data)\nFROM `p.d.t`" + }, + { + "dialect": "bigquery", + "sql": "SELECT EXP(1)", + "formatted": "SELECT EXP(1)" + }, + { + "dialect": "bigquery", + "sql": "SELECT ARRAY_CONCAT([1])", + "formatted": "SELECT ARRAY_CONCAT([1])" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM READ_CSV('bla.csv')", + "formatted": "SELECT *\nFROM READ_CSV('bla.csv')" + }, + { + "dialect": "bigquery", + "sql": "SELECT jsondoc['some_key']", + "formatted": "SELECT jsondoc['some_key']" + }, + { + "dialect": "bigquery", + "sql": "SELECT `p.d.UdF`(data).* FROM `p.d.t`", + "formatted": "SELECT `p.d.UdF`(data).*\nFROM `p.d.t`" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM `my-project.my-dataset.my-table`", + "formatted": "SELECT *\nFROM `my-project.my-dataset.my-table`" + }, + { + "dialect": "bigquery", + "sql": "SELECT x, 1 AS y GROUP BY 1 ORDER BY 1", + "formatted": "SELECT\n x,\n 1 AS y\nGROUP BY\n 1\nORDER BY\n 1" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM x.*", + "formatted": "SELECT *\nFROM x.*" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM x.y*", + "formatted": "SELECT *\nFROM x.y*" + }, + { + "dialect": "bigquery", + "sql": "SELECT SEARCH(data_to_search, 'search_query')", + "formatted": "SELECT SEARCH(data_to_search, 'search_query')" + }, + { + "dialect": "bigquery", + "sql": "SELECT SEARCH(data_to_search, 'search_query', json_scope => 'JSON_KEYS_AND_VALUES')", + "formatted": "SELECT SEARCH(data_to_search, 'search_query', json_scope => 'JSON_KEYS_AND_VALUES')" + }, + { + "dialect": "bigquery", + "sql": "SELECT SEARCH(data_to_search, 'search_query', analyzer => 'PATTERN_ANALYZER')", + "formatted": "SELECT SEARCH(data_to_search, 'search_query', analyzer => 'PATTERN_ANALYZER')" + }, + { + "dialect": "bigquery", + "sql": "SELECT SEARCH(data_to_search, 'search_query', analyzer_options => 'analyzer_options_values')", + "formatted": "SELECT SEARCH(data_to_search, 'search_query', analyzer_options => 'analyzer_options_values')" + }, + { + "dialect": "bigquery", + "sql": "SELECT SEARCH(data_to_search, 'search_query', json_scope => 'JSON_VALUES', analyzer => 'LOG_ANALYZER')", + "formatted": "SELECT\n SEARCH(\n data_to_search,\n 'search_query',\n json_scope => 'JSON_VALUES',\n analyzer => 'LOG_ANALYZER'\n )" + }, + { + "dialect": "bigquery", + "sql": "SELECT SEARCH(data_to_search, 'search_query', analyzer => 'PATTERN_ANALYZER', analyzer_options => 'options')", + "formatted": "SELECT\n SEARCH(\n data_to_search,\n 'search_query',\n analyzer => 'PATTERN_ANALYZER',\n analyzer_options => 'options'\n )" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM dataset.my_table TABLESAMPLE SYSTEM (10 PERCENT)", + "formatted": "SELECT *\nFROM dataset.my_table TABLESAMPLE SYSTEM (10 PERCENT)" + }, + { + "dialect": "bigquery", + "sql": "SELECT '\\n\\r\\a\\v\\f\\t'", + "formatted": "SELECT '\\n\\r\\a\\v\\f\\t'" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM tbl FOR SYSTEM_TIME AS OF z", + "formatted": "SELECT *\nFROM tbl FOR SYSTEM_TIME AS OF z" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM tbl FOR SYSTEM TIME AS OF z", + "formatted": "SELECT *\nFROM tbl FOR SYSTEM TIME AS OF z" + }, + { + "dialect": "bigquery", + "sql": "SELECT PARSE_TIMESTAMP('%c', 'Thu Dec 25 07:30:00 2008', 'UTC')", + "formatted": "SELECT PARSE_TIMESTAMP('%c', 'Thu Dec 25 07:30:00 2008', 'UTC')" + }, + { + "dialect": "bigquery", + "sql": "SELECT ANY_VALUE(fruit HAVING MAX sold) FROM fruits", + "formatted": "SELECT ANY_VALUE(fruit HAVING MAX sold)\nFROM fruits" + }, + { + "dialect": "bigquery", + "sql": "SELECT ANY_VALUE(fruit HAVING MIN sold) FROM fruits", + "formatted": "SELECT ANY_VALUE(fruit HAVING MIN sold)\nFROM fruits" + }, + { + "dialect": "bigquery", + "sql": "SELECT `project-id`.udfs.func(call.dir)", + "formatted": "SELECT `project-id`.udfs.func(call.dir)" + }, + { + "dialect": "bigquery", + "sql": "SELECT CAST(CURRENT_DATE AS STRING FORMAT 'DAY') AS current_day", + "formatted": "SELECT CAST(CURRENT_DATE AS STRING FORMAT 'DAY') AS current_day" + }, + { + "dialect": "bigquery", + "sql": "SELECT foo IN UNNEST(bar) AS bla", + "formatted": "SELECT foo IN UNNEST(bar) AS bla" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM x-0.a", + "formatted": "SELECT *\nFROM x-0.a" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM pivot CROSS JOIN foo", + "formatted": "SELECT *\nFROM pivot\nCROSS JOIN foo" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM a-b-c.mydataset.mytable", + "formatted": "SELECT *\nFROM a-b-c.mydataset.mytable" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM abc-def-ghi", + "formatted": "SELECT *\nFROM abc-def-ghi" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM a-b-c", + "formatted": "SELECT *\nFROM a-b-c" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM my-table", + "formatted": "SELECT *\nFROM my-table" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM my-project.mydataset.mytable", + "formatted": "SELECT *\nFROM my-project.mydataset.mytable" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM pro-ject_id.c.d CROSS JOIN foo-bar", + "formatted": "SELECT *\nFROM pro-ject_id.c.d\nCROSS JOIN foo-bar" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM foo.bar.25", + "formatted": "SELECT *\nFROM foo.bar.25" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM foo.bar.25_", + "formatted": "SELECT *\nFROM foo.bar.25_" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM foo.bar.25x a", + "formatted": "SELECT *\nFROM foo.bar.25x a" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM foo.bar.25ab c", + "formatted": "SELECT *\nFROM foo.bar.25ab c" + }, + { + "dialect": "bigquery", + "sql": "SELECT b'abc'", + "formatted": "SELECT b'abc'" + }, + { + "dialect": "bigquery", + "sql": "SELECT AS STRUCT 1 AS a, 2 AS b", + "formatted": "SELECT AS STRUCT\n 1 AS a,\n 2 AS b" + }, + { + "dialect": "bigquery", + "sql": "SELECT DISTINCT AS STRUCT 1 AS a, 2 AS b", + "formatted": "SELECT DISTINCT AS STRUCT\n 1 AS a,\n 2 AS b" + }, + { + "dialect": "bigquery", + "sql": "SELECT AS VALUE STRUCT(1 AS a, 2 AS b)", + "formatted": "SELECT AS VALUE STRUCT(1 AS a, 2 AS b)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM q UNPIVOT(values FOR quarter IN (b, c))", + "formatted": "SELECT *\nFROM q UNPIVOT(values FOR quarter IN (b, c))" + }, + { + "dialect": "bigquery", + "sql": "SELECT MAKE_INTERVAL(100, 11, 1, 12, 30, 10)", + "formatted": "SELECT MAKE_INTERVAL(100, 11, 1, 12, 30, 10)" + }, + { + "dialect": "bigquery", + "sql": "SELECT y + 1 FROM x GROUP BY y + 1 ORDER BY 1", + "formatted": "SELECT y + 1\nFROM x\nGROUP BY\n y + 1\nORDER BY\n 1" + }, + { + "dialect": "bigquery", + "sql": "SELECT TIMESTAMP_SECONDS(2) AS t", + "formatted": "SELECT TIMESTAMP_SECONDS(2) AS t" + }, + { + "dialect": "bigquery", + "sql": "SELECT TIMESTAMP_MILLIS(2) AS t", + "formatted": "SELECT TIMESTAMP_MILLIS(2) AS t" + }, + { + "dialect": "bigquery", + "sql": "SELECT COUNT(x RESPECT NULLS)", + "formatted": "SELECT COUNT(x RESPECT NULLS)" + }, + { + "dialect": "bigquery", + "sql": "SELECT LAST_VALUE(x IGNORE NULLS) OVER y AS x", + "formatted": "SELECT LAST_VALUE(x IGNORE NULLS) OVER y AS x" + }, + { + "dialect": "bigquery", + "sql": "SELECT ARRAY((SELECT AS STRUCT 1 AS a, 2 AS b))", + "formatted": "SELECT\n ARRAY((\n SELECT AS STRUCT\n 1 AS a,\n 2 AS b\n ))" + }, + { + "dialect": "bigquery", + "sql": "SELECT ARRAY((SELECT AS STRUCT 1 AS a, 2 AS b) LIMIT 10)", + "formatted": "SELECT\n ARRAY((\n SELECT AS STRUCT\n 1 AS a,\n 2 AS b\n ) LIMIT 10)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM x WHERE x.y >= (SELECT MAX(a) FROM b-c) - 20", + "formatted": "SELECT *\nFROM x\nWHERE\n x.y >= (\n SELECT MAX(a)\n FROM b-c\n )\n - 20" + }, + { + "dialect": "bigquery", + "sql": "SELECT FORMAT_TIMESTAMP('%F %T', CURRENT_TIMESTAMP(), 'Europe/Berlin') AS ts", + "formatted": "SELECT FORMAT_TIMESTAMP('%F %T', CURRENT_TIMESTAMP(), 'Europe/Berlin') AS ts" + }, + { + "dialect": "bigquery", + "sql": "SELECT cars, apples FROM some_table PIVOT(SUM(total_counts) FOR products IN ('general.cars' AS cars, 'food.apples' AS apples))", + "formatted": "SELECT\n cars,\n apples\nFROM some_table PIVOT(SUM(total_counts) FOR products IN ('general.cars' AS cars, 'food.apples' AS apples))" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM test QUALIFY a IS DISTINCT FROM b WINDOW c AS (PARTITION BY d)", + "formatted": "SELECT *\nFROM test\nQUALIFY a IS DISTINCT FROM b\nWINDOW\n c AS (PARTITION BY d)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM (SELECT * FROM `t`) AS a UNPIVOT((c) FOR c_name IN (v1, v2))", + "formatted": "SELECT *\nFROM (\n SELECT *\n FROM `t`\n) AS a UNPIVOT((c) FOR c_name IN (v1, v2))" + }, + { + "dialect": "bigquery", + "sql": "SELECT ROW() OVER (y ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) FROM x WINDOW y AS (PARTITION BY CATEGORY)", + "formatted": "SELECT ROW() OVER (y ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING)\nFROM x\nWINDOW\n y AS (PARTITION BY CATEGORY)" + }, + { + "dialect": "bigquery", + "sql": "SELECT item, purchases, LAST_VALUE(item) OVER (item_window ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING) AS most_popular FROM Produce WINDOW item_window AS (ORDER BY purchases)", + "formatted": "SELECT\n item,\n purchases,\n LAST_VALUE(item) OVER (item_window ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING) AS most_popular\nFROM Produce\nWINDOW\n item_window AS (ORDER BY purchases)" + }, + { + "dialect": "bigquery", + "sql": "SELECT LAST_VALUE(a IGNORE NULLS) OVER y FROM x WINDOW y AS (PARTITION BY CATEGORY)", + "formatted": "SELECT LAST_VALUE(a IGNORE NULLS) OVER y\nFROM x\nWINDOW\n y AS (PARTITION BY CATEGORY)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM foo AS t0 FOR SYSTEM_TIME AS OF '2026-02-12T23:22:21.743416+00:00'", + "formatted": "SELECT *\nFROM foo AS t0 FOR SYSTEM_TIME AS OF '2026-02-12T23:22:21.743416+00:00'" + }, + { + "dialect": "bigquery", + "sql": "SELECT b\"\\x0a$'x'00\"", + "formatted": "SELECT b\"\\x0a$'x'00\"" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM t1, t2", + "formatted": "SELECT *\nFROM t1, t2" + }, + { + "dialect": "bigquery", + "sql": "SELECT r\"\\t\"", + "formatted": "SELECT r\"\\t\"" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM `proj.dataset.INFORMATION_SCHEMA.SOME_VIEW`", + "formatted": "SELECT *\nFROM `proj.dataset.INFORMATION_SCHEMA.SOME_VIEW`" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM region_or_dataset.INFORMATION_SCHEMA.TABLES", + "formatted": "SELECT *\nFROM region_or_dataset.INFORMATION_SCHEMA.TABLES" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM region_or_dataset.INFORMATION_SCHEMA.TABLES AS some_name", + "formatted": "SELECT *\nFROM region_or_dataset.INFORMATION_SCHEMA.TABLES AS some_name" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM proj.region_or_dataset.INFORMATION_SCHEMA.TABLES", + "formatted": "SELECT *\nFROM proj.region_or_dataset.INFORMATION_SCHEMA.TABLES" + }, + { + "dialect": "bigquery", + "sql": "SELECT ARRAY(SELECT AS STRUCT 1 a, 2 b)", + "formatted": "SELECT\n ARRAY(SELECT AS STRUCT\n 1 a,\n 2 b)", + "previously_formatted": "SELECT\n ARRAY(SELECT AS STRUCT\n 1 AS a,\n 2 AS b)" + }, + { + "dialect": "bigquery", + "sql": "select array_contains([1, 2, 3], 1)", + "formatted": "SELECT array_contains([1, 2, 3], 1)" + }, + { + "dialect": "bigquery", + "sql": "SELECT SPLIT(foo)", + "formatted": "SELECT SPLIT(foo)" + }, + { + "dialect": "bigquery", + "sql": "SELECT 1 AS hash", + "formatted": "SELECT 1 AS hash" + }, + { + "dialect": "bigquery", + "sql": "SELECT 1 AS at", + "formatted": "SELECT 1 AS at" + }, + { + "dialect": "bigquery", + "sql": "SELECT \"\"\"ends with \\\"word\\\"\"\"\"", + "formatted": "SELECT \"\"\"ends with \\\"word\\\"\"\"\"" + }, + { + "dialect": "bigquery", + "sql": "SELECT '''ends with \\'word\\''''", + "formatted": "SELECT '''ends with \\'word\\''''" + }, + { + "dialect": "bigquery", + "sql": "SELECT \"\"\"a\\\"b\"\"\"", + "formatted": "SELECT \"\"\"a\\\"b\"\"\"" + }, + { + "dialect": "bigquery", + "sql": "SELECT r\"\"\"ends with \\\"\"\"\"", + "formatted": "SELECT r\"\"\"ends with \\\"\"\"\"" + }, + { + "dialect": "bigquery", + "sql": "SELECT r\"\"\"a\\\"b\"\"\"", + "formatted": "SELECT r\"\"\"a\\\"b\"\"\"" + }, + { + "dialect": "bigquery", + "sql": "SELECT a overlaps", + "formatted": "SELECT a overlaps", + "previously_formatted": "SELECT\n a AS overlaps" + }, + { + "dialect": "bigquery", + "sql": "SELECT y + 1 z FROM x GROUP BY y + 1 ORDER BY z", + "formatted": "SELECT y + 1 z\nFROM x\nGROUP BY\n y + 1\nORDER BY\n z", + "previously_formatted": "SELECT\n y + 1 AS z\nFROM x\nGROUP BY\n y + 1\nORDER BY\n z" + }, + { + "dialect": "bigquery", + "sql": "SELECT y + 1 z FROM x GROUP BY y + 1", + "formatted": "SELECT y + 1 z\nFROM x\nGROUP BY\n y + 1", + "previously_formatted": "SELECT\n y + 1 AS z\nFROM x\nGROUP BY\n y + 1" + }, + { + "dialect": "bigquery", + "sql": "SELECT JSON '\"foo\"' AS json_data", + "formatted": "SELECT JSON '\"foo\"' AS json_data" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM (SELECT a, b, c FROM test) PIVOT(SUM(b) d, COUNT(*) e FOR c IN ('x', 'y'))", + "formatted": "SELECT *\nFROM (\n SELECT\n a,\n b,\n c\n FROM test\n) PIVOT(SUM(b) d, COUNT(*) e FOR c IN ('x', 'y'))", + "previously_formatted": "SELECT\n *\nFROM (\n SELECT\n a,\n b,\n c\n FROM test\n) PIVOT(SUM(b) AS d, COUNT(*) AS e FOR c IN ('x', 'y'))" + }, + { + "dialect": "bigquery", + "sql": "SELECT CAST(1 AS BYTEINT)", + "formatted": "SELECT CAST(1 AS BYTEINT)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM a-b c", + "formatted": "SELECT *\nFROM a-b c", + "previously_formatted": "SELECT\n *\nFROM a-b AS c" + }, + { + "dialect": "bigquery", + "sql": "SELECT PARSE_DATETIME('%a %b %e %I:%M:%S %Y', 'Thu Dec 25 07:30:00 2008')", + "formatted": "SELECT PARSE_DATETIME('%a %b %e %I:%M:%S %Y', 'Thu Dec 25 07:30:00 2008')" + }, + { + "dialect": "bigquery", + "sql": "SELECT TRANSLATE(MODEL, 'in', 't') FROM (SELECT 'input' AS MODEL)", + "formatted": "SELECT TRANSLATE(MODEL, 'in', 't')\nFROM (\n SELECT 'input' AS MODEL\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT GRANT FROM (SELECT 'input' AS GRANT)", + "formatted": "SELECT GRANT\nFROM (\n SELECT 'input' AS GRANT\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT CAST(1 AS NUMERIC(10, 2))", + "formatted": "SELECT CAST(1 AS NUMERIC(10, 2))" + }, + { + "dialect": "bigquery", + "sql": "SELECT CAST('1' AS STRING(10)) UNION ALL SELECT CAST('2' AS STRING(10))", + "formatted": "SELECT CAST('1' AS STRING(10))\nUNION ALL\nSELECT CAST('2' AS STRING(10))" + }, + { + "dialect": "bigquery", + "sql": "SELECT cola FROM (SELECT CAST('1' AS STRING(10)) AS cola UNION ALL SELECT CAST('2' AS STRING(10)) AS cola)", + "formatted": "SELECT cola\nFROM (\n SELECT CAST('1' AS STRING(10)) AS cola\n UNION ALL\n SELECT CAST('2' AS STRING(10)) AS cola\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM GAP_FILL(TABLE device_data, ts_column => 'time', bucket_width => INTERVAL '1' MINUTE, value_columns => [('signal', 'locf')]) ORDER BY time", + "formatted": "SELECT *\nFROM GAP_FILL(\n TABLE device_data,\n ts_column => 'time',\n bucket_width => INTERVAL '1' MINUTE,\n value_columns => [('signal', 'locf')]\n)\nORDER BY\n time" + }, + { + "dialect": "bigquery", + "sql": "SELECT a, b, c, d, e FROM GAP_FILL(TABLE foo, ts_column => 'b', partitioning_columns => ['a'], value_columns => [('c', 'bar'), ('d', 'baz'), ('e', 'bla')], bucket_width => INTERVAL '1' DAY)", + "formatted": "SELECT\n a,\n b,\n c,\n d,\n e\nFROM GAP_FILL(\n TABLE foo,\n ts_column => 'b',\n partitioning_columns => ['a'],\n value_columns => [\n ('c', 'bar'),\n ('d', 'baz'),\n ('e', 'bla')\n ],\n bucket_width => INTERVAL '1' DAY\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM GAP_FILL(TABLE device_data, ts_column => 'time', bucket_width => INTERVAL '1' MINUTE, value_columns => [('signal', 'linear')], ignore_null_values => FALSE) ORDER BY time", + "formatted": "SELECT *\nFROM GAP_FILL(\n TABLE device_data,\n ts_column => 'time',\n bucket_width => INTERVAL '1' MINUTE,\n value_columns => [('signal', 'linear')],\n ignore_null_values => FALSE\n)\nORDER BY\n time" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM GAP_FILL(TABLE device_data, ts_column => 'time', bucket_width => INTERVAL '1' MINUTE) ORDER BY time", + "formatted": "SELECT *\nFROM GAP_FILL(\n TABLE device_data,\n ts_column => 'time',\n bucket_width => INTERVAL '1' MINUTE\n)\nORDER BY\n time" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM GAP_FILL(TABLE device_data, ts_column => 'time', bucket_width => INTERVAL '1' MINUTE, value_columns => [('signal', 'null')], origin => CAST('2023-11-01 09:30:01' AS DATETIME)) ORDER BY time", + "formatted": "SELECT *\nFROM GAP_FILL(\n TABLE device_data,\n ts_column => 'time',\n bucket_width => INTERVAL '1' MINUTE,\n value_columns => [('signal', 'null')],\n origin => CAST('2023-11-01 09:30:01' AS DATETIME)\n)\nORDER BY\n time" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM GAP_FILL(TABLE device_data, ts_column => 'time', bucket_width => INTERVAL '1' MINUTE, value_columns => [('signal', 'locf')]) ORDER BY time", + "formatted": "SELECT *\nFROM GAP_FILL(\n TABLE device_data,\n ts_column => 'time',\n bucket_width => INTERVAL '1' MINUTE,\n value_columns => [('signal', 'locf')]\n)\nORDER BY\n time" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.PREDICT(MODEL mydataset.mymodel, (SELECT label, column1, column2 FROM mydataset.mytable))", + "formatted": "SELECT *\nFROM ML.PREDICT(MODEL mydataset.mymodel, (\n SELECT\n label,\n column1,\n column2\n FROM mydataset.mytable\n))" + }, + { + "dialect": "bigquery", + "sql": "SELECT label, predicted_label1, predicted_label AS predicted_label2 FROM ML.PREDICT(MODEL mydataset.mymodel2, (SELECT * EXCEPT (predicted_label), predicted_label AS predicted_label1 FROM ML.PREDICT(MODEL mydataset.mymodel1, TABLE mydataset.mytable)))", + "formatted": "SELECT\n label,\n predicted_label1,\n predicted_label AS predicted_label2\nFROM ML.PREDICT(\n MODEL mydataset.mymodel2,\n (\n SELECT\n * EXCEPT (predicted_label),\n predicted_label AS predicted_label1\n FROM ML.PREDICT(MODEL mydataset.mymodel1, TABLE mydataset.mytable)\n )\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.PREDICT(MODEL mydataset.mymodel, (SELECT custom_label, column1, column2 FROM mydataset.mytable), STRUCT(0.55 AS threshold))", + "formatted": "SELECT *\nFROM ML.PREDICT(\n MODEL mydataset.mymodel,\n (\n SELECT\n custom_label,\n column1,\n column2\n FROM mydataset.mytable\n ),\n STRUCT(0.55 AS threshold)\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT COSH(1.5)", + "formatted": "SELECT COSH(1.5)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.PREDICT(MODEL `my_project`.my_dataset.my_model, (SELECT * FROM input_data))", + "formatted": "SELECT *\nFROM ML.PREDICT(MODEL `my_project`.my_dataset.my_model, (\n SELECT *\n FROM input_data\n))" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.PREDICT(MODEL my_dataset.vision_model, (SELECT uri, ML.RESIZE_IMAGE(ML.DECODE_IMAGE(data), 480, 480, FALSE) AS input FROM my_dataset.object_table))", + "formatted": "SELECT *\nFROM ML.PREDICT(\n MODEL my_dataset.vision_model,\n (\n SELECT\n uri,\n ML.RESIZE_IMAGE(ML.DECODE_IMAGE(data), 480, 480, FALSE) AS input\n FROM my_dataset.object_table\n )\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.PREDICT(MODEL my_dataset.vision_model, (SELECT uri, ML.CONVERT_COLOR_SPACE(ML.RESIZE_IMAGE(ML.DECODE_IMAGE(data), 224, 280, TRUE), 'YIQ') AS input FROM my_dataset.object_table WHERE content_type = 'image/jpeg'))", + "formatted": "SELECT *\nFROM ML.PREDICT(\n MODEL my_dataset.vision_model,\n (\n SELECT\n uri,\n ML.CONVERT_COLOR_SPACE(ML.RESIZE_IMAGE(ML.DECODE_IMAGE(data), 224, 280, TRUE), 'YIQ') AS input\n FROM my_dataset.object_table\n WHERE content_type = 'image/jpeg'\n )\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.FEATURES_AT_TIME((SELECT 1), num_rows => 1)", + "formatted": "SELECT *\nFROM ML.FEATURES_AT_TIME((\n SELECT 1\n), num_rows => 1)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.FEATURES_AT_TIME(TABLE mydataset.feature_table, time => '2022-06-11 10:00:00+00', num_rows => 1, ignore_feature_nulls => TRUE)", + "formatted": "SELECT *\nFROM ML.FEATURES_AT_TIME(\n TABLE mydataset.feature_table,\n time => '2022-06-11 10:00:00+00',\n num_rows => 1,\n ignore_feature_nulls => TRUE\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM VECTOR_SEARCH(TABLE mydataset.base_table, 'column_to_search', TABLE mydataset.query_table, 'query_column_to_search', top_k => 2, distance_type => 'cosine', options => '{\"fraction_lists_to_search\":0.15}')", + "formatted": "SELECT *\nFROM VECTOR_SEARCH(\n TABLE mydataset.base_table,\n 'column_to_search',\n TABLE mydataset.query_table,\n 'query_column_to_search',\n top_k => 2,\n distance_type => 'cosine',\n options => '{\"fraction_lists_to_search\":0.15}'\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM VECTOR_SEARCH(TABLE mydataset.base_table, 'column_to_search', TABLE mydataset.query_table, query_column_to_search => 'query_column_to_search', top_k => 2, distance_type => 'cosine', options => '{\"fraction_lists_to_search\":0.15}')", + "formatted": "SELECT *\nFROM VECTOR_SEARCH(\n TABLE mydataset.base_table,\n 'column_to_search',\n TABLE mydataset.query_table,\n query_column_to_search => 'query_column_to_search',\n top_k => 2,\n distance_type => 'cosine',\n options => '{\"fraction_lists_to_search\":0.15}'\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM VECTOR_SEARCH((SELECT * FROM mydataset.base_table), 'column_to_search', (SELECT * FROM mydataset.query_table), 'query_column_to_search')", + "formatted": "SELECT *\nFROM VECTOR_SEARCH(\n (\n SELECT *\n FROM mydataset.base_table\n ),\n 'column_to_search',\n (\n SELECT *\n FROM mydataset.query_table\n ),\n 'query_column_to_search'\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM VECTOR_SEARCH(TABLE mydataset.base_table, 'column_to_search', TABLE mydataset.query_table)", + "formatted": "SELECT *\nFROM VECTOR_SEARCH(TABLE mydataset.base_table, 'column_to_search', TABLE mydataset.query_table)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.TRANSLATE(MODEL `mydataset.mytranslatemodel`, TABLE `mydataset.mybqtable`, STRUCT('translate_text' AS translate_mode, 'zh-CN' AS target_language_code))", + "formatted": "SELECT *\nFROM ML.TRANSLATE(\n MODEL `mydataset.mytranslatemodel`,\n TABLE `mydataset.mybqtable`,\n STRUCT(\n 'translate_text' AS translate_mode,\n 'zh-CN' AS target_language_code\n )\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.FORECAST(MODEL `mydataset.mymodel`, STRUCT(2 AS horizon))", + "formatted": "SELECT *\nFROM ML.FORECAST(MODEL `mydataset.mymodel`, STRUCT(2 AS horizon))" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.FORECAST(MODEL `mydataset.mymodel`, TABLE `mydataset.mybqtable`, STRUCT(2 AS horizon, 4 AS confidence_level))", + "formatted": "SELECT *\nFROM ML.FORECAST(\n MODEL `mydataset.mymodel`,\n TABLE `mydataset.mybqtable`,\n STRUCT(2 AS horizon, 4 AS confidence_level)\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.FORECAST(MODEL `mydataset.mymodel`, (SELECT * FROM mydataset.query_table), STRUCT())", + "formatted": "SELECT *\nFROM ML.FORECAST(MODEL `mydataset.mymodel`, (\n SELECT *\n FROM mydataset.query_table\n), STRUCT())" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM AI.FORECAST(TABLE citibike_trips, data_col => 'num_trips', timestamp_col => 'date', horizon => 30)", + "formatted": "SELECT *\nFROM AI.FORECAST(\n TABLE citibike_trips,\n data_col => 'num_trips',\n timestamp_col => 'date',\n horizon => 30\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM AI.FORECAST((SELECT * FROM citibike_trips), data_col => 'num_trips', timestamp_col => 'date', horizon => 30)", + "formatted": "SELECT *\nFROM AI.FORECAST(\n (\n SELECT *\n FROM citibike_trips\n ),\n data_col => 'num_trips',\n timestamp_col => 'date',\n horizon => 30\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.GENERATE_TEXT(MODEL `mydataset.gemini_model`, TABLE `mydataset.prompt_table`, STRUCT(0.15 AS temperature))", + "formatted": "SELECT *\nFROM ML.GENERATE_TEXT(\n MODEL `mydataset.gemini_model`,\n TABLE `mydataset.prompt_table`,\n STRUCT(0.15 AS temperature)\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM AI.GENERATE_TEXT(MODEL `mydataset.gemini_model`, TABLE `mydataset.prompt_table`, STRUCT(0.15 AS temperature))", + "formatted": "SELECT *\nFROM AI.GENERATE_TEXT(\n MODEL `mydataset.gemini_model`,\n TABLE `mydataset.prompt_table`,\n STRUCT(0.15 AS temperature)\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM AI.GENERATE_TABLE(MODEL `mydataset.gemini_model`, (SELECT 'Q' AS prompt), STRUCT('name STRING' AS output_schema))", + "formatted": "SELECT *\nFROM AI.GENERATE_TABLE(\n MODEL `mydataset.gemini_model`,\n (\n SELECT 'Q' AS prompt\n ),\n STRUCT('name STRING' AS output_schema)\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT AI.GENERATE_BOOL(MODEL `mydataset.gemini_model`, 'Is sky blue?')", + "formatted": "SELECT AI.GENERATE_BOOL(MODEL `mydataset.gemini_model`, 'Is sky blue?')" + }, + { + "dialect": "bigquery", + "sql": "SELECT AI.EMBED('hello')", + "formatted": "SELECT AI.EMBED('hello')" + }, + { + "dialect": "bigquery", + "sql": "SELECT AI.SIMILARITY('a', 'b')", + "formatted": "SELECT AI.SIMILARITY('a', 'b')" + }, + { + "dialect": "bigquery", + "sql": "SELECT AI.GENERATE('Write a haiku')", + "formatted": "SELECT AI.GENERATE('Write a haiku')" + }, + { + "dialect": "bigquery", + "sql": "SELECT JSON_OBJECT() AS json_data", + "formatted": "SELECT JSON_OBJECT() AS json_data" + }, + { + "dialect": "bigquery", + "sql": "SELECT JSON_OBJECT('foo', 10, 'bar', TRUE) AS json_data", + "formatted": "SELECT JSON_OBJECT('foo', 10, 'bar', TRUE) AS json_data" + }, + { + "dialect": "bigquery", + "sql": "SELECT JSON_OBJECT('foo', 10, 'bar', ['a', 'b']) AS json_data", + "formatted": "SELECT JSON_OBJECT('foo', 10, 'bar', ['a', 'b']) AS json_data" + }, + { + "dialect": "bigquery", + "sql": "SELECT JSON_OBJECT('a', 10, 'a', 'foo') AS json_data", + "formatted": "SELECT JSON_OBJECT('a', 10, 'a', 'foo') AS json_data" + }, + { + "dialect": "bigquery", + "sql": "SELECT JSON_OBJECT(['a', 'b'], [10, NULL]) AS json_data", + "formatted": "SELECT JSON_OBJECT(['a', 'b'], [10, NULL]) AS json_data" + }, + { + "dialect": "bigquery", + "sql": "SELECT JSON_OBJECT(['a', 'b'], [JSON '10', JSON '\"foo\"']) AS json_data", + "formatted": "SELECT JSON_OBJECT(['a', 'b'], [JSON '10', JSON '\"foo\"']) AS json_data" + }, + { + "dialect": "bigquery", + "sql": "SELECT JSON_OBJECT(['a', 'b'], [STRUCT(10 AS id, 'Red' AS color), STRUCT(20 AS id, 'Blue' AS color)]) AS json_data", + "formatted": "SELECT\n JSON_OBJECT(['a', 'b'], [\n STRUCT(10 AS id, 'Red' AS color),\n STRUCT(20 AS id, 'Blue' AS color)\n ]) AS json_data" + }, + { + "dialect": "bigquery", + "sql": "SELECT JSON_OBJECT(['a', 'b'], [TO_JSON(10), TO_JSON(['foo', 'bar'])]) AS json_data", + "formatted": "SELECT JSON_OBJECT(['a', 'b'], [TO_JSON(10), TO_JSON(['foo', 'bar'])]) AS json_data" + }, + { + "dialect": "bigquery", + "sql": "SELECT MOD((SELECT 1), 2)", + "formatted": "SELECT\n MOD((\n SELECT 1\n ), 2)" + }, + { + "dialect": "bigquery", + "sql": "SELECT STRUCT>([\"2023-01-17\"])", + "formatted": "SELECT STRUCT> ([\"2023-01-17\"])" + }, + { + "dialect": "bigquery", + "sql": "SELECT STRUCT((SELECT 'foo')).*", + "formatted": "SELECT\n STRUCT ((\n SELECT 'foo'\n )) .*" + }, + { + "dialect": "bigquery", + "sql": "SELECT RANGE(CAST('2022-12-01' AS DATE), CAST('2022-12-31' AS DATE))", + "formatted": "SELECT\n RANGE(\n CAST('2022-12-01' AS DATE),\n CAST('2022-12-31' AS DATE)\n )" + }, + { + "dialect": "bigquery", + "sql": "SELECT RANGE(NULL, CAST('2022-12-31' AS DATE))", + "formatted": "SELECT RANGE(NULL, CAST('2022-12-31' AS DATE))" + }, + { + "dialect": "bigquery", + "sql": "SELECT RANGE(CAST('2022-10-01 14:53:27' AS DATETIME), CAST('2022-10-01 16:00:00' AS DATETIME))", + "formatted": "SELECT\n RANGE(\n CAST('2022-10-01 14:53:27' AS DATETIME),\n CAST('2022-10-01 16:00:00' AS DATETIME)\n )" + }, + { + "dialect": "bigquery", + "sql": "SELECT RANGE(CAST('2022-10-01 14:53:27 America/Los_Angeles' AS TIMESTAMP), CAST('2022-10-01 16:00:00 America/Los_Angeles' AS TIMESTAMP))", + "formatted": "SELECT\n RANGE(\n CAST('2022-10-01 14:53:27 America/Los_Angeles' AS TIMESTAMP),\n CAST('2022-10-01 16:00:00 America/Los_Angeles' AS TIMESTAMP)\n )" + }, + { + "dialect": "bigquery", + "sql": "SELECT CAST('2026-03-24' AS STRING FORMAT ('YYYY'))", + "formatted": "SELECT CAST('2026-03-24' AS STRING FORMAT('YYYY'))" + }, + { + "dialect": "bigquery", + "sql": "SELECT CAST(date AS STRING FORMAT ('YYYY')) FROM (SELECT DATE('2026-03-24') AS date)", + "formatted": "SELECT CAST(date AS STRING FORMAT('YYYY'))\nFROM (\n SELECT DATE('2026-03-24') AS date\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT CAST(date AS STRING FORMAT ('YYYY-MM-DD'))", + "formatted": "SELECT CAST(date AS STRING FORMAT('YYYY-MM-DD'))" + }, + { + "dialect": "bigquery", + "sql": "SELECT CAST(timestamp AS STRING FORMAT ('YYYY-MM-DD') AT TIME ZONE 'UTC')", + "formatted": "SELECT CAST(timestamp AS STRING FORMAT('YYYY-MM-DD') AT TIME ZONE 'UTC')" + }, + { + "dialect": "bigquery", + "sql": "SELECT CAST(date AS TIMESTAMP FORMAT ('YYYY-MM-DD HH24:MI:SS'))", + "formatted": "SELECT CAST(date AS TIMESTAMP FORMAT('YYYY-MM-DD HH24:MI:SS'))" + }, + { + "dialect": "bigquery", + "sql": "SELECT a, GROUP_CONCAT(b) FROM table GROUP BY a", + "formatted": "SELECT\n a,\n GROUP_CONCAT(b)\nFROM table\nGROUP BY\n a" + }, + { + "dialect": "bigquery", + "sql": "SELECT 1 AS foo INNER UNION ALL SELECT 3 AS foo, 4 AS bar", + "formatted": "SELECT 1 AS foo\nINNER UNION ALL\nSELECT\n 3 AS foo,\n 4 AS bar" + }, + { + "dialect": "bigquery", + "sql": "SELECT 1 AS x UNION ALL CORRESPONDING SELECT 2 AS x", + "formatted": "SELECT 1 AS x\nUNION ALL CORRESPONDING\nSELECT 2 AS x" + }, + { + "dialect": "bigquery", + "sql": "SELECT 1 AS x UNION ALL CORRESPONDING BY (foo, bar) SELECT 2 AS x", + "formatted": "SELECT 1 AS x\nUNION ALL CORRESPONDING BY (foo, bar)\nSELECT 2 AS x" + }, + { + "dialect": "bigquery", + "sql": "SELECT 1 AS x LEFT UNION ALL CORRESPONDING SELECT 2 AS x", + "formatted": "SELECT 1 AS x\nLEFT UNION ALL CORRESPONDING\nSELECT 2 AS x" + }, + { + "dialect": "bigquery", + "sql": "SELECT 1 AS x UNION ALL STRICT CORRESPONDING SELECT 2 AS x", + "formatted": "SELECT 1 AS x\nUNION ALL STRICT CORRESPONDING\nSELECT 2 AS x" + }, + { + "dialect": "bigquery", + "sql": "SELECT 1 AS x UNION ALL STRICT CORRESPONDING BY (foo, bar) SELECT 2 AS x", + "formatted": "SELECT 1 AS x\nUNION ALL STRICT CORRESPONDING BY (foo, bar)\nSELECT 2 AS x" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM UNNEST(x) WITH OFFSET EXCEPT DISTINCT SELECT * FROM UNNEST(y) WITH OFFSET", + "formatted": "SELECT *\nFROM UNNEST(x) WITH OFFSET\nEXCEPT DISTINCT\nSELECT *\nFROM UNNEST(y) WITH OFFSET" + }, + { + "dialect": "bigquery", + "sql": "SELECT col FROM t WHERE _PARTITIONTIME BETWEEN a AND b", + "formatted": "SELECT col\nFROM t\nWHERE _PARTITIONTIME BETWEEN a AND b" + }, + { + "dialect": "bigquery", + "sql": "SELECT _DBT_MAX_PARTITION FROM t", + "formatted": "SELECT _DBT_MAX_PARTITION\nFROM t" + }, + { + "dialect": "bigquery", + "sql": "WITH cte(c) AS (SELECT * FROM t) SELECT * FROM cte", + "formatted": "WITH cte(c) AS (\n SELECT *\n FROM t\n)\n\nSELECT *\nFROM cte" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM t AS t(c1, c2)", + "formatted": "SELECT *\nFROM t AS t(c1, c2)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM t AS t(c1, c2)", + "formatted": "SELECT *\nFROM t AS t(c1, c2)" + }, + { + "dialect": "bigquery", + "sql": "WITH foo AS (SELECT [1, 2, 3] AS array_col) SELECT array_col[offset] FROM foo CROSS JOIN UNNEST(array_col) WITH OFFSET AS offset", + "formatted": "WITH foo AS (\n SELECT [1, 2, 3] AS array_col\n)\n\nSELECT array_col[offset]\nFROM foo\nCROSS JOIN UNNEST(array_col) WITH OFFSET AS offset" + }, + { + "dialect": "bigquery", + "sql": "SELECT `db.t`.`c` FROM `db.t`", + "formatted": "SELECT `db.t`.`c`\nFROM `db.t`" + }, + { + "dialect": "bigquery", + "sql": "SELECT `p.d.t`.`c`.`f` FROM `p.d.t`", + "formatted": "SELECT `p.d.t`.`c`.`f`\nFROM `p.d.t`" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.TRANSLATE(MODEL `mydataset.mymodel`, (SELECT comment AS text_content FROM mydataset.mytable), STRUCT('translate_text' AS translate_mode, 'en' AS target_language_code))", + "formatted": "SELECT *\nFROM ML.TRANSLATE(\n MODEL `mydataset.mymodel`,\n (\n SELECT comment AS text_content\n FROM mydataset.mytable\n ),\n STRUCT(\n 'translate_text' AS translate_mode,\n 'en' AS target_language_code\n )\n)" + }, + { + "dialect": "bigquery", + "sql": "WITH t AS (SELECT '{\"x-y\": \"z\"}' AS c) SELECT JSON_EXTRACT(c, '$.x-y') FROM t", + "formatted": "WITH t AS (\n SELECT '{\"x-y\": \"z\"}' AS c\n)\n\nSELECT JSON_EXTRACT(c, '$.x-y')\nFROM t" + }, + { + "dialect": "databricks", + "sql": "SELECT CAST('a' AS STRING COLLATE UTF8_BINARY)", + "formatted": "SELECT CAST('a' AS STRING COLLATE UTF8_BINARY)" + }, + { + "dialect": "databricks", + "sql": "SELECT COSH(1.5)", + "formatted": "SELECT COSH(1.5)" + }, + { + "dialect": "databricks", + "sql": "SELECT BITMAP_BIT_POSITION(10)", + "formatted": "SELECT BITMAP_BIT_POSITION(10)" + }, + { + "dialect": "databricks", + "sql": "SELECT BITMAP_BUCKET_NUMBER(32769)", + "formatted": "SELECT BITMAP_BUCKET_NUMBER(32769)" + }, + { + "dialect": "databricks", + "sql": "SELECT BITMAP_CONSTRUCT_AGG(value)", + "formatted": "SELECT BITMAP_CONSTRUCT_AGG(value)" + }, + { + "dialect": "databricks", + "sql": "SELECT EXP(1)", + "formatted": "SELECT EXP(1)" + }, + { + "dialect": "databricks", + "sql": "SELECT MODE(category)", + "formatted": "SELECT MODE(category)" + }, + { + "dialect": "databricks", + "sql": "SELECT MODE(price, TRUE) AS deterministic_mode FROM products", + "formatted": "SELECT MODE(price, TRUE) AS deterministic_mode\nFROM products" + }, + { + "dialect": "databricks", + "sql": "SELECT CAST(NULL AS VOID)", + "formatted": "SELECT CAST(NULL AS VOID)" + }, + { + "dialect": "databricks", + "sql": "SELECT void FROM t", + "formatted": "SELECT void\nFROM t" + }, + { + "dialect": "databricks", + "sql": "SELECT * FROM stream", + "formatted": "SELECT *\nFROM stream" + }, + { + "dialect": "databricks", + "sql": "SELECT * FROM STREAM t", + "formatted": "SELECT *\nFROM STREAM t" + }, + { + "dialect": "databricks", + "sql": "SELECT t.current_time FROM t", + "formatted": "SELECT t.current_time\nFROM t" + }, + { + "dialect": "databricks", + "sql": "SELECT col1 FROM t CLUSTER BY col1", + "formatted": "SELECT col1\nFROM t\nCLUSTER BY\n col1" + }, + { + "dialect": "databricks", + "sql": "SELECT col1, col2 FROM t CLUSTER BY col1, col2", + "formatted": "SELECT\n col1,\n col2\nFROM t\nCLUSTER BY\n col1,\n col2" + }, + { + "dialect": "databricks", + "sql": "SELECT CAST('11 23:4:0' AS INTERVAL DAY TO HOUR)", + "formatted": "SELECT CAST('11 23:4:0' AS INTERVAL DAY TO HOUR)" + }, + { + "dialect": "databricks", + "sql": "SELECT CAST('11 23:4:0' AS INTERVAL DAY TO MINUTE)", + "formatted": "SELECT CAST('11 23:4:0' AS INTERVAL DAY TO MINUTE)" + }, + { + "dialect": "databricks", + "sql": "SELECT CAST('11 23:4:0' AS INTERVAL DAY TO SECOND)", + "formatted": "SELECT CAST('11 23:4:0' AS INTERVAL DAY TO SECOND)" + }, + { + "dialect": "databricks", + "sql": "SELECT CAST('23:00:00' AS INTERVAL HOUR TO MINUTE)", + "formatted": "SELECT CAST('23:00:00' AS INTERVAL HOUR TO MINUTE)" + }, + { + "dialect": "databricks", + "sql": "SELECT CAST('23:00:00' AS INTERVAL HOUR TO SECOND)", + "formatted": "SELECT CAST('23:00:00' AS INTERVAL HOUR TO SECOND)" + }, + { + "dialect": "databricks", + "sql": "SELECT CAST('23:00:00' AS INTERVAL MINUTE TO SECOND)", + "formatted": "SELECT CAST('23:00:00' AS INTERVAL MINUTE TO SECOND)" + }, + { + "dialect": "databricks", + "sql": "SELECT ${x} FROM ${y} WHERE ${z} > 1", + "formatted": "SELECT ${x}\nFROM ${y}\nWHERE ${z} > 1" + }, + { + "dialect": "databricks", + "sql": "SELECT PARSE_JSON('{}')", + "formatted": "SELECT PARSE_JSON('{}')" + }, + { + "dialect": "databricks", + "sql": "SELECT RANDSTR(123)", + "formatted": "SELECT RANDSTR(123)" + }, + { + "dialect": "databricks", + "sql": "SELECT RANDSTR(123, 456)", + "formatted": "SELECT RANDSTR(123, 456)" + }, + { + "dialect": "databricks", + "sql": "SELECT * FROM sales UNPIVOT INCLUDE NULLS (sales FOR quarter IN (q1 AS `Jan-Mar`))", + "formatted": "SELECT *\nFROM sales UNPIVOT INCLUDE NULLS (sales FOR quarter IN (q1 AS `Jan-Mar`))" + }, + { + "dialect": "databricks", + "sql": "SELECT * FROM sales UNPIVOT EXCLUDE NULLS (sales FOR quarter IN (q1 AS `Jan-Mar`))", + "formatted": "SELECT *\nFROM sales UNPIVOT EXCLUDE NULLS (sales FOR quarter IN (q1 AS `Jan-Mar`))" + }, + { + "dialect": "databricks", + "sql": "SELECT * FROM t1, t2", + "formatted": "SELECT *\nFROM t1, t2" + }, + { + "dialect": "databricks", + "sql": "SELECT TIMESTAMP '2025-04-29 18.47.18'::DATE", + "formatted": "SELECT TIMESTAMP '2025-04-29 18.47.18'::DATE" + }, + { + "dialect": "databricks", + "sql": "SELECT DATE_FORMAT(CAST(FROM_UTC_TIMESTAMP(foo, 'America/Los_Angeles') AS TIMESTAMP), 'yyyy-MM-dd HH:mm:ss') AS foo FROM t", + "formatted": "SELECT\n DATE_FORMAT(\n CAST(FROM_UTC_TIMESTAMP(foo, 'America/Los_Angeles') AS TIMESTAMP),\n 'yyyy-MM-dd HH:mm:ss'\n ) AS foo\nFROM t" + }, + { + "dialect": "databricks", + "sql": "SELECT r\"\\\\foo.bar\\\"", + "formatted": "SELECT r\"\\\\foo.bar\\\"" + }, + { + "dialect": "databricks", + "sql": "SELECT SUBSTRING_INDEX(str, delim, count)", + "formatted": "SELECT SUBSTRING_INDEX(str, delim, count)" + }, + { + "dialect": "databricks", + "sql": "SELECT SUBSTR('Spark' FROM 5 FOR 1)", + "formatted": "SELECT SUBSTR('Spark' FROM 5 FOR 1)" + }, + { + "dialect": "databricks", + "sql": "SELECT SUBSTR('Spark SQL', 5)", + "formatted": "SELECT SUBSTR('Spark SQL', 5)" + }, + { + "dialect": "databricks", + "sql": "SELECT SUBSTR(ENCODE('Spark SQL', 'utf-8'), 5)", + "formatted": "SELECT SUBSTR(ENCODE('Spark SQL', 'utf-8'), 5)" + }, + { + "dialect": "databricks", + "sql": "SELECT test, LISTAGG(email, '') AS Email FROM organizations GROUP BY test", + "formatted": "SELECT\n test,\n LISTAGG(email, '') AS Email\nFROM organizations\nGROUP BY\n test" + }, + { + "dialect": "databricks", + "sql": "WITH t AS (VALUES ('foo_val') AS t(foo1)) SELECT foo1 FROM t", + "formatted": "WITH t AS (\n VALUES ('foo_val') AS t(foo1)\n)\n\nSELECT foo1\nFROM t" + }, + { + "dialect": "databricks", + "sql": "SELECT ELT(2, 'foo', 'bar', 'baz') AS Result", + "formatted": "SELECT ELT(2, 'foo', 'bar', 'baz') AS Result" + }, + { + "dialect": "databricks", + "sql": "SELECT MAKE_INTERVAL(100, 11, 12, 13, 14, 14, 15)", + "formatted": "SELECT MAKE_INTERVAL(100, 11, 12, 13, 14, 14, 15)" + }, + { + "dialect": "databricks", + "sql": "SELECT name, GROUPING_ID() FROM customer GROUP BY ROLLUP (name)", + "formatted": "SELECT\n name,\n GROUPING_ID()\nFROM customer\nGROUP BY\n ROLLUP (\n name\n )" + }, + { + "dialect": "databricks", + "sql": "SELECT CURDATE()", + "formatted": "SELECT CURDATE()" + }, + { + "dialect": "databricks", + "sql": "SELECT c1:price, c1:price.foo, c1:price.bar[1]", + "formatted": "SELECT\n c1:price,\n c1:price.foo,\n c1:price.bar[1]" + }, + { + "dialect": "databricks", + "sql": "SELECT TRY_CAST(c1:price AS ARRAY)", + "formatted": "SELECT TRY_CAST(c1:price AS ARRAY)" + }, + { + "dialect": "databricks", + "sql": "SELECT TRY_CAST(c1:[\"foo bar\"][\"baz qux\"] AS ARRAY)", + "formatted": "SELECT TRY_CAST(c1:[\"foo bar\"][\"baz qux\"] AS ARRAY)" + }, + { + "dialect": "databricks", + "sql": "SELECT c1:item[1].price FROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)", + "formatted": "SELECT c1:item[1].price\nFROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)" + }, + { + "dialect": "databricks", + "sql": "SELECT c1:item[*].price FROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)", + "formatted": "SELECT c1:item[*].price\nFROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)" + }, + { + "dialect": "databricks", + "sql": "SELECT FROM_JSON(c1:item[*].price, 'ARRAY')[0] FROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)", + "formatted": "SELECT FROM_JSON(c1:item[*].price, 'ARRAY')[0]\nFROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)" + }, + { + "dialect": "databricks", + "sql": "SELECT INLINE(FROM_JSON(c1:item[*], 'ARRAY>')) FROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)", + "formatted": "SELECT INLINE(FROM_JSON(c1:item[*], 'ARRAY>'))\nFROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)" + }, + { + "dialect": "databricks", + "sql": "SELECT c1:['price'] FROM VALUES ('{ \"price\": 5 }') AS T(c1)", + "formatted": "SELECT c1:['price']\nFROM VALUES ('{ \"price\": 5 }') AS T(c1)" + }, + { + "dialect": "databricks", + "sql": "SELECT GET_JSON_OBJECT(c1, '$.price') FROM VALUES ('{ \"price\": 5 }') AS T(c1)", + "formatted": "SELECT GET_JSON_OBJECT(c1, '$.price')\nFROM VALUES ('{ \"price\": 5 }') AS T(c1)" + }, + { + "dialect": "databricks", + "sql": "SELECT GET_JSON_OBJECT(col, path_col)", + "formatted": "SELECT GET_JSON_OBJECT(col, path_col)" + }, + { + "dialect": "databricks", + "sql": "SELECT GET_JSON_OBJECT(col, CONCAT('$.', field_name))", + "formatted": "SELECT GET_JSON_OBJECT(col, CONCAT('$.', field_name))" + }, + { + "dialect": "databricks", + "sql": "SELECT GET_JSON_OBJECT(GET_JSON_OBJECT(col, '$[0]'), '$.a')", + "formatted": "SELECT GET_JSON_OBJECT(GET_JSON_OBJECT(col, '$[0]'), '$.a')" + }, + { + "dialect": "databricks", + "sql": "SELECT raw:`zip code`, raw:`fb:testid`, raw:store['bicycle'], raw:store[\"zip code\"]", + "formatted": "SELECT\n raw:`zip code`,\n raw:`fb:testid`,\n raw:store['bicycle'],\n raw:store[\"zip code\"]" + }, + { + "dialect": "databricks", + "sql": "SELECT DATE_ADD(MONTH, 1, '2020-01-01')", + "formatted": "SELECT DATE_ADD(MONTH, 1, '2020-01-01')" + }, + { + "dialect": "databricks", + "sql": "SELECT DATEADD(e, 24) FROM t", + "formatted": "SELECT DATEADD(e, 24)\nFROM t" + }, + { + "dialect": "databricks", + "sql": "SELECT '20'?::INTEGER", + "formatted": "SELECT '20'?::INTEGER" + }, + { + "dialect": "databricks", + "sql": "SELECT OVERLAY('Spark SQL', 'ANSI ', 7, 0)", + "formatted": "SELECT OVERLAY('Spark SQL', 'ANSI ', 7, 0)" + }, + { + "dialect": "databricks", + "sql": "SELECT OVERLAY('Spark SQL' PLACING 'CORE' FROM 7)", + "formatted": "SELECT OVERLAY('Spark SQL' PLACING 'CORE' FROM 7)" + }, + { + "dialect": "databricks", + "sql": "SELECT OVERLAY(ENCODE('Spark SQL', 'utf-8') PLACING ENCODE('_', 'utf-8') FROM 6)", + "formatted": "SELECT OVERLAY(ENCODE('Spark SQL', 'utf-8') PLACING ENCODE('_', 'utf-8') FROM 6)" + }, + { + "dialect": "databricks", + "sql": "SELECT OVERLAY('Spark SQL' PLACING 'ANSI ' FROM 7 FOR 0)", + "formatted": "SELECT OVERLAY('Spark SQL' PLACING 'ANSI ' FROM 7 FOR 0)" + }, + { + "dialect": "databricks", + "sql": "WITH t AS (SELECT '{\"x-y\": \"z\"}' AS c) SELECT get_json_object(c, '$.x-y') FROM t", + "formatted": "WITH t AS (\n SELECT '{\"x-y\": \"z\"}' AS c\n)\n\nSELECT get_json_object(c, '$.x-y')\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT ([1,2,3])[:-:-1]", + "formatted": "SELECT\n (\n [1, 2, 3]\n )[:-:-1]" + }, + { + "dialect": "duckdb", + "sql": "SELECT INTERVAL '1 hour'::VARCHAR", + "formatted": "SELECT INTERVAL '1 hour'::VARCHAR" + }, + { + "dialect": "duckdb", + "sql": "SELECT str[0:1]", + "formatted": null + }, + { + "dialect": "duckdb", + "sql": "SELECT COSH(1.5)", + "formatted": "SELECT COSH(1.5)" + }, + { + "dialect": "duckdb", + "sql": "SELECT MODE(category)", + "formatted": "SELECT MODE(category)" + }, + { + "dialect": "duckdb", + "sql": "SELECT e'\\n'", + "formatted": "SELECT e'\\n'" + }, + { + "dialect": "duckdb", + "sql": "SELECT e'\\t'", + "formatted": "SELECT e'\\t'" + }, + { + "dialect": "duckdb", + "sql": "SELECT e'update table_name set a = \\'foo\\' where 1 = 0' AS x FROM tab", + "formatted": "SELECT e'update table_name set a = \\'foo\\' where 1 = 0' AS x\nFROM tab" + }, + { + "dialect": "duckdb", + "sql": "SELECT GET_BIT(CAST('0110010' AS BIT), 2)", + "formatted": "SELECT GET_BIT(CAST('0110010' AS BIT), 2)" + }, + { + "dialect": "duckdb", + "sql": "SELECT 1 WHERE x > $1", + "formatted": "SELECT 1\nWHERE x > $1" + }, + { + "dialect": "duckdb", + "sql": "SELECT 1 WHERE x > $name", + "formatted": "SELECT 1\nWHERE x > $name" + }, + { + "dialect": "duckdb", + "sql": "SELECT 1 AS a$b", + "formatted": "SELECT 1 AS a$b" + }, + { + "dialect": "duckdb", + "sql": "SELECT '{\"x\": 1}' -> c FROM t", + "formatted": "SELECT '{\"x\": 1}' -> c\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT EXP(1)", + "formatted": "SELECT EXP(1)" + }, + { + "dialect": "duckdb", + "sql": "SELECT '{\"duck\": [1, 2, 3]}' -> '$.duck[#-1]'", + "formatted": "SELECT '{\"duck\": [1, 2, 3]}' -> '$.duck[#-1]'" + }, + { + "dialect": "duckdb", + "sql": "SELECT tbl.x*1e4+tbl.y FROM tbl", + "formatted": "SELECT tbl.x * 1e4 + tbl.y\nFROM tbl" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_TRANSFORM([5, NULL, 6], (x, y) -> COALESCE(x, y, 0) + 1)", + "formatted": "SELECT LIST_TRANSFORM([5, NULL, 6], (x, y) -> COALESCE(x, y, 0) + 1)" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_TRANSFORM([5, NULL, 6], LAMBDA x, y : COALESCE(x, y, 0) + 1)", + "formatted": null + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_TRANSFORM(LIST_FILTER([0, 1, 2, 3, 4, 5], LAMBDA x : x % 2 = 0), LAMBDA y : y * y)", + "formatted": null + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_TRANSFORM([5, NULL, 6], LAMBDA x : COALESCE(x, 0) + 1)", + "formatted": null + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_TRANSFORM(nbr, LAMBDA x : x + 1) FROM article AS a", + "formatted": null + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM my_ducklake.demo AT (VERSION => 2)", + "formatted": "SELECT *\nFROM my_ducklake.demo AT (VERSION => 2)" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM t1 AS a AT (VERSION => 3) JOIN t2 AS b AT (TIMESTAMP => NOW() - INTERVAL '1' WEEK) ON a.id = b.id", + "formatted": "SELECT *\nFROM t1 AS a AT (VERSION => 3)\nJOIN t2 AS b AT (TIMESTAMP => NOW() - INTERVAL '1' WEEK)\n ON a.id = b.id" + }, + { + "dialect": "duckdb", + "sql": "SELECT TO_BINARY('test')", + "formatted": "SELECT TO_BINARY('test')" + }, + { + "dialect": "duckdb", + "sql": "SELECT UUIDV7()", + "formatted": "SELECT UUIDV7()" + }, + { + "dialect": "duckdb", + "sql": "SELECT TRY(LOG(0))", + "formatted": "SELECT TRY(LOG(0))" + }, + { + "dialect": "duckdb", + "sql": "SELECT 1 AS x UNION ALL BY NAME SELECT 2 AS x", + "formatted": "SELECT 1 AS x\nUNION ALL BY NAME\nSELECT 2 AS x" + }, + { + "dialect": "duckdb", + "sql": "SELECT SUM(x) FILTER (x = 1)", + "formatted": "SELECT SUM(x) FILTER(x = 1)" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM GLOB(x)", + "formatted": "SELECT *\nFROM GLOB(x)" + }, + { + "dialect": "duckdb", + "sql": "SELECT MAP(['key1', 'key2', 'key3'], [10, 20, 30])", + "formatted": "SELECT\n MAP(\n ['key1', 'key2', 'key3'],\n [10, 20, 30]\n )" + }, + { + "dialect": "duckdb", + "sql": "SELECT MAP {'x': 1}", + "formatted": "SELECT MAP {'x': 1}" + }, + { + "dialect": "duckdb", + "sql": "SELECT (MAP {'x': 1})['x']", + "formatted": "SELECT\n (\n MAP {'x': 1}\n )['x']" + }, + { + "dialect": "duckdb", + "sql": "SELECT df1.*, df2.* FROM df1 POSITIONAL JOIN df2", + "formatted": "SELECT\n df1.*,\n df2.*\nFROM df1\nPOSITIONAL JOIN df2" + }, + { + "dialect": "duckdb", + "sql": "SELECT EPOCH_MS(10) AS t", + "formatted": "SELECT EPOCH_MS(10) AS t" + }, + { + "dialect": "duckdb", + "sql": "SELECT MAKE_TIMESTAMP(10) AS t", + "formatted": "SELECT MAKE_TIMESTAMP(10) AS t" + }, + { + "dialect": "duckdb", + "sql": "SELECT TO_TIMESTAMP(10) AS t", + "formatted": "SELECT TO_TIMESTAMP(10) AS t" + }, + { + "dialect": "duckdb", + "sql": "SELECT UNNEST(col, recursive := TRUE) FROM t", + "formatted": "SELECT UNNEST(col, recursive := TRUE)\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM foo ASOF LEFT JOIN bar ON a = b", + "formatted": "SELECT *\nFROM foo\nASOF LEFT JOIN bar\n ON a = b" + }, + { + "dialect": "duckdb", + "sql": "SELECT {'a': 1} AS x", + "formatted": "SELECT {'a': 1} AS x" + }, + { + "dialect": "duckdb", + "sql": "SELECT {'a': {'b': {'c': 1}}, 'd': {'e': 2}} AS x", + "formatted": "SELECT {'a': {'b': {'c': 1}}, 'd': {'e': 2}} AS x" + }, + { + "dialect": "duckdb", + "sql": "SELECT {'x': 1, 'y': 2, 'z': 3}", + "formatted": "SELECT {'x': 1, 'y': 2, 'z': 3}" + }, + { + "dialect": "duckdb", + "sql": "SELECT {'key1': 'string', 'key2': 1, 'key3': 12.345}", + "formatted": "SELECT {'key1': 'string', 'key2': 1, 'key3': 12.345}" + }, + { + "dialect": "duckdb", + "sql": "SELECT ROW(x, x + 1, y) FROM (SELECT 1 AS x, 'a' AS y)", + "formatted": "SELECT ROW(x, x + 1, y)\nFROM (\n SELECT\n 1 AS x,\n 'a' AS y\n)" + }, + { + "dialect": "duckdb", + "sql": "SELECT (x, x + 1, y) FROM (SELECT 1 AS x, 'a' AS y)", + "formatted": "SELECT (x, x + 1, y)\nFROM (\n SELECT\n 1 AS x,\n 'a' AS y\n)" + }, + { + "dialect": "duckdb", + "sql": "SELECT a.x FROM (SELECT {'x': 1, 'y': 2, 'z': 3} AS a)", + "formatted": "SELECT a.x\nFROM (\n SELECT {'x': 1, 'y': 2, 'z': 3} AS a\n)" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM t1 WHERE NOT EXISTS(FROM t2 WHERE t2.id = t1.id)", + "formatted": "SELECT *\nFROM t1\nWHERE\n NOT EXISTS(\n FROM t2\n WHERE t2.id = t1.id\n )" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM foo WHERE bar > $baz AND bla = $bob", + "formatted": "SELECT *\nFROM foo\nWHERE\n bar > $baz AND bla = $bob" + }, + { + "dialect": "duckdb", + "sql": "SELECT STAR(tbl, exclude := [foo])", + "formatted": "SELECT STAR(tbl, exclude := [foo])" + }, + { + "dialect": "duckdb", + "sql": "SELECT species, island, COUNT(*) FROM t GROUP BY GROUPING SETS (species), GROUPING SETS (island)", + "formatted": "SELECT\n species,\n island,\n COUNT(*)\nFROM t\nGROUP BY\n GROUPING SETS (\n species\n ),\n GROUPING SETS (\n island\n )" + }, + { + "dialect": "duckdb", + "sql": "SELECT species, island, COUNT(*) FROM t GROUP BY CUBE (species), CUBE (island)", + "formatted": "SELECT\n species,\n island,\n COUNT(*)\nFROM t\nGROUP BY\n CUBE (\n species\n ),\n CUBE (\n island\n )" + }, + { + "dialect": "duckdb", + "sql": "SELECT species, island, COUNT(*) FROM t GROUP BY ROLLUP (species), ROLLUP (island)", + "formatted": "SELECT\n species,\n island,\n COUNT(*)\nFROM t\nGROUP BY\n ROLLUP (\n species\n ),\n ROLLUP (\n island\n )" + }, + { + "dialect": "duckdb", + "sql": "SELECT '{ \"family\": \"anatidae\", \"species\": [ \"duck\", \"goose\", \"swan\", null ] }' ->> ['$.family', '$.species']", + "formatted": "SELECT '{ \"family\": \"anatidae\", \"species\": [ \"duck\", \"goose\", \"swan\", null ] }' ->> ['$.family', '$.species']" + }, + { + "dialect": "duckdb", + "sql": "SELECT $🦆$foo$🦆$", + "formatted": "SELECT $🦆$foo$🦆$" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM t PIVOT(FIRST(t) AS t, FOR quarter IN ('Q1', 'Q2'))", + "formatted": "SELECT *\nFROM t PIVOT(FIRST(t) AS t, FOR quarter IN ('Q1', 'Q2'))" + }, + { + "dialect": "duckdb", + "sql": "SELECT JSON_EXTRACT_STRING('{ \"family\": \"anatidae\", \"species\": [ \"duck\", \"goose\", \"swan\", null ] }', ['$.family', '$.species'])", + "formatted": "SELECT JSON_EXTRACT_STRING('{ \"family\": \"anatidae\", \"species\": [ \"duck\", \"goose\", \"swan\", null ] }', ['$.family', '$.species'])" + }, + { + "dialect": "duckdb", + "sql": "SELECT col FROM t WHERE JSON_EXTRACT_STRING(col, '$.id') NOT IN ('b')", + "formatted": "SELECT col\nFROM t\nWHERE JSON_EXTRACT_STRING(col, '$.id') NOT IN ('b')" + }, + { + "dialect": "duckdb", + "sql": "SELECT a, LOGICAL_OR(b) FROM foo GROUP BY a", + "formatted": "SELECT\n a,\n LOGICAL_OR(b)\nFROM foo\nGROUP BY\n a" + }, + { + "dialect": "duckdb", + "sql": "SELECT JSON_EXTRACT_STRING(c, '$.k1') = 'v1'", + "formatted": "SELECT JSON_EXTRACT_STRING(c, '$.k1') = 'v1'" + }, + { + "dialect": "duckdb", + "sql": "SELECT JSON_EXTRACT(c, '$.k1') = 'v1'", + "formatted": "SELECT JSON_EXTRACT(c, '$.k1') = 'v1'" + }, + { + "dialect": "duckdb", + "sql": "SELECT JSON_EXTRACT(c, '$[*].id')[0:2]", + "formatted": null + }, + { + "dialect": "duckdb", + "sql": "SELECT JSON_EXTRACT_STRING(c, '$[*].id')[0:2]", + "formatted": null + }, + { + "dialect": "duckdb", + "sql": "SELECT '{\"foo\": [1, 2, 3]}' -> 'foo' -> 0", + "formatted": "SELECT '{\"foo\": [1, 2, 3]}' -> 'foo' -> 0" + }, + { + "dialect": "duckdb", + "sql": "SELECT ($$hello)'world$$)", + "formatted": "SELECT\n (\n $$hello)'world$$\n )" + }, + { + "dialect": "duckdb", + "sql": "SELECT $$foo$$", + "formatted": "SELECT $$foo$$" + }, + { + "dialect": "duckdb", + "sql": "SELECT $tag$foo$tag$", + "formatted": "SELECT $tag$foo$tag$" + }, + { + "dialect": "duckdb", + "sql": "SELECT {'yes': 'duck', 'maybe': 'goose', 'huh': NULL, 'no': 'heron'}", + "formatted": "SELECT {'yes': 'duck', 'maybe': 'goose', 'huh': NULL, 'no': 'heron'}" + }, + { + "dialect": "duckdb", + "sql": "SELECT a['x space'] FROM (SELECT {'x space': 1, 'y': 2, 'z': 3} AS a)", + "formatted": "SELECT a['x space']\nFROM (\n SELECT {'x space': 1, 'y': 2, 'z': 3} AS a\n)" + }, + { + "dialect": "duckdb", + "sql": "WITH pivot_alias AS (PIVOT Cities ON Year USING SUM(Population) GROUP BY Country) SELECT * FROM pivot_alias", + "formatted": "WITH pivot_alias AS (\n PIVOT Cities ON Year USING SUM(Population) GROUP BY Country\n)\n\nSELECT *\nFROM pivot_alias" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM (PIVOT Cities ON Year USING SUM(Population) GROUP BY Country) AS pivot_alias", + "formatted": "SELECT *\nFROM (PIVOT Cities ON Year USING SUM(Population) GROUP BY Country) AS pivot_alias" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM cities PIVOT(SUM(population) FOR year IN (2000, 2010, 2020) GROUP BY country)", + "formatted": "SELECT *\nFROM cities PIVOT(SUM(population) FOR year IN (2000, 2010, 2020) GROUP BY country)" + }, + { + "dialect": "duckdb", + "sql": "SELECT schema_name, function_name, ROW_NUMBER() OVER my_window AS function_rank FROM DUCKDB_FUNCTIONS() WINDOW my_window AS (PARTITION BY schema_name ORDER BY function_name) QUALIFY ROW_NUMBER() OVER my_window < 3", + "formatted": "SELECT\n schema_name,\n function_name,\n ROW_NUMBER() OVER my_window AS function_rank\nFROM DUCKDB_FUNCTIONS()\nWINDOW\n my_window AS (PARTITION BY schema_name ORDER BY function_name)\nQUALIFY ROW_NUMBER() OVER my_window < 3" + }, + { + "dialect": "duckdb", + "sql": "SELECT SHA256('abc')", + "formatted": "SELECT SHA256('abc')" + }, + { + "dialect": "duckdb", + "sql": "SELECT SPLIT_PART('11.22.33', '.', 1)", + "formatted": "SELECT SPLIT_PART('11.22.33', '.', 1)" + }, + { + "dialect": "duckdb", + "sql": "SELECT NTH_VALUE(is_deleted, 2) OVER (PARTITION BY id) AS nth_is_deleted FROM my_table", + "formatted": "SELECT NTH_VALUE(is_deleted, 2) OVER (PARTITION BY id) AS nth_is_deleted\nFROM my_table" + }, + { + "dialect": "duckdb", + "sql": "SELECT NTH_VALUE(is_deleted, 2 IGNORE NULLS) OVER (PARTITION BY id) AS nth_is_deleted FROM my_table", + "formatted": "SELECT NTH_VALUE(is_deleted, 2 IGNORE NULLS) OVER (PARTITION BY id) AS nth_is_deleted\nFROM my_table" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_MAX(values) FROM table1", + "formatted": "SELECT LIST_MAX(values)\nFROM table1" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_MIN(values) FROM table1", + "formatted": "SELECT LIST_MIN(values)\nFROM table1" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_SORT(x, 'ASC')", + "formatted": "SELECT LIST_SORT(x, 'ASC')" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_SORT(x, 'DESC')", + "formatted": "SELECT LIST_SORT(x, 'DESC')" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_SORT(x, 'ASC', 'NULLS FIRST')", + "formatted": "SELECT LIST_SORT(x, 'ASC', 'NULLS FIRST')" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_SORT(x, 'ASC', 'NULLS LAST')", + "formatted": "SELECT LIST_SORT(x, 'ASC', 'NULLS LAST')" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_SORT(x, 'DESC', 'NULLS FIRST')", + "formatted": "SELECT LIST_SORT(x, 'DESC', 'NULLS FIRST')" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_SORT(x, 'DESC', 'NULLS LAST')", + "formatted": "SELECT LIST_SORT(x, 'DESC', 'NULLS LAST')" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_SORT(x, 'DE' || 'SC')", + "formatted": "SELECT LIST_SORT(x, 'DE' || 'SC')" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_SORT(x, 'DESC', 'NULLS' || ' FIRST')", + "formatted": "SELECT LIST_SORT(x, 'DESC', 'NULLS' || ' FIRST')" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_SORT(x, 'DE' || 'SC', 'NULLS' || ' FIRST')", + "formatted": "SELECT\n LIST_SORT(\n x,\n 'DE' || 'SC',\n 'NULLS' || ' FIRST'\n )" + }, + { + "dialect": "duckdb", + "sql": "SELECT PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY y DESC) FROM t", + "formatted": "SELECT PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY y DESC)\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT PERCENTILE_DISC(0.25) WITHIN GROUP (ORDER BY y DESC) FROM t", + "formatted": "SELECT PERCENTILE_DISC(0.25) WITHIN GROUP (ORDER BY y DESC)\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT REGEXP_EXTRACT(a, 'pattern', 0)", + "formatted": "SELECT REGEXP_EXTRACT(a, 'pattern', 0)" + }, + { + "dialect": "duckdb", + "sql": "SELECT REGEXP_EXTRACT(a, 'pattern', 0, 'i')", + "formatted": "SELECT REGEXP_EXTRACT(a, 'pattern', 0, 'i')" + }, + { + "dialect": "duckdb", + "sql": "SELECT REGEXP_EXTRACT(a, 'pattern', 1, 'i')", + "formatted": "SELECT REGEXP_EXTRACT(a, 'pattern', 1, 'i')" + }, + { + "dialect": "duckdb", + "sql": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 0)", + "formatted": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 0)" + }, + { + "dialect": "duckdb", + "sql": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 1)", + "formatted": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 1)" + }, + { + "dialect": "duckdb", + "sql": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 0, 'i')", + "formatted": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 0, 'i')" + }, + { + "dialect": "duckdb", + "sql": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 1, 'im')", + "formatted": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 1, 'im')" + }, + { + "dialect": "duckdb", + "sql": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 0)[2:]", + "formatted": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 0)[2:]" + }, + { + "dialect": "duckdb", + "sql": "SELECT ISNAN(x)", + "formatted": "SELECT ISNAN(x)" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM RANGE(1, 5, 10)", + "formatted": "SELECT *\nFROM RANGE(1, 5, 10)" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM GENERATE_SERIES(2, 13, 4)", + "formatted": "SELECT *\nFROM GENERATE_SERIES(2, 13, 4)" + }, + { + "dialect": "duckdb", + "sql": "SELECT i FROM RANGE(5) AS _(i) ORDER BY i ASC", + "formatted": "SELECT i\nFROM RANGE(5) AS _(i)\nORDER BY\n i ASC" + }, + { + "dialect": "duckdb", + "sql": "SELECT i FROM GENERATE_SERIES(12) AS _(i) ORDER BY i ASC", + "formatted": "SELECT i\nFROM GENERATE_SERIES(12) AS _(i)\nORDER BY\n i ASC" + }, + { + "dialect": "duckdb", + "sql": "SELECT LENGTH(foo)", + "formatted": "SELECT LENGTH(foo)" + }, + { + "dialect": "duckdb", + "sql": "SELECT ARRAY[1, 2, 3]", + "formatted": "SELECT ARRAY[1, 2, 3]" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM (DESCRIBE t)", + "formatted": "SELECT *\nFROM (DESCRIBE t)" + }, + { + "dialect": "duckdb", + "sql": "SELECT UNNEST([*COLUMNS('alias_.*')]) AS column_name", + "formatted": "SELECT UNNEST([*COLUMNS('alias_.*')]) AS column_name" + }, + { + "dialect": "duckdb", + "sql": "SELECT COALESCE(*COLUMNS(*)) FROM (SELECT NULL, 2, 3) AS t(a, b, c)", + "formatted": "SELECT COALESCE(*COLUMNS(*))\nFROM (\n SELECT\n NULL,\n 2,\n 3\n) AS t(a, b, c)" + }, + { + "dialect": "duckdb", + "sql": "SELECT id, STRUCT_PACK(*COLUMNS('m\\d')) AS measurements FROM many_measurements", + "formatted": "SELECT\n id,\n STRUCT_PACK(*COLUMNS('m\\d')) AS measurements\nFROM many_measurements" + }, + { + "dialect": "duckdb", + "sql": "SELECT COLUMNS(c -> c LIKE '%num%') FROM numbers", + "formatted": "SELECT COLUMNS(c -> c LIKE '%num%')\nFROM numbers" + }, + { + "dialect": "duckdb", + "sql": "SELECT MIN(COLUMNS(* REPLACE (number + id AS number))), COUNT(COLUMNS(* EXCLUDE (number))) FROM numbers", + "formatted": "SELECT\n MIN(COLUMNS(* REPLACE (number + id AS number))),\n COUNT(COLUMNS(* EXCLUDE (number)))\nFROM numbers" + }, + { + "dialect": "duckdb", + "sql": "SELECT COLUMNS(*) + COLUMNS(*) FROM numbers", + "formatted": "SELECT COLUMNS(*) + COLUMNS(*)\nFROM numbers" + }, + { + "dialect": "duckdb", + "sql": "SELECT COLUMNS('(id|numbers?)') FROM numbers", + "formatted": "SELECT COLUMNS('(id|numbers?)')\nFROM numbers" + }, + { + "dialect": "duckdb", + "sql": "SELECT COALESCE(COLUMNS(['a', 'b', 'c'])) AS result FROM (SELECT NULL AS a, 42 AS b, TRUE AS c)", + "formatted": "SELECT COALESCE(COLUMNS(['a', 'b', 'c'])) AS result\nFROM (\n SELECT\n NULL AS a,\n 42 AS b,\n TRUE AS c\n)" + }, + { + "dialect": "duckdb", + "sql": "SELECT COALESCE(*COLUMNS(['a', 'b', 'c'])) AS result FROM (SELECT NULL AS a, 42 AS b, TRUE AS c)", + "formatted": "SELECT COALESCE(*COLUMNS(['a', 'b', 'c'])) AS result\nFROM (\n SELECT\n NULL AS a,\n 42 AS b,\n TRUE AS c\n)" + }, + { + "dialect": "duckdb", + "sql": "SELECT DATE_ADD(CAST('2020-01-01' AS DATE), INTERVAL 1 DAY)", + "formatted": "SELECT DATE_ADD(CAST('2020-01-01' AS DATE), INTERVAL 1 DAY)" + }, + { + "dialect": "duckdb", + "sql": "SELECT #2, #1 FROM (VALUES (1, 'foo'))", + "formatted": "SELECT\n #2,\n #1\nFROM (VALUES (1, 'foo'))" + }, + { + "dialect": "duckdb", + "sql": "SELECT #2 AS a, #1 AS b FROM (VALUES (1, 'foo'))", + "formatted": "SELECT\n #2 AS a,\n #1 AS b\nFROM (VALUES (1, 'foo'))" + }, + { + "dialect": "duckdb", + "sql": "SELECT CUME_DIST( ORDER BY foo) OVER (ORDER BY 1) FROM (SELECT 1 AS foo)", + "formatted": "SELECT CUME_DIST(ORDER BY foo) OVER (ORDER BY 1)\nFROM (\n SELECT 1 AS foo\n)" + }, + { + "dialect": "duckdb", + "sql": "SELECT NTILE(1 ORDER BY foo) OVER (ORDER BY 1) FROM (SELECT 1 AS foo)", + "formatted": "SELECT NTILE(1 ORDER BY foo) OVER (ORDER BY 1)\nFROM (\n SELECT 1 AS foo\n)" + }, + { + "dialect": "duckdb", + "sql": "SELECT RANK( ORDER BY foo) OVER (ORDER BY 1) FROM (SELECT 1 AS foo)", + "formatted": "SELECT RANK(ORDER BY foo) OVER (ORDER BY 1)\nFROM (\n SELECT 1 AS foo\n)" + }, + { + "dialect": "duckdb", + "sql": "SELECT PERCENT_RANK( ORDER BY foo) OVER (ORDER BY 1) FROM (SELECT 1 AS foo)", + "formatted": "SELECT PERCENT_RANK(ORDER BY foo) OVER (ORDER BY 1)\nFROM (\n SELECT 1 AS foo\n)" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM t LIMIT 10 PERCENT", + "formatted": "SELECT *\nFROM t\nLIMIT 10 PERCENT" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM t LIMIT 10%", + "formatted": "SELECT *\nFROM t\nLIMIT 10%", + "previously_formatted": "SELECT\n *\nFROM t\nLIMIT 10 %" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM t LIMIT 10 PERCENT OFFSET 1", + "formatted": "SELECT *\nFROM t\nLIMIT 10 PERCENT\nOFFSET 1" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM t LIMIT 10% OFFSET 1", + "formatted": "SELECT *\nFROM t\nLIMIT 10%\nOFFSET 1", + "previously_formatted": "SELECT\n *\nFROM t\nLIMIT 10 %\nOFFSET 1" + }, + { + "dialect": "duckdb", + "sql": "SELECT CAST(ROW(1, 2) AS ROW(a INTEGER, b INTEGER))", + "formatted": "SELECT CAST(ROW(1, 2) AS ROW(a INTEGER, b INTEGER))" + }, + { + "dialect": "duckdb", + "sql": "SELECT row", + "formatted": "SELECT row" + }, + { + "dialect": "duckdb", + "sql": "SELECT TRY_STRPTIME('2013-04-28T20:57:01.123456789+07:00', '%Y-%m-%dT%H:%M:%S.%n%z')", + "formatted": "SELECT TRY_STRPTIME('2013-04-28T20:57:01.123456789+07:00', '%Y-%m-%dT%H:%M:%S.%n%z')" + }, + { + "dialect": "duckdb", + "sql": "SELECT REPLACE('apple pie', 'pie', 'cobbler') AS result", + "formatted": "SELECT REPLACE('apple pie', 'pie', 'cobbler') AS result" + }, + { + "dialect": "duckdb", + "sql": "SELECT REPLACE(CAST(CAST('apple pie' AS BLOB) AS TEXT), CAST(CAST('pie' AS BLOB) AS TEXT), CAST(CAST('cobbler' AS BLOB) AS TEXT)) AS result", + "formatted": "SELECT\n REPLACE(\n CAST(CAST('apple pie' AS BLOB) AS TEXT),\n CAST(CAST('pie' AS BLOB) AS TEXT),\n CAST(CAST('cobbler' AS BLOB) AS TEXT)\n ) AS result" + }, + { + "dialect": "duckdb", + "sql": "SELECT TRIM('***apple***', '*') AS result", + "formatted": "SELECT TRIM('***apple***', '*') AS result" + }, + { + "dialect": "duckdb", + "sql": "SELECT CAST(TRIM(CAST(CAST('***apple***' AS BLOB) AS TEXT), CAST(CAST('*' AS BLOB) AS TEXT)) AS BLOB) AS result", + "formatted": "SELECT CAST(TRIM(CAST(CAST('***apple***' AS BLOB) AS TEXT), CAST(CAST('*' AS BLOB) AS TEXT)) AS BLOB) AS result" + }, + { + "dialect": "duckdb", + "sql": "SELECT GREATEST(1.0, 2.5, NULL, 3.7)", + "formatted": "SELECT GREATEST(1.0, 2.5, NULL, 3.7)" + }, + { + "dialect": "duckdb", + "sql": "SELECT MAKE_DATE(DATE_PART(['year', 'month', 'day'], CURRENT_DATE))", + "formatted": "SELECT\n MAKE_DATE(\n DATE_PART(\n ['year', 'month', 'day'],\n CURRENT_DATE\n )\n )" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM t PIVOT(SUM(y) FOR foo IN y_enum)", + "formatted": "SELECT *\nFROM t PIVOT(SUM(y) FOR foo IN y_enum)" + }, + { + "dialect": "duckdb", + "sql": "SELECT 20_000 AS literal", + "formatted": "SELECT 20_000 AS literal" + }, + { + "dialect": "duckdb", + "sql": "SELECT 1_2E+1_0::FLOAT", + "formatted": "SELECT 1_2E+1_0::FLOAT" + }, + { + "dialect": "duckdb", + "sql": "SELECT [1, 2, 3][1 + 1:LENGTH([1, 2, 3]) + -1]", + "formatted": null + }, + { + "dialect": "duckdb", + "sql": "SELECT TODAY()", + "formatted": "SELECT TODAY()" + }, + { + "dialect": "duckdb", + "sql": "SELECT GET_CURRENT_TIME()", + "formatted": "SELECT GET_CURRENT_TIME()" + }, + { + "dialect": "duckdb", + "sql": "SELECT SUM(x) OVER (ORDER BY x GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) FROM t", + "formatted": "SELECT SUM(x) OVER (ORDER BY x GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW)\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT LAST_VALUE(x ORDER BY x IGNORE NULLS) OVER (ORDER BY x) FROM t", + "formatted": "SELECT LAST_VALUE(x ORDER BY x IGNORE NULLS) OVER (ORDER BY x)\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT LAST_VALUE(x ORDER BY x RESPECT NULLS) OVER (ORDER BY x) FROM t", + "formatted": "SELECT LAST_VALUE(x ORDER BY x RESPECT NULLS) OVER (ORDER BY x)\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT CURRENT_DATE", + "formatted": "SELECT CURRENT_DATE" + }, + { + "dialect": "duckdb", + "sql": "SELECT CURRENT_TIMESTAMP", + "formatted": "SELECT CURRENT_TIMESTAMP" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM tbl USING SAMPLE 5", + "formatted": "SELECT *\nFROM tbl\nUSING SAMPLE 5" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM tbl USING SAMPLE 10%", + "formatted": "SELECT *\nFROM tbl\nUSING SAMPLE 10%" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM tbl USING SAMPLE 10 PERCENT (bernoulli)", + "formatted": "SELECT *\nFROM tbl\nUSING SAMPLE 10 PERCENT (bernoulli)" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM tbl USING SAMPLE reservoir(50 ROWS) REPEATABLE (100)", + "formatted": "SELECT *\nFROM tbl\nUSING SAMPLE reservoir (50 ROWS) REPEATABLE (100)" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM tbl USING SAMPLE 10% (system, 377)", + "formatted": "SELECT *\nFROM tbl\nUSING SAMPLE 10% (system, 377)" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM tbl TABLESAMPLE RESERVOIR(20%), tbl2 WHERE tbl.i=tbl2.i", + "formatted": "SELECT *\nFROM tbl TABLESAMPLE RESERVOIR (20%), tbl2\nWHERE tbl.i = tbl2.i", + "previously_formatted": "SELECT\n *\nFROM tbl TABLESAMPLE RESERVOIR (20 %), tbl2\nWHERE\n tbl.i = tbl2.i" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM tbl, tbl2 WHERE tbl.i=tbl2.i USING SAMPLE RESERVOIR(20%)", + "formatted": "SELECT *\nFROM tbl, tbl2\nWHERE tbl.i = tbl2.i\nUSING SAMPLE RESERVOIR (20%)", + "previously_formatted": "SELECT\n *\nFROM tbl, tbl2\nWHERE\n tbl.i = tbl2.i\nUSING SAMPLE RESERVOIR (20 %)" + }, + { + "dialect": "duckdb", + "sql": "SELECT x::INT[3][3]", + "formatted": null + }, + { + "dialect": "duckdb", + "sql": "SELECT ARRAY[[[1]]]::INT[1][1][1]", + "formatted": null + }, + { + "dialect": "duckdb", + "sql": "WITH unpivot_alias AS (UNPIVOT monthly_sales ON COLUMNS(* EXCLUDE (empid, dept)) INTO NAME month VALUE sales) SELECT * FROM unpivot_alias", + "formatted": "WITH unpivot_alias AS (\n UNPIVOT monthly_sales ON COLUMNS(* EXCLUDE (empid, dept)) INTO NAME month VALUE sales\n)\n\nSELECT *\nFROM unpivot_alias" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM (UNPIVOT monthly_sales ON COLUMNS(* EXCLUDE (empid, dept)) INTO NAME month VALUE sales) AS unpivot_alias", + "formatted": "SELECT *\nFROM (UNPIVOT monthly_sales ON COLUMNS(* EXCLUDE (empid, dept)) INTO NAME month VALUE sales) AS unpivot_alias" + }, + { + "dialect": "duckdb", + "sql": "WITH cities(country, name, year, population) AS (SELECT 'NL', 'Amsterdam', 2000, 1005 UNION ALL SELECT 'US', 'Seattle', 2020, 738) PIVOT cities ON year USING SUM(population)", + "formatted": "WITH cities(country, name, year, population) AS (\n SELECT\n 'NL',\n 'Amsterdam',\n 2000,\n 1005\n UNION ALL\n SELECT\n 'US',\n 'Seattle',\n 2020,\n 738\n)\n\nPIVOT cities ON year USING SUM(population)" + }, + { + "dialect": "duckdb", + "sql": "WITH t1 AS (FROM (FROM t2 SELECT foo1, foo2)) FROM t1", + "formatted": "WITH t1 AS (\n FROM (FROM t2\n SELECT\n foo1,\n foo2\n )\n)\n\nFROM t1" + }, + { + "dialect": "duckdb", + "sql": "SELECT foo: 1", + "formatted": "SELECT foo:1" + }, + { + "dialect": "duckdb", + "sql": "SELECT foo: bar", + "formatted": "SELECT foo:bar" + }, + { + "dialect": "duckdb", + "sql": "SELECT foo: t.col FROM t", + "formatted": "SELECT foo:t.col\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT \"foo\" /* bla */: 1", + "formatted": "SELECT\n \"foo\" /* bla */:1" + }, + { + "dialect": "duckdb", + "sql": "SELECT \"foo\": 1 /* bla */", + "formatted": "SELECT \"foo\":1 /* bla */" + }, + { + "dialect": "duckdb", + "sql": "SELECT \"foo\": /* bla */ 1", + "formatted": "SELECT\n \"foo\": /* bla */1" + }, + { + "dialect": "duckdb", + "sql": "SELECT \"foo\": /* bla */ 1 /* foo */", + "formatted": "SELECT\n \"foo\": /* bla */1 /* foo */" + }, + { + "dialect": "duckdb", + "sql": "SELECT \"foo\": 1", + "formatted": "SELECT \"foo\":1" + }, + { + "dialect": "duckdb", + "sql": "SELECT foo: 1, bar: 2, baz: 3", + "formatted": "SELECT\n foo:1,\n bar:2,\n baz:3" + }, + { + "dialect": "duckdb", + "sql": "SELECT e: 1 + 2, f: len('asdf'), s: (SELECT 42)", + "formatted": "SELECT\n e:1 + 2,\n f:len('asdf'), s:\n (\n SELECT 42\n )" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM foo: bar", + "formatted": "SELECT *\nFROM foo:bar" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM foo: c.db.tbl", + "formatted": "SELECT *\nFROM foo:c.db.tbl" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM foo /* bla */: bar", + "formatted": "SELECT *\nFROM foo /* bla */:bar" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM foo /* bla */: bar /* baz */", + "formatted": "SELECT *\nFROM foo /* bla */:bar /* baz */" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM foo /* bla */: /* baz */ bar /* boo */", + "formatted": "SELECT *\nFROM foo /* bla */: /* baz */bar /* boo */" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM r: range(10), v: (VALUES (42)), s: (FROM range(10))", + "formatted": "SELECT *\nFROM r:range(10), v:(VALUES (42)), s:(\n FROM RANGE(10)\n)" + }, + { + "dialect": "duckdb", + "sql": "\n SELECT\n l_returnflag,\n l_linestatus,\n sum_qty: sum(l_quantity),\n sum_base_price: sum(l_extendedprice),\n sum_disc_price: sum(l_extendedprice * (1-l_discount)),\n sum_charge: sum(l_extendedprice * (1-l_discount) * (1+l_tax)),\n avg_qty: avg(l_quantity),\n avg_price: avg(l_extendedprice),\n avg_disc: avg(l_discount),\n count_order: count(*)\n ", + "formatted": "SELECT\n l_returnflag,\n l_linestatus,\n sum_qty:sum(l_quantity),\n sum_base_price:sum(l_extendedprice),\n sum_disc_price:sum(l_extendedprice\n * (\n 1 - l_discount\n )),\n sum_charge:sum(l_extendedprice\n * (\n 1 - l_discount\n )\n * (\n 1 + l_tax\n )),\n avg_qty:avg(l_quantity),\n avg_price:avg(l_extendedprice),\n avg_disc:avg(l_discount),\n count_order:count(*)" + }, + { + "dialect": "duckdb", + "sql": "SELECT @col FROM t", + "formatted": "SELECT @col\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT @col + 1 FROM t", + "formatted": "SELECT @col + 1\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT (@col) + 1 FROM t", + "formatted": "SELECT\n (@col\n )\n + 1\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT @(-1)", + "formatted": "SELECT\n @(\n -1\n )" + }, + { + "dialect": "duckdb", + "sql": "SELECT @(-1) + 1", + "formatted": "SELECT\n @(\n -1\n )\n + 1" + }, + { + "dialect": "duckdb", + "sql": "SELECT (@-1) + 1", + "formatted": "SELECT\n (@-1\n )\n + 1" + }, + { + "dialect": "duckdb", + "sql": "WITH RECURSIVE tbl(a, b) USING KEY (a) AS (SELECT a, b FROM (VALUES (1, 3), (2, 4)) AS t(a, b) UNION SELECT a + 1, b FROM tbl WHERE a < 3) SELECT * FROM tbl", + "formatted": "WITH RECURSIVE tbl(a, b) USING KEY (a) AS (\n SELECT\n a,\n b\n FROM (VALUES (1, 3), (2, 4)) AS t(a, b)\n UNION\n SELECT\n a + 1,\n b\n FROM tbl\n WHERE a < 3\n)\nSELECT *\nFROM tbl" + }, + { + "dialect": "duckdb", + "sql": "WITH RECURSIVE tbl(a, b) USING KEY (a, b) AS (SELECT a, b FROM (VALUES (1, 3), (2, 4)) AS t(a, b) UNION SELECT a + 1, b FROM tbl WHERE a < 3) SELECT * FROM tbl", + "formatted": "WITH RECURSIVE tbl(a, b) USING KEY (a, b) AS (\n SELECT\n a,\n b\n FROM (VALUES (1, 3), (2, 4)) AS t(a, b)\n UNION\n SELECT\n a + 1,\n b\n FROM tbl\n WHERE a < 3\n)\nSELECT *\nFROM tbl" + }, + { + "dialect": "duckdb", + "sql": "WITH _data AS (SELECT [{'a': 1, 'b': 2}, {'a': 2, 'b': 3}] AS col) SELECT t.col['b'] FROM _data, UNNEST(_data.col) AS t(col) WHERE t.col['a'] = 1", + "formatted": "WITH _data AS (\n SELECT\n [\n {'a': 1, 'b': 2},\n {'a': 2, 'b': 3}\n ] AS col\n)\n\nSELECT t.col['b']\nFROM _data, UNNEST(_data.col) AS t(col)\nWHERE t.col['a'] = 1" + }, + { + "dialect": "duckdb", + "sql": "SELECT [4, 5, 6] AS l, [x FOR x, i IN l IF i = 2] AS filtered", + "formatted": "SELECT\n [4, 5, 6] AS l,\n [x FOR x, i IN l IF i = 2] AS filtered" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_VALUE(1)[i]", + "formatted": "SELECT LIST_VALUE(1)[i]" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_APPLY(RANGE(1, 4), i -> {'f1': LIST_VALUE(1, 2, 3)[i], 'f2': LIST_VALUE(1, 2, 3)[i]})", + "formatted": "SELECT\n LIST_APPLY(\n RANGE(1, 4),\n i -> {'f1': LIST_VALUE(1, 2, 3)[i], 'f2': LIST_VALUE(1, 2, 3)[i]}\n )" + }, + { + "dialect": "duckdb", + "sql": "SELECT UNNEST([1, 2])", + "formatted": "SELECT UNNEST([1, 2])" + }, + { + "dialect": "duckdb", + "sql": "SELECT file[:256] FROM GLOB('*')", + "formatted": null + }, + { + "dialect": "duckdb", + "sql": "SELECT file[256] FROM GLOB('*')", + "formatted": "SELECT file[256]\nFROM GLOB('*')" + }, + { + "dialect": "postgres", + "sql": "WITH t AS (SELECT 1 AS \"null\") SELECT t.null FROM t", + "formatted": "WITH t AS (\n SELECT 1 AS \"null\"\n)\n\nSELECT t.null\nFROM t" + }, + { + "dialect": "postgres", + "sql": "WITH t AS (SELECT 1 AS \"true\") SELECT t.true FROM t", + "formatted": "WITH t AS (\n SELECT 1 AS \"true\"\n)\n\nSELECT t.true\nFROM t" + }, + { + "dialect": "postgres", + "sql": "SELECT '%' SIMILAR TO '^%' ESCAPE '^'", + "formatted": "SELECT '%' SIMILAR TO '^%' ESCAPE '^'" + }, + { + "dialect": "postgres", + "sql": "SELECT GET_BIT(CAST(44 AS BIT(10)), 6)", + "formatted": "SELECT GET_BIT(CAST(44 AS BIT(10)), 6)" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM t GROUP BY ROLLUP (a || '^' || b)", + "formatted": "SELECT *\nFROM t\nGROUP BY\n ROLLUP (\n a || '^' || b\n )" + }, + { + "dialect": "postgres", + "sql": "SELECT COSH(1.5)", + "formatted": "SELECT COSH(1.5)" + }, + { + "dialect": "postgres", + "sql": "SELECT EXP(1)", + "formatted": "SELECT EXP(1)" + }, + { + "dialect": "postgres", + "sql": "SELECT MODE() WITHIN GROUP (ORDER BY status DESC) AS most_common FROM orders", + "formatted": "SELECT MODE() WITHIN GROUP (ORDER BY status DESC) AS most_common\nFROM orders" + }, + { + "dialect": "postgres", + "sql": "SELECT ST_DISTANCE(gg1, gg2, FALSE) AS sphere_dist", + "formatted": "SELECT ST_DISTANCE(gg1, gg2, FALSE) AS sphere_dist" + }, + { + "dialect": "postgres", + "sql": "SELECT EXTRACT(QUARTER FROM CAST('2025-04-26' AS DATE))", + "formatted": "SELECT EXTRACT(QUARTER FROM CAST('2025-04-26' AS DATE))" + }, + { + "dialect": "postgres", + "sql": "SELECT DATE_TRUNC('QUARTER', CAST('2025-04-26' AS DATE))", + "formatted": "SELECT DATE_TRUNC('QUARTER', CAST('2025-04-26' AS DATE))" + }, + { + "dialect": "postgres", + "sql": "SELECT x FROM t WHERE CAST($1 AS TEXT) = 'ok'", + "formatted": "SELECT x\nFROM t\nWHERE CAST($1 AS TEXT) = 'ok'" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM t TABLESAMPLE SYSTEM (50) REPEATABLE (55)", + "formatted": "SELECT *\nFROM t TABLESAMPLE SYSTEM (50) REPEATABLE (55)" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM events WHERE doc @? '$.a[*] ? (@ > 2)'", + "formatted": "SELECT *\nFROM events\nWHERE doc @? '$.a[*] ? (@ > 2)'" + }, + { + "dialect": "postgres", + "sql": "SELECT ARRAY[1, 2, 3]", + "formatted": "SELECT ARRAY[1, 2, 3]" + }, + { + "dialect": "postgres", + "sql": "SELECT ARRAY(SELECT 1)", + "formatted": "SELECT\n ARRAY(SELECT\n 1)" + }, + { + "dialect": "postgres", + "sql": "SELECT CASE WHEN SUBSTRING('abcdefg') IN ('ab') THEN 1 ELSE 0 END", + "formatted": "SELECT\n CASE\n WHEN SUBSTRING('abcdefg') IN ('ab')\n THEN 1\n ELSE 0\n END" + }, + { + "dialect": "postgres", + "sql": "SELECT e'\\xDEADBEEF'", + "formatted": "SELECT e'\\xDEADBEEF'" + }, + { + "dialect": "postgres", + "sql": "SELECT CAST(e'\\176' AS BYTEA)", + "formatted": "SELECT CAST(e'\\176' AS BYTEA)" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM x WHERE SUBSTRING('Thomas' FROM '...$') IN ('mas')", + "formatted": "SELECT *\nFROM x\nWHERE SUBSTRING('Thomas' FROM '...$') IN ('mas')" + }, + { + "dialect": "postgres", + "sql": "SELECT TRIM(' X' FROM ' XXX ')", + "formatted": "SELECT TRIM(' X' FROM ' XXX ')" + }, + { + "dialect": "postgres", + "sql": "SELECT TRIM(LEADING 'bla' FROM ' XXX ' COLLATE utf8_bin)", + "formatted": "SELECT TRIM(LEADING 'bla' FROM ' XXX ' COLLATE utf8_bin)" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM JSON_TO_RECORDSET(z) AS y(\"rank\" INT)", + "formatted": "SELECT *\nFROM JSON_TO_RECORDSET(z) AS y(\"rank\" INT)" + }, + { + "dialect": "postgres", + "sql": "SELECT ~x", + "formatted": "SELECT ~x" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM r CROSS JOIN LATERAL UNNEST(ARRAY[1]) AS s(location)", + "formatted": "SELECT *\nFROM r\nCROSS JOIN LATERAL UNNEST(ARRAY[1]) AS s(location)" + }, + { + "dialect": "postgres", + "sql": "SELECT CURRENT_SCHEMA", + "formatted": "SELECT CURRENT_SCHEMA" + }, + { + "dialect": "postgres", + "sql": "SELECT CURRENT_USER", + "formatted": "SELECT CURRENT_USER" + }, + { + "dialect": "postgres", + "sql": "SELECT CURRENT_ROLE", + "formatted": "SELECT CURRENT_ROLE" + }, + { + "dialect": "postgres", + "sql": "SELECT VERSION()", + "formatted": "SELECT VERSION()" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM ONLY t1", + "formatted": "SELECT *\nFROM ONLY t1" + }, + { + "dialect": "postgres", + "sql": "SELECT INTERVAL '-1 MONTH'", + "formatted": "SELECT INTERVAL '-1 MONTH'" + }, + { + "dialect": "postgres", + "sql": "SELECT INTERVAL '4.1 DAY'", + "formatted": "SELECT INTERVAL '4.1 DAY'" + }, + { + "dialect": "postgres", + "sql": "SELECT INTERVAL '3.14159 HOUR'", + "formatted": "SELECT INTERVAL '3.14159 HOUR'" + }, + { + "dialect": "postgres", + "sql": "SELECT INTERVAL '2.5 MONTH'", + "formatted": "SELECT INTERVAL '2.5 MONTH'" + }, + { + "dialect": "postgres", + "sql": "SELECT INTERVAL '-10.75 MINUTE'", + "formatted": "SELECT INTERVAL '-10.75 MINUTE'" + }, + { + "dialect": "postgres", + "sql": "SELECT INTERVAL '0.123456789 SECOND'", + "formatted": "SELECT INTERVAL '0.123456789 SECOND'" + }, + { + "dialect": "postgres", + "sql": "SELECT date_col - INTERVAL '30' FROM t", + "formatted": "SELECT date_col - INTERVAL '30'\nFROM t" + }, + { + "dialect": "postgres", + "sql": "SELECT date_col - INTERVAL '1' AS one_second_later", + "formatted": "SELECT date_col - INTERVAL '1' AS one_second_later" + }, + { + "dialect": "postgres", + "sql": "SELECT date_col - INTERVAL '30' DAY FROM t", + "formatted": null + }, + { + "dialect": "postgres", + "sql": "SELECT date_col - INTERVAL '1' HOUR AS one_hour_later", + "formatted": null + }, + { + "dialect": "postgres", + "sql": "SELECT SUM(x) OVER (PARTITION BY y ORDER BY interval ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) - SUM(x) OVER (PARTITION BY y ORDER BY interval ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS total", + "formatted": "SELECT SUM(x) OVER (PARTITION BY y ORDER BY interval ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) - SUM(x) OVER (PARTITION BY y ORDER BY interval ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS total" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM test_data, LATERAL JSONB_ARRAY_ELEMENTS(data) WITH ORDINALITY AS elem(value, ordinality)", + "formatted": "SELECT *\nFROM test_data, LATERAL JSONB_ARRAY_ELEMENTS(data) WITH ORDINALITY AS elem(value, ordinality)" + }, + { + "dialect": "postgres", + "sql": "SELECT id, name FROM xml_data AS t, XMLTABLE('/root/user' PASSING t.xml COLUMNS id INT PATH '@id', name TEXT PATH 'name/text()') AS x", + "formatted": "SELECT\n id,\n name\nFROM xml_data AS t, XMLTABLE(\n '/root/user'\n PASSING\n t.xml\n COLUMNS\n id INT PATH '@id',\n name TEXT PATH 'name/text()'\n) AS x" + }, + { + "dialect": "postgres", + "sql": "SELECT id, value FROM xml_content AS t, XMLTABLE(XMLNAMESPACES('http://example.com/ns1' AS ns1, 'http://example.com/ns2' AS ns2), '/root/data' PASSING t.xml COLUMNS id INT PATH '@ns1:id', value TEXT PATH 'ns2:value/text()') AS x", + "formatted": null + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM t WHERE some_column >= CURRENT_DATE + INTERVAL '1 day 1 hour' AND some_another_column IS TRUE", + "formatted": "SELECT *\nFROM t\nWHERE\n some_column >= CURRENT_DATE + INTERVAL '1 day 1 hour' AND some_another_column IS TRUE" + }, + { + "dialect": "postgres", + "sql": "WITH t1 AS MATERIALIZED (SELECT 1), t2 AS NOT MATERIALIZED (SELECT 2) SELECT * FROM t1, t2", + "formatted": "WITH t1 AS MATERIALIZED (\n SELECT 1\n),\n\nt2 AS NOT MATERIALIZED (\n SELECT 2\n)\n\nSELECT *\nFROM t1, t2" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM JSON_ARRAY_ELEMENTS('[1,true, [2,false]]') WITH ORDINALITY", + "formatted": "SELECT *\nFROM JSON_ARRAY_ELEMENTS('[1,true, [2,false]]') WITH ORDINALITY" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM JSON_ARRAY_ELEMENTS('[1,true, [2,false]]') WITH ORDINALITY AS kv_json", + "formatted": "SELECT *\nFROM JSON_ARRAY_ELEMENTS('[1,true, [2,false]]') WITH ORDINALITY AS kv_json" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM JSON_ARRAY_ELEMENTS('[1,true, [2,false]]') WITH ORDINALITY AS kv_json(a, b)", + "formatted": "SELECT *\nFROM JSON_ARRAY_ELEMENTS('[1,true, [2,false]]') WITH ORDINALITY AS kv_json(a, b)" + }, + { + "dialect": "postgres", + "sql": "SELECT SUM(x) OVER a, SUM(y) OVER b FROM c WINDOW a AS (PARTITION BY d), b AS (PARTITION BY e)", + "formatted": "SELECT\n SUM(x) OVER a,\n SUM(y) OVER b\nFROM c\nWINDOW\n a AS (PARTITION BY d),\n b AS (PARTITION BY e)" + }, + { + "dialect": "postgres", + "sql": "SELECT CASE WHEN SUBSTRING('abcdefg' FROM 1) IN ('ab') THEN 1 ELSE 0 END", + "formatted": "SELECT\n CASE\n WHEN SUBSTRING('abcdefg' FROM 1) IN ('ab')\n THEN 1\n ELSE 0\n END" + }, + { + "dialect": "postgres", + "sql": "SELECT CASE WHEN SUBSTRING('abcdefg' FROM 1 FOR 2) IN ('ab') THEN 1 ELSE 0 END", + "formatted": "SELECT\n CASE\n WHEN SUBSTRING('abcdefg' FROM 1 FOR 2) IN ('ab')\n THEN 1\n ELSE 0\n END" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM \"x\" WHERE SUBSTRING(\"x\".\"foo\" FROM 1 FOR 2) IN ('mas')", + "formatted": "SELECT *\nFROM \"x\"\nWHERE SUBSTRING(\"x\".\"foo\" FROM 1 FOR 2) IN ('mas')" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM x WHERE SUBSTRING('Thomas' FROM '%#\"o_a#\"_' FOR '#') IN ('mas')", + "formatted": "SELECT *\nFROM x\nWHERE SUBSTRING('Thomas' FROM '%#\"o_a#\"_' FOR '#') IN ('mas')" + }, + { + "dialect": "postgres", + "sql": "SELECT SUBSTRING('bla' + 'foo' || 'bar' FROM 3 - 1 + 5 FOR 4 + SOME_FUNC(arg1, arg2))", + "formatted": "SELECT SUBSTRING('bla' + 'foo' || 'bar' FROM 3 - 1 + 5 FOR 4 + SOME_FUNC(arg1, arg2))" + }, + { + "dialect": "postgres", + "sql": "SELECT TO_TIMESTAMP(1284352323.5), TO_TIMESTAMP('05 Dec 2000', 'DD Mon YYYY')", + "formatted": "SELECT\n TO_TIMESTAMP(1284352323.5),\n TO_TIMESTAMP('05 Dec 2000', 'DD Mon YYYY')" + }, + { + "dialect": "postgres", + "sql": "SELECT TO_TIMESTAMP('05 Dec 2000 10:00 AM', 'DD Mon YYYY HH:MI AM')", + "formatted": "SELECT TO_TIMESTAMP('05 Dec 2000 10:00 AM', 'DD Mon YYYY HH:MI AM')" + }, + { + "dialect": "postgres", + "sql": "SELECT TO_TIMESTAMP('05 Dec 2000 10:00 PM', 'DD Mon YYYY HH:MI PM')", + "formatted": "SELECT TO_TIMESTAMP('05 Dec 2000 10:00 PM', 'DD Mon YYYY HH:MI PM')" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM foo, LATERAL (SELECT * FROM bar WHERE bar.id = foo.bar_id) AS ss", + "formatted": "SELECT *\nFROM foo, LATERAL (\n SELECT *\n FROM bar\n WHERE bar.id = foo.bar_id\n) AS ss" + }, + { + "dialect": "postgres", + "sql": "SELECT c.oid, n.nspname, c.relname FROM pg_catalog.pg_class AS c LEFT JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace WHERE c.relname OPERATOR(pg_catalog.~) '^(courses)$' COLLATE pg_catalog.default AND pg_catalog.PG_TABLE_IS_VISIBLE(c.oid) ORDER BY 2, 3", + "formatted": "SELECT\n c.oid,\n n.nspname,\n c.relname\nFROM pg_catalog.pg_class AS c\nLEFT JOIN pg_catalog.pg_namespace AS n\n ON n.oid = c.relnamespace\nWHERE\n c.relname OPERATOR(pg_catalog.~) '^(courses)$' COLLATE pg_catalog.default AND pg_catalog.PG_TABLE_IS_VISIBLE(c.oid)\nORDER BY\n 2,\n 3" + }, + { + "dialect": "postgres", + "sql": "SELECT date_add(current_date, interval '7' day)", + "formatted": null + }, + { + "dialect": "postgres", + "sql": "SELECT e'foo \\' bar'", + "formatted": "SELECT e'foo \\' bar'" + }, + { + "dialect": "postgres", + "sql": "SELECT e'\\n'", + "formatted": "SELECT e'\\n'" + }, + { + "dialect": "postgres", + "sql": "SELECT e'\\t'", + "formatted": "SELECT e'\\t'" + }, + { + "dialect": "postgres", + "sql": "SELECT e'update table_name set a = \\'foo\\' where 1 = 0' AS x FROM tab", + "formatted": "SELECT e'update table_name set a = \\'foo\\' where 1 = 0' AS x\nFROM tab" + }, + { + "dialect": "postgres", + "sql": "select count() OVER(partition by a order by a range offset preceding exclude current row)", + "formatted": "SELECT COUNT() OVER (PARTITION BY a ORDER BY a range offset preceding exclude CURRENT ROW)" + }, + { + "dialect": "postgres", + "sql": "SELECT SUBSTRING('Thomas' FOR 3 FROM 2)", + "formatted": "SELECT SUBSTRING('Thomas' FOR 3 FROM 2)" + }, + { + "dialect": "postgres", + "sql": "SELECT DATE_PART('isodow'::varchar(6), current_date)", + "formatted": "SELECT DATE_PART('isodow'::varchar(6), CURRENT_DATE)" + }, + { + "dialect": "postgres", + "sql": "SELECT $$a$$", + "formatted": "SELECT $$a$$" + }, + { + "dialect": "postgres", + "sql": "SELECT $$Dianne's order$$", + "formatted": "SELECT $$Dianne's order$$" + }, + { + "dialect": "postgres", + "sql": "SELECT $$The price is $9.95$$ AS msg", + "formatted": "SELECT $$The price is $9.95$$ AS msg" + }, + { + "dialect": "postgres", + "sql": "SELECT 'testa 1' NOT LIKE ALL (ARRAY['testa%', 'testb%'])", + "formatted": "SELECT\n 'testa 1' NOT LIKE ALL (\n ARRAY['testa%', 'testb%']\n )" + }, + { + "dialect": "postgres", + "sql": "SELECT 'testa 1' NOT LIKE ANY(ARRAY['testa%', 'testb%'])", + "formatted": "SELECT\n 'testa 1' NOT LIKE ANY(\n ARRAY['testa%', 'testb%']\n )" + }, + { + "dialect": "postgres", + "sql": "SELECT 'testa 1' NOT ILIKE ALL (ARRAY['testa%', 'testb%'])", + "formatted": "SELECT\n 'testa 1' NOT ILIKE ALL (\n ARRAY['testa%', 'testb%']\n )" + }, + { + "dialect": "postgres", + "sql": "SELECT 'testa 1' NOT ILIKE ANY(ARRAY['testa%', 'testb%'])", + "formatted": "SELECT\n 'testa 1' NOT ILIKE ANY(\n ARRAY['testa%', 'testb%']\n )" + }, + { + "dialect": "postgres", + "sql": "SELECT NOT 'testa 1' LIKE ALL (ARRAY['testa%', 'testb%'])", + "formatted": "SELECT\n NOT 'testa 1' LIKE ALL (\n ARRAY['testa%', 'testb%']\n )" + }, + { + "dialect": "postgres", + "sql": "SELECT NOT ('testa 1' LIKE ALL (ARRAY['testa%', 'testb%']))", + "formatted": "SELECT\n NOT (\n 'testa 1' LIKE ALL (\n ARRAY['testa%', 'testb%']\n )\n )" + }, + { + "dialect": "postgres", + "sql": "SELECT to_timestamp(123)::time without time zone", + "formatted": "SELECT to_timestamp(123)::time without time zone" + }, + { + "dialect": "postgres", + "sql": "SELECT SUM(x) OVER (PARTITION BY a ORDER BY d ROWS 1 PRECEDING)", + "formatted": "SELECT SUM(x) OVER (PARTITION BY a ORDER BY d ROWS 1 PRECEDING)" + }, + { + "dialect": "postgres", + "sql": "SELECT SUBSTRING(2022::CHAR(4) || LPAD(3::CHAR(2), 2, '0') FROM 3 FOR 4)", + "formatted": "SELECT SUBSTRING(2022::CHAR(4) || LPAD(3::CHAR(2), 2, '0') FROM 3 FOR 4)" + }, + { + "dialect": "postgres", + "sql": "SELECT m.name FROM manufacturers AS m LEFT JOIN LATERAL GET_PRODUCT_NAMES(m.id) pname ON TRUE WHERE pname IS NULL", + "formatted": "SELECT m.name\nFROM manufacturers AS m\nLEFT JOIN LATERAL GET_PRODUCT_NAMES(m.id) pname\n ON TRUE\nWHERE pname IS NULL", + "previously_formatted": "SELECT\n m.name\nFROM manufacturers AS m\nLEFT JOIN LATERAL GET_PRODUCT_NAMES(m.id) AS pname\n ON TRUE\nWHERE\n pname IS NULL" + }, + { + "dialect": "postgres", + "sql": "SELECT p1.id, p2.id, v1, v2 FROM polygons AS p1, polygons AS p2, LATERAL VERTICES(p1.poly) v1, LATERAL VERTICES(p2.poly) v2 WHERE (v1 <-> v2) < 10 AND p1.id <> p2.id", + "formatted": "SELECT\n p1.id,\n p2.id,\n v1,\n v2\nFROM polygons AS p1, polygons AS p2, LATERAL VERTICES(p1.poly) v1, LATERAL VERTICES(p2.poly) v2\nWHERE\n (\n v1 <-> v2\n ) < 10 AND p1.id <> p2.id", + "previously_formatted": "SELECT\n p1.id,\n p2.id,\n v1,\n v2\nFROM polygons AS p1, polygons AS p2, LATERAL VERTICES(p1.poly) AS v1, LATERAL VERTICES(p2.poly) AS v2\nWHERE\n (\n v1 <-> v2\n ) < 10 AND p1.id <> p2.id" + }, + { + "dialect": "postgres", + "sql": "SELECT id, email, CAST(deleted AS TEXT) FROM users WHERE deleted NOTNULL", + "formatted": "SELECT\n id,\n email,\n CAST(deleted AS TEXT)\nFROM users\nWHERE deleted NOTNULL" + }, + { + "dialect": "postgres", + "sql": "SELECT id, email, CAST(deleted AS TEXT) FROM users WHERE NOT deleted ISNULL", + "formatted": "SELECT\n id,\n email,\n CAST(deleted AS TEXT)\nFROM users\nWHERE NOT deleted ISNULL" + }, + { + "dialect": "postgres", + "sql": "SELECT JSON_ARRAY_ELEMENTS((foo->'sections')::JSON) AS sections", + "formatted": "SELECT\n JSON_ARRAY_ELEMENTS((\n foo -> 'sections'\n )::JSON) AS sections" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM t1*", + "formatted": "SELECT *\nFROM t1*" + }, + { + "dialect": "postgres", + "sql": "SELECT SUBSTRING('afafa' for 1)", + "formatted": "SELECT SUBSTRING('afafa' FOR 1)" + }, + { + "dialect": "postgres", + "sql": "\n WITH\n json_data AS (SELECT '{\"field_id\": [1, 2, 3]}'::JSON AS data),\n field_ids AS (SELECT 'field_id' AS field_id)\n\n SELECT\n JSON_ARRAY_ELEMENTS(json_data.data -> field_ids.field_id) AS element\n FROM json_data, field_ids\n ", + "formatted": "WITH json_data AS (\n SELECT '{\"field_id\": [1, 2, 3]}'::JSON AS data\n),\n\nfield_ids AS (\n SELECT 'field_id' AS field_id\n)\n\nSELECT JSON_ARRAY_ELEMENTS(json_data.data -> field_ids.field_id) AS element\nFROM json_data, field_ids" + }, + { + "dialect": "postgres", + "sql": "SELECT TRIM(LEADING ' XXX ' COLLATE \"de_DE\")", + "formatted": "SELECT TRIM(LEADING ' XXX ' COLLATE \"de_DE\")" + }, + { + "dialect": "postgres", + "sql": "SELECT TRIM(TRAILING ' XXX ' COLLATE \"de_DE\")", + "formatted": "SELECT TRIM(TRAILING ' XXX ' COLLATE \"de_DE\")" + }, + { + "dialect": "postgres", + "sql": "SELECT js, js IS JSON AS \"json?\", js IS JSON VALUE AS \"scalar?\", js IS JSON SCALAR AS \"scalar?\", js IS JSON OBJECT AS \"object?\", js IS JSON ARRAY AS \"array?\" FROM t", + "formatted": "SELECT\n js,\n js IS JSON AS \"json?\",\n js IS JSON VALUE AS \"scalar?\",\n js IS JSON SCALAR AS \"scalar?\",\n js IS JSON OBJECT AS \"object?\",\n js IS JSON ARRAY AS \"array?\"\nFROM t" + }, + { + "dialect": "postgres", + "sql": "SELECT js, js IS JSON ARRAY WITH UNIQUE KEYS AS \"array w. UK?\", js IS JSON ARRAY WITHOUT UNIQUE KEYS AS \"array w/o UK?\", js IS JSON ARRAY UNIQUE KEYS AS \"array w UK 2?\" FROM t", + "formatted": "SELECT\n js,\n js IS JSON ARRAY WITH UNIQUE KEYS AS \"array w. UK?\",\n js IS JSON ARRAY WITHOUT UNIQUE KEYS AS \"array w/o UK?\",\n js IS JSON ARRAY UNIQUE KEYS AS \"array w UK 2?\"\nFROM t" + }, + { + "dialect": "postgres", + "sql": "SELECT 1 FROM ((VALUES (1)) AS vals(id) LEFT OUTER JOIN tbl ON vals.id = tbl.id)", + "formatted": "SELECT 1\nFROM ((VALUES (1)) AS vals(id)\nLEFT OUTER JOIN tbl\n ON vals.id = tbl.id)" + }, + { + "dialect": "postgres", + "sql": "SELECT OVERLAY(a PLACING b FROM 1)", + "formatted": "SELECT OVERLAY(a PLACING b FROM 1)" + }, + { + "dialect": "postgres", + "sql": "SELECT OVERLAY(a PLACING b FROM 1 FOR 1)", + "formatted": "SELECT OVERLAY(a PLACING b FROM 1 FOR 1)" + }, + { + "dialect": "postgres", + "sql": "SELECT PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY a) FILTER(WHERE CAST(b AS BOOLEAN)) AS mean_value FROM (VALUES (0, 't')) AS fake_data(a, b)", + "formatted": "SELECT PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY a) FILTER(WHERE CAST(b AS BOOLEAN)) AS mean_value\nFROM (VALUES (0, 't')) AS fake_data(a, b)" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM foo WHERE id = %s", + "formatted": "SELECT *\nFROM foo\nWHERE id = %s" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM foo WHERE id = %(id_param)s", + "formatted": "SELECT *\nFROM foo\nWHERE id = %(id_param)s" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM foo WHERE id = ?", + "formatted": "SELECT *\nFROM foo\nWHERE id = ?" + }, + { + "dialect": "postgres", + "sql": "SELECT '{\"a\":1, \"b\":2, \"c\":3}'::jsonb ?| array['b', 'c']", + "formatted": "SELECT '{\"a\":1, \"b\":2, \"c\":3}'::jsonb ?| ARRAY['b', 'c']" + }, + { + "dialect": "postgres", + "sql": "SELECT '[\"a\", \"b\"]'::jsonb ?& array['a', 'b']", + "formatted": "SELECT '[\"a\", \"b\"]'::jsonb ?& ARRAY['a', 'b']" + }, + { + "dialect": "postgres", + "sql": "SELECT '[\"a\", {\"b\":1}]'::jsonb #- '{1,b}'", + "formatted": "SELECT '[\"a\", {\"b\":1}]'::jsonb #- '{1,b}'" + }, + { + "dialect": "postgres", + "sql": "SELECT JSON_AGG(DISTINCT name) FROM users", + "formatted": "SELECT JSON_AGG(DISTINCT name)\nFROM users" + }, + { + "dialect": "postgres", + "sql": "SELECT JSON_AGG(c1 ORDER BY c1) FROM (VALUES ('c'), ('b'), ('a')) AS t(c1)", + "formatted": "SELECT JSON_AGG(c1 ORDER BY c1)\nFROM (VALUES ('c'), ('b'), ('a')) AS t(c1)" + }, + { + "dialect": "postgres", + "sql": "SELECT JSON_AGG(DISTINCT c1 ORDER BY c1) FROM (VALUES ('c'), ('b'), ('a')) AS t(c1)", + "formatted": "SELECT JSON_AGG(DISTINCT c1 ORDER BY c1)\nFROM (VALUES ('c'), ('b'), ('a')) AS t(c1)" + }, + { + "dialect": "postgres", + "sql": "SELECT NUMRANGE(1.1, 2.2) -|- NUMRANGE(2.2, 3.3)", + "formatted": "SELECT NUMRANGE(1.1, 2.2) -|- NUMRANGE(2.2, 3.3)" + }, + { + "dialect": "postgres", + "sql": "SELECT SLOPE(point '(4,4)', point '(0,0)')", + "formatted": "SELECT SLOPE(point '(4,4)', point '(0,0)')" + }, + { + "dialect": "postgres", + "sql": "SELECT MLEAST(VARIADIC ARRAY[10, -1, 5, 4.4])", + "formatted": "SELECT MLEAST(VARIADIC ARRAY[10, -1, 5, 4.4])" + }, + { + "dialect": "postgres", + "sql": "SELECT MLEAST(VARIADIC ARRAY[]::numeric[])", + "formatted": null + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM schema_name.table_name st WHERE JSON_EXTRACT_PATH_TEXT((st.data)::json, variadic array['test'::text]) = 'test'::text", + "formatted": "SELECT *\nFROM schema_name.table_name st\nWHERE\n JSON_EXTRACT_PATH_TEXT(\n (\n st.data\n )::json,\n VARIADIC ARRAY['test'::text]\n ) = 'test'::text", + "previously_formatted": "SELECT\n *\nFROM schema_name.table_name AS st\nWHERE\n JSON_EXTRACT_PATH_TEXT(\n (\n st.data\n )::json,\n VARIADIC ARRAY['test'::text]\n ) = 'test'::text" + }, + { + "dialect": "postgres", + "sql": "WITH t(c) AS (SELECT 1) SELECT * INTO UNLOGGED foo FROM (SELECT c AS c FROM t) AS temp", + "formatted": "WITH t(c) AS (\n SELECT 1\n)\n\nSELECT *\nINTO UNLOGGED foo\nFROM (\n SELECT c AS c\n FROM t\n) AS temp" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM UNNEST(ARRAY[1, 2], ARRAY['foo', 'bar', 'baz']) AS x(a, b)", + "formatted": "SELECT *\nFROM UNNEST(ARRAY[1, 2], ARRAY['foo', 'bar', 'baz']) AS x(a, b)" + }, + { + "dialect": "postgres", + "sql": "SELECT TRIM(ARRAY_TO_STRING(ARRAY(SELECT val FROM UNNEST(ARRAY['a', 'b']) WITH ORDINALITY AS u(val, rn)), ' '))", + "formatted": "SELECT\n TRIM(ARRAY_TO_STRING(\n ARRAY(\n SELECT val\n FROM UNNEST(ARRAY['a', 'b']) WITH ORDINALITY AS u(val, rn)\n ),\n ' '\n ))" + }, + { + "dialect": "postgres", + "sql": "SELECT a, LOGICAL_OR(b) FROM table GROUP BY a", + "formatted": "SELECT\n a,\n LOGICAL_OR(b)\nFROM table\nGROUP BY\n a" + }, + { + "dialect": "postgres", + "sql": "SELECT CONCAT('abcde', 2, NULL, 22)", + "formatted": "SELECT CONCAT('abcde', 2, NULL, 22)" + }, + { + "dialect": "postgres", + "sql": "SELECT u & 5 FROM t", + "formatted": "SELECT u & 5\nFROM t" + }, + { + "dialect": "postgres", + "sql": "SELECT (U&'\\FE01' || 'Test literal') AS label FROM data", + "formatted": "SELECT\n (\n U&'\\FE01' || 'Test literal'\n ) AS label\nFROM data" + }, + { + "dialect": "postgres", + "sql": "SELECT U&'d!0061t!+000061' UESCAPE '!' AS label", + "formatted": "SELECT U&'d!0061t!+000061' UESCAPE '!' AS label" + }, + { + "dialect": "postgres", + "sql": "SELECT u&'\\0441\\043B\\043E\\043D'", + "formatted": "SELECT u&'\\0441\\043B\\043E\\043D'" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM ROWS FROM (FUNC1(col1, col2))", + "formatted": "SELECT *\nFROM ROWS FROM (FUNC1(col1, col2))" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM ROWS FROM (FUNC1(col1) AS alias1(\"col1\" TEXT), FUNC2(col2) AS alias2(\"col2\" INT)) WITH ORDINALITY", + "formatted": "SELECT *\nFROM ROWS FROM (FUNC1(col1) AS alias1(\"col1\" TEXT), FUNC2(col2) AS alias2(\"col2\" INT)) WITH ORDINALITY" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM table1, ROWS FROM (FUNC1(col1) AS alias1(\"col1\" TEXT)) WITH ORDINALITY AS alias3(\"col3\" INT, \"col4\" TEXT)", + "formatted": "SELECT *\nFROM table1, ROWS FROM (FUNC1(col1) AS alias1(\"col1\" TEXT)) WITH ORDINALITY AS alias3(\"col3\" INT, \"col4\" TEXT)" + }, + { + "dialect": "postgres", + "sql": "SELECT ARRAY_LENGTH(ARRAY[1, 2, 3], 1)", + "formatted": "SELECT ARRAY_LENGTH(ARRAY[1, 2, 3], 1)" + }, + { + "dialect": "postgres", + "sql": "SELECT XMLELEMENT(NAME foo)", + "formatted": "SELECT XMLELEMENT(NAME foo)" + }, + { + "dialect": "postgres", + "sql": "SELECT XMLELEMENT(NAME foo, XMLATTRIBUTES('xyz' AS bar))", + "formatted": "SELECT XMLELEMENT(NAME foo, XMLATTRIBUTES('xyz' AS bar))" + }, + { + "dialect": "postgres", + "sql": "SELECT XMLELEMENT(NAME test, XMLATTRIBUTES(a, b)) FROM test", + "formatted": "SELECT XMLELEMENT(NAME test, XMLATTRIBUTES(a, b))\nFROM test" + }, + { + "dialect": "postgres", + "sql": "SELECT XMLELEMENT(NAME foo, XMLATTRIBUTES(CURRENT_DATE AS bar), 'cont', 'ent')", + "formatted": "SELECT XMLELEMENT(NAME foo, XMLATTRIBUTES(CURRENT_DATE AS bar), 'cont', 'ent')" + }, + { + "dialect": "postgres", + "sql": "SELECT XMLELEMENT(NAME \"foo$bar\", XMLATTRIBUTES('xyz' AS \"a&b\"))", + "formatted": "SELECT XMLELEMENT(NAME \"foo$bar\", XMLATTRIBUTES('xyz' AS \"a&b\"))" + }, + { + "dialect": "postgres", + "sql": "SELECT XMLELEMENT(NAME foo, XMLATTRIBUTES('xyz' AS bar), XMLELEMENT(NAME abc), XMLCOMMENT('test'), XMLELEMENT(NAME xyz))", + "formatted": "SELECT XMLELEMENT(NAME foo, XMLATTRIBUTES('xyz' AS bar), XMLELEMENT(NAME abc), XMLCOMMENT('test'), XMLELEMENT(NAME xyz))" + }, + { + "dialect": "postgres", + "sql": "WITH RECURSIVE search_graph(id, link, data, depth) AS (SELECT g.id, g.link, g.data, 1 FROM graph AS g UNION ALL SELECT g.id, g.link, g.data, sg.depth + 1 FROM graph AS g, search_graph AS sg WHERE g.id = sg.link) CYCLE id SET is_cycle USING path SELECT * FROM search_graph", + "formatted": "WITH RECURSIVE search_graph(id, link, data, depth) AS (\n SELECT\n g.id,\n g.link,\n g.data,\n 1\n FROM graph AS g\n UNION ALL\n SELECT\n g.id,\n g.link,\n g.data,\n sg.depth + 1\n FROM graph AS g, search_graph AS sg\n WHERE g.id = sg.link\n)\n\nCYCLE id SET is_cycle USING path\nSELECT *\nFROM search_graph" + }, + { + "dialect": "postgres", + "sql": "SELECT a <<->> b", + "formatted": "SELECT a <<->> b" + }, + { + "dialect": "postgres", + "sql": "SELECT r IS NOT NULL FROM t", + "formatted": "SELECT r IS NOT NULL\nFROM t" + }, + { + "dialect": "postgres", + "sql": "SELECT NOT r IS NULL FROM t", + "formatted": "SELECT NOT r IS NULL\nFROM t" + }, + { + "dialect": "postgres", + "sql": "SELECT NOT r IS NOT NULL FROM t", + "formatted": "SELECT NOT r IS NOT NULL\nFROM t" + }, + { + "dialect": "postgres", + "sql": "SELECT r NOTNULL FROM t", + "formatted": "SELECT r NOTNULL\nFROM t" + }, + { + "dialect": "postgres", + "sql": "SELECT r ISNULL FROM t", + "formatted": "SELECT r ISNULL\nFROM t" + }, + { + "dialect": "postgres", + "sql": "SELECT ARRAY[1, 2, 3] <@ ARRAY[1, 2]", + "formatted": "SELECT ARRAY[1, 2, 3] <@ ARRAY[1, 2]" + }, + { + "dialect": "snowflake", + "sql": "WITH t AS (SELECT PARSE_JSON('{\"level1\": {\"level2\": {\"level3\": \"value\"}}}') AS data) SELECT data: level1 : level2 : level3::VARIANT FROM t", + "formatted": "WITH t AS (\n SELECT PARSE_JSON('{\"level1\": {\"level2\": {\"level3\": \"value\"}}}') AS data\n)\n\nSELECT data:level1:level2:level3::VARIANT\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM x ASOF JOIN y OFFSET MATCH_CONDITION (x.a > y.a)", + "formatted": "SELECT *\nFROM x\nASOF JOIN y OFFSET MATCH_CONDITION (x.a > y.a)", + "previously_formatted": "SELECT\n *\nFROM x\nASOF JOIN y AS OFFSET MATCH_CONDITION (x.a > y.a)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM x ASOF JOIN y LIMIT MATCH_CONDITION (x.a > y.a)", + "formatted": "SELECT *\nFROM x\nASOF JOIN y LIMIT MATCH_CONDITION (x.a > y.a)", + "previously_formatted": "SELECT\n *\nFROM x\nASOF JOIN y AS LIMIT MATCH_CONDITION (x.a > y.a)" + }, + { + "dialect": "snowflake", + "sql": "SELECT session", + "formatted": "SELECT session" + }, + { + "dialect": "snowflake", + "sql": "SELECT DATE_PART(EPOCH_MILLISECOND, CURRENT_TIMESTAMP()) AS a", + "formatted": "SELECT DATE_PART(EPOCH_MILLISECOND, CURRENT_TIMESTAMP()) AS a" + }, + { + "dialect": "snowflake", + "sql": "SELECT GET(a, b)", + "formatted": "SELECT GET(a, b)" + }, + { + "dialect": "snowflake", + "sql": "SELECT HASH_AGG(a, b, c, d)", + "formatted": "SELECT HASH_AGG(a, b, c, d)" + }, + { + "dialect": "snowflake", + "sql": "SELECT GREATEST(1, 2, 3, NULL)", + "formatted": "SELECT GREATEST(1, 2, 3, NULL)" + }, + { + "dialect": "snowflake", + "sql": "SELECT GREATEST_IGNORE_NULLS(1, 2, 3, NULL)", + "formatted": "SELECT GREATEST_IGNORE_NULLS(1, 2, 3, NULL)" + }, + { + "dialect": "snowflake", + "sql": "SELECT LEAST(5, NULL, 7, 3)", + "formatted": "SELECT LEAST(5, NULL, 7, 3)" + }, + { + "dialect": "snowflake", + "sql": "SELECT LEAST_IGNORE_NULLS(5, NULL, 7, 3)", + "formatted": "SELECT LEAST_IGNORE_NULLS(5, NULL, 7, 3)" + }, + { + "dialect": "snowflake", + "sql": "SELECT MAX(x)", + "formatted": "SELECT MAX(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT COUNT(x)", + "formatted": "SELECT COUNT(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT MIN(amount)", + "formatted": "SELECT MIN(amount)" + }, + { + "dialect": "snowflake", + "sql": "SELECT MODE(x)", + "formatted": "SELECT MODE(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT MODE(status) OVER (PARTITION BY region) FROM orders", + "formatted": "SELECT MODE(status) OVER (PARTITION BY region)\nFROM orders" + }, + { + "dialect": "snowflake", + "sql": "SELECT TAN(x)", + "formatted": "SELECT TAN(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT COS(x)", + "formatted": "SELECT COS(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT SINH(1.5)", + "formatted": "SELECT SINH(1.5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT MOD(x, y)", + "formatted": "SELECT MOD(x, y)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ROUND(x)", + "formatted": "SELECT ROUND(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ROUND(123.456, -1)", + "formatted": "SELECT ROUND(123.456, -1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ROUND(123.456, 2, 'HALF_AWAY_FROM_ZERO')", + "formatted": "SELECT ROUND(123.456, 2, 'HALF_AWAY_FROM_ZERO')" + }, + { + "dialect": "snowflake", + "sql": "SELECT FLOOR(x)", + "formatted": "SELECT FLOOR(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT FLOOR(135.135, 1)", + "formatted": "SELECT FLOOR(135.135, 1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT FLOOR(x, -1)", + "formatted": "SELECT FLOOR(x, -1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY salary) FROM employees", + "formatted": "SELECT PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY salary)\nFROM employees" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROX_TOP_K(col) FROM t", + "formatted": "SELECT APPROX_TOP_K(col)\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROX_TOP_K(category, 3) FROM t", + "formatted": "SELECT APPROX_TOP_K(category, 3)\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT MINHASH(5, col)", + "formatted": "SELECT MINHASH(5, col)" + }, + { + "dialect": "snowflake", + "sql": "SELECT MINHASH(5, col1, col2)", + "formatted": "SELECT MINHASH(5, col1, col2)" + }, + { + "dialect": "snowflake", + "sql": "SELECT MINHASH(5, *)", + "formatted": "SELECT MINHASH(5, *)" + }, + { + "dialect": "snowflake", + "sql": "SELECT MINHASH_COMBINE(minhash_col)", + "formatted": "SELECT MINHASH_COMBINE(minhash_col)" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROXIMATE_SIMILARITY(minhash_col)", + "formatted": "SELECT APPROXIMATE_SIMILARITY(minhash_col)" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROXIMATE_JACCARD_INDEX(minhash_col)", + "formatted": "SELECT APPROXIMATE_JACCARD_INDEX(minhash_col)" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROX_PERCENTILE_ACCUMULATE(col)", + "formatted": "SELECT APPROX_PERCENTILE_ACCUMULATE(col)" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROX_PERCENTILE_ESTIMATE(state, 0.5)", + "formatted": "SELECT APPROX_PERCENTILE_ESTIMATE(state, 0.5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROX_TOP_K_ACCUMULATE(col, 10)", + "formatted": "SELECT APPROX_TOP_K_ACCUMULATE(col, 10)" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROX_TOP_K_COMBINE(state, 2)", + "formatted": "SELECT APPROX_TOP_K_COMBINE(state, 2)" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROX_TOP_K_COMBINE(state)", + "formatted": "SELECT APPROX_TOP_K_COMBINE(state)" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROX_TOP_K_ESTIMATE(state_column, 4)", + "formatted": "SELECT APPROX_TOP_K_ESTIMATE(state_column, 4)" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROX_TOP_K_ESTIMATE(state_column)", + "formatted": "SELECT APPROX_TOP_K_ESTIMATE(state_column)" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROX_PERCENTILE_COMBINE(state_column)", + "formatted": "SELECT APPROX_PERCENTILE_COMBINE(state_column)" + }, + { + "dialect": "snowflake", + "sql": "SELECT EQUAL_NULL(1, 2)", + "formatted": "SELECT EQUAL_NULL(1, 2)" + }, + { + "dialect": "snowflake", + "sql": "SELECT EXP(1)", + "formatted": "SELECT EXP(1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT FACTORIAL(5)", + "formatted": "SELECT FACTORIAL(5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BIT_LENGTH('abc')", + "formatted": "SELECT BIT_LENGTH('abc')" + }, + { + "dialect": "snowflake", + "sql": "SELECT BIT_LENGTH(x'A1B2')", + "formatted": "SELECT BIT_LENGTH(x'A1B2')" + }, + { + "dialect": "snowflake", + "sql": "SELECT BITMAP_BUCKET_NUMBER(32769)", + "formatted": "SELECT BITMAP_BUCKET_NUMBER(32769)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BITMAP_CONSTRUCT_AGG(value)", + "formatted": "SELECT BITMAP_CONSTRUCT_AGG(value)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BITMAP_COUNT(BITMAP_CONSTRUCT_AGG(value)) FROM TABLE(FLATTEN(INPUT => ARRAY_CONSTRUCT(1, 2, 3, 5)))", + "formatted": "SELECT BITMAP_COUNT(BITMAP_CONSTRUCT_AGG(value))\nFROM TABLE(FLATTEN(INPUT => ARRAY_CONSTRUCT(1, 2, 3, 5)))" + }, + { + "dialect": "snowflake", + "sql": "SELECT BOOLAND(1, -2)", + "formatted": "SELECT BOOLAND(1, -2)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BOOLXOR(2, 0)", + "formatted": "SELECT BOOLXOR(2, 0)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BOOLOR(1, 0)", + "formatted": "SELECT BOOLOR(1, 0)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_BOOLEAN('true')", + "formatted": "SELECT TO_BOOLEAN('true')" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_BOOLEAN(1)", + "formatted": "SELECT TO_BOOLEAN(1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_VARIANT(123)", + "formatted": "SELECT TO_VARIANT(123)" + }, + { + "dialect": "snowflake", + "sql": "SELECT IS_NULL_VALUE(GET_PATH(payload, 'field'))", + "formatted": "SELECT IS_NULL_VALUE(GET_PATH(payload, 'field'))" + }, + { + "dialect": "snowflake", + "sql": "SELECT RTRIMMED_LENGTH(' ABCD ')", + "formatted": "SELECT RTRIMMED_LENGTH(' ABCD ')" + }, + { + "dialect": "snowflake", + "sql": "SELECT HEX_DECODE_STRING('48656C6C6F')", + "formatted": "SELECT HEX_DECODE_STRING('48656C6C6F')" + }, + { + "dialect": "snowflake", + "sql": "SELECT HEX_ENCODE('Hello World')", + "formatted": "SELECT HEX_ENCODE('Hello World')" + }, + { + "dialect": "snowflake", + "sql": "SELECT HEX_ENCODE('Hello World', 1)", + "formatted": "SELECT HEX_ENCODE('Hello World', 1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT HEX_ENCODE('Hello World', 0)", + "formatted": "SELECT HEX_ENCODE('Hello World', 0)" + }, + { + "dialect": "snowflake", + "sql": "SELECT IFNULL(col1, col2)", + "formatted": "SELECT IFNULL(col1, col2)" + }, + { + "dialect": "snowflake", + "sql": "SELECT NEXT_DAY('2025-10-15', 'FRIDAY')", + "formatted": "SELECT NEXT_DAY('2025-10-15', 'FRIDAY')" + }, + { + "dialect": "snowflake", + "sql": "SELECT NVL2(col1, col2, col3)", + "formatted": "SELECT NVL2(col1, col2, col3)" + }, + { + "dialect": "snowflake", + "sql": "SELECT NVL(col1, col2)", + "formatted": "SELECT NVL(col1, col2)" + }, + { + "dialect": "snowflake", + "sql": "SELECT CHR(8364)", + "formatted": "SELECT CHR(8364)" + }, + { + "dialect": "snowflake", + "sql": "SELECT CHECK_JSON('{\"key\": \"value\"}')", + "formatted": "SELECT CHECK_JSON('{\"key\": \"value\"}')" + }, + { + "dialect": "snowflake", + "sql": "SELECT CHECK_XML('value')", + "formatted": "SELECT CHECK_XML('value')" + }, + { + "dialect": "snowflake", + "sql": "SELECT CHECK_XML('value', TRUE)", + "formatted": "SELECT CHECK_XML('value', TRUE)" + }, + { + "dialect": "snowflake", + "sql": "SELECT COMPRESS('Hello World', 'ZLIB')", + "formatted": "SELECT COMPRESS('Hello World', 'ZLIB')" + }, + { + "dialect": "snowflake", + "sql": "SELECT DECOMPRESS_BINARY('compressed_data', 'SNAPPY')", + "formatted": "SELECT DECOMPRESS_BINARY('compressed_data', 'SNAPPY')" + }, + { + "dialect": "snowflake", + "sql": "SELECT DECOMPRESS_STRING('compressed_data', 'ZSTD')", + "formatted": "SELECT DECOMPRESS_STRING('compressed_data', 'ZSTD')" + }, + { + "dialect": "snowflake", + "sql": "SELECT LPAD('Hello', 10, '*')", + "formatted": "SELECT LPAD('Hello', 10, '*')" + }, + { + "dialect": "snowflake", + "sql": "SELECT LPAD(tbl.bin_col, 10)", + "formatted": "SELECT LPAD(tbl.bin_col, 10)" + }, + { + "dialect": "snowflake", + "sql": "SELECT RPAD('Hello', 10, '*')", + "formatted": "SELECT RPAD('Hello', 10, '*')" + }, + { + "dialect": "snowflake", + "sql": "SELECT RPAD(tbl.bin_col, 10)", + "formatted": "SELECT RPAD(tbl.bin_col, 10)" + }, + { + "dialect": "snowflake", + "sql": "SELECT RPAD(TO_BINARY('Hi', 'UTF8'), 10, TO_BINARY('_', 'UTF8'))", + "formatted": "SELECT\n RPAD(\n TO_BINARY('Hi', 'UTF8'),\n 10,\n TO_BINARY('_', 'UTF8')\n )" + }, + { + "dialect": "snowflake", + "sql": "SELECT SOUNDEX(column_name)", + "formatted": "SELECT SOUNDEX(column_name)" + }, + { + "dialect": "snowflake", + "sql": "SELECT SOUNDEX_P123(column_name)", + "formatted": "SELECT SOUNDEX_P123(column_name)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ABS(x)", + "formatted": "SELECT ABS(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ASIN(0.5)", + "formatted": "SELECT ASIN(0.5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ASINH(0.5)", + "formatted": "SELECT ASINH(0.5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ATAN(0.5)", + "formatted": "SELECT ATAN(0.5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ATAN2(0.5, 0.3)", + "formatted": "SELECT ATAN2(0.5, 0.3)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ATANH(0.5)", + "formatted": "SELECT ATANH(0.5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT CBRT(27.0)", + "formatted": "SELECT CBRT(27.0)" + }, + { + "dialect": "snowflake", + "sql": "SELECT POW(2, 3)", + "formatted": "SELECT POW(2, 3)" + }, + { + "dialect": "snowflake", + "sql": "SELECT POW(2.5, 3.0)", + "formatted": "SELECT POW(2.5, 3.0)" + }, + { + "dialect": "snowflake", + "sql": "SELECT SQUARE(2.5)", + "formatted": "SELECT SQUARE(2.5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT SIGN(x)", + "formatted": "SELECT SIGN(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT COSH(1.5)", + "formatted": "SELECT COSH(1.5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TANH(0.5)", + "formatted": "SELECT TANH(0.5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TRANSLATE(column_name, 'abc', '123')", + "formatted": "SELECT TRANSLATE(column_name, 'abc', '123')" + }, + { + "dialect": "snowflake", + "sql": "SELECT UNICODE(column_name)", + "formatted": "SELECT UNICODE(column_name)" + }, + { + "dialect": "snowflake", + "sql": "SELECT WIDTH_BUCKET(col, 0, 100, 10)", + "formatted": "SELECT WIDTH_BUCKET(col, 0, 100, 10)" + }, + { + "dialect": "snowflake", + "sql": "SELECT PI()", + "formatted": "SELECT PI()" + }, + { + "dialect": "snowflake", + "sql": "SELECT DEGREES(PI() / 3)", + "formatted": "SELECT DEGREES(PI() / 3)" + }, + { + "dialect": "snowflake", + "sql": "SELECT DEGREES(1)", + "formatted": "SELECT DEGREES(1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT RADIANS(180)", + "formatted": "SELECT RADIANS(180)" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGR_AVGX(y, x)", + "formatted": "SELECT REGR_AVGX(y, x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGR_AVGY(y, x)", + "formatted": "SELECT REGR_AVGY(y, x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGR_COUNT(y, x)", + "formatted": "SELECT REGR_COUNT(y, x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGR_INTERCEPT(y, x)", + "formatted": "SELECT REGR_INTERCEPT(y, x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGR_R2(y, x)", + "formatted": "SELECT REGR_R2(y, x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGR_SXX(y, x)", + "formatted": "SELECT REGR_SXX(y, x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGR_SXY(y, x)", + "formatted": "SELECT REGR_SXY(y, x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGR_SYY(y, x)", + "formatted": "SELECT REGR_SYY(y, x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGR_SLOPE(y, x)", + "formatted": "SELECT REGR_SLOPE(y, x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT RANDSTR(123, 456)", + "formatted": "SELECT RANDSTR(123, 456)" + }, + { + "dialect": "snowflake", + "sql": "SELECT RANDSTR(123, RANDOM())", + "formatted": "SELECT RANDSTR(123, RANDOM())" + }, + { + "dialect": "snowflake", + "sql": "SELECT NORMAL(0, 1, RANDOM())", + "formatted": "SELECT NORMAL(0, 1, RANDOM())" + }, + { + "dialect": "snowflake", + "sql": "SELECT GROUPING_ID(a, b) AS g_id FROM x GROUP BY ROLLUP (a, b)", + "formatted": "SELECT GROUPING_ID(a, b) AS g_id\nFROM x\nGROUP BY\n ROLLUP (\n a,\n b\n )" + }, + { + "dialect": "snowflake", + "sql": "SELECT XMLGET(object_col, 'level2')", + "formatted": "SELECT XMLGET(object_col, 'level2')" + }, + { + "dialect": "snowflake", + "sql": "SELECT XMLGET(object_col, 'level3', 1)", + "formatted": "SELECT XMLGET(object_col, 'level3', 1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT {*} FROM my_table", + "formatted": "SELECT {*}\nFROM my_table" + }, + { + "dialect": "snowflake", + "sql": "SELECT {my_table.*} FROM my_table", + "formatted": "SELECT {my_table.*}\nFROM my_table" + }, + { + "dialect": "snowflake", + "sql": "SELECT {* ILIKE 'col1%'} FROM my_table", + "formatted": "SELECT {* ILIKE 'col1%'}\nFROM my_table" + }, + { + "dialect": "snowflake", + "sql": "SELECT {* EXCLUDE (col1)} FROM my_table", + "formatted": "SELECT {* EXCLUDE (col1)}\nFROM my_table" + }, + { + "dialect": "snowflake", + "sql": "SELECT {* EXCLUDE (col1, col2)} FROM my_table", + "formatted": "SELECT {* EXCLUDE (col1, col2)}\nFROM my_table" + }, + { + "dialect": "snowflake", + "sql": "SELECT a, b, COUNT(*) FROM x GROUP BY ALL LIMIT 100", + "formatted": "SELECT\n a,\n b,\n COUNT(*)\nFROM x\nGROUP BY ALL\nLIMIT 100" + }, + { + "dialect": "snowflake", + "sql": "SELECT STAR(tbl, exclude := [foo])", + "formatted": "SELECT STAR(tbl, exclude := [foo])" + }, + { + "dialect": "snowflake", + "sql": "SELECT CAST([1, 2, 3] AS VECTOR(FLOAT, 3))", + "formatted": "SELECT CAST([1, 2, 3] AS VECTOR(FLOAT, 3))" + }, + { + "dialect": "snowflake", + "sql": "SELECT VECTOR_COSINE_SIMILARITY(a, b)", + "formatted": "SELECT VECTOR_COSINE_SIMILARITY(a, b)" + }, + { + "dialect": "snowflake", + "sql": "SELECT VECTOR_INNER_PRODUCT(a, b)", + "formatted": "SELECT VECTOR_INNER_PRODUCT(a, b)" + }, + { + "dialect": "snowflake", + "sql": "SELECT VECTOR_L1_DISTANCE(a, b)", + "formatted": "SELECT VECTOR_L1_DISTANCE(a, b)" + }, + { + "dialect": "snowflake", + "sql": "SELECT VECTOR_L2_DISTANCE(a, b)", + "formatted": "SELECT VECTOR_L2_DISTANCE(a, b)" + }, + { + "dialect": "snowflake", + "sql": "SELECT CONNECT_BY_ROOT test AS test_column_alias", + "formatted": "SELECT CONNECT_BY_ROOT test AS test_column_alias" + }, + { + "dialect": "snowflake", + "sql": "SELECT rename, replace", + "formatted": "SELECT\n rename,\n replace" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMEADD(HOUR, 2, CAST('09:05:03' AS TIME))", + "formatted": "SELECT TIMEADD(HOUR, 2, CAST('09:05:03' AS TIME))" + }, + { + "dialect": "snowflake", + "sql": "SELECT CAST(OBJECT_CONSTRUCT('a', 1) AS MAP(VARCHAR, INT))", + "formatted": "SELECT CAST(OBJECT_CONSTRUCT('a', 1) AS MAP(VARCHAR, INT))" + }, + { + "dialect": "snowflake", + "sql": "SELECT MAP_CAT(CAST(col AS MAP(VARCHAR, VARCHAR)), CAST(col AS MAP(VARCHAR, VARCHAR)))", + "formatted": "SELECT MAP_CAT(CAST(col AS MAP(VARCHAR, VARCHAR)), CAST(col AS MAP(VARCHAR, VARCHAR)))" + }, + { + "dialect": "snowflake", + "sql": "SELECT MAP_CONTAINS_KEY('k1', CAST(col AS MAP(VARCHAR, VARCHAR)))", + "formatted": "SELECT MAP_CONTAINS_KEY('k1', CAST(col AS MAP(VARCHAR, VARCHAR)))" + }, + { + "dialect": "snowflake", + "sql": "SELECT MAP_DELETE(CAST(col AS MAP(VARCHAR, VARCHAR)), 'k1')", + "formatted": "SELECT MAP_DELETE(CAST(col AS MAP(VARCHAR, VARCHAR)), 'k1')" + }, + { + "dialect": "snowflake", + "sql": "SELECT MAP_INSERT(CAST(col AS MAP(VARCHAR, VARCHAR)), 'b', '2')", + "formatted": "SELECT MAP_INSERT(CAST(col AS MAP(VARCHAR, VARCHAR)), 'b', '2')" + }, + { + "dialect": "snowflake", + "sql": "SELECT MAP_KEYS(CAST(col AS MAP(VARCHAR, VARCHAR)))", + "formatted": "SELECT MAP_KEYS(CAST(col AS MAP(VARCHAR, VARCHAR)))" + }, + { + "dialect": "snowflake", + "sql": "SELECT MAP_PICK(CAST(col AS MAP(VARCHAR, VARCHAR)), 'a', 'c')", + "formatted": "SELECT MAP_PICK(CAST(col AS MAP(VARCHAR, VARCHAR)), 'a', 'c')" + }, + { + "dialect": "snowflake", + "sql": "SELECT MAP_SIZE(CAST(col AS MAP(VARCHAR, VARCHAR)))", + "formatted": "SELECT MAP_SIZE(CAST(col AS MAP(VARCHAR, VARCHAR)))" + }, + { + "dialect": "snowflake", + "sql": "SELECT CAST(OBJECT_CONSTRUCT('a', 1) AS OBJECT(a CHAR NOT NULL))", + "formatted": "SELECT CAST(OBJECT_CONSTRUCT('a', 1) AS OBJECT(a CHAR NOT NULL))" + }, + { + "dialect": "snowflake", + "sql": "SELECT CAST([1, 2, 3] AS ARRAY(INT))", + "formatted": "SELECT CAST([1, 2, 3] AS ARRAY(INT))" + }, + { + "dialect": "snowflake", + "sql": "SELECT CAST(obj AS OBJECT(x CHAR) RENAME FIELDS)", + "formatted": "SELECT CAST(obj AS OBJECT(x CHAR) RENAME FIELDS)" + }, + { + "dialect": "snowflake", + "sql": "SELECT CAST(obj AS OBJECT(x CHAR, y VARCHAR) ADD FIELDS)", + "formatted": "SELECT CAST(obj AS OBJECT(x CHAR, y VARCHAR) ADD FIELDS)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_TIMESTAMP(x) FROM t", + "formatted": "SELECT TO_TIMESTAMP(x)\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_TIMESTAMP_NTZ(x) FROM t", + "formatted": "SELECT TO_TIMESTAMP_NTZ(x)\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_TIMESTAMP_LTZ(x) FROM t", + "formatted": "SELECT TO_TIMESTAMP_LTZ(x)\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_TIMESTAMP_TZ(x) FROM t", + "formatted": "SELECT TO_TIMESTAMP_TZ(x)\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMESTAMP_FROM_PARTS(2024, 5, 9, 14, 30, 45)", + "formatted": "SELECT TIMESTAMP_FROM_PARTS(2024, 5, 9, 14, 30, 45)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMESTAMP_FROM_PARTS(2024, 5, 9, 14, 30, 45, 123)", + "formatted": "SELECT TIMESTAMP_FROM_PARTS(2024, 5, 9, 14, 30, 45, 123)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMESTAMP_LTZ_FROM_PARTS(2013, 4, 5, 12, 00, 00)", + "formatted": "SELECT TIMESTAMP_LTZ_FROM_PARTS(2013, 4, 5, 12, 00, 00)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMESTAMP_TZ_FROM_PARTS(2013, 4, 5, 12, 00, 00)", + "formatted": "SELECT TIMESTAMP_TZ_FROM_PARTS(2013, 4, 5, 12, 00, 00)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMESTAMP_TZ_FROM_PARTS(2013, 4, 5, 12, 00, 00, 0, 'America/Los_Angeles')", + "formatted": "SELECT TIMESTAMP_TZ_FROM_PARTS(2013, 4, 5, 12, 00, 00, 0, 'America/Los_Angeles')" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMESTAMP_FROM_PARTS(CAST('2024-05-09' AS DATE), CAST('14:30:45' AS TIME))", + "formatted": "SELECT\n TIMESTAMP_FROM_PARTS(\n CAST('2024-05-09' AS DATE),\n CAST('14:30:45' AS TIME)\n )" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMESTAMP_NTZ_FROM_PARTS(TO_DATE('2013-04-05'), TO_TIME('12:00:00'))", + "formatted": "SELECT TIMESTAMP_NTZ_FROM_PARTS(TO_DATE('2013-04-05'), TO_TIME('12:00:00'))" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMESTAMP_NTZ_FROM_PARTS(2013, 4, 5, 12, 00, 00, 987654321)", + "formatted": "SELECT TIMESTAMP_NTZ_FROM_PARTS(2013, 4, 5, 12, 00, 00, 987654321)" + }, + { + "dialect": "snowflake", + "sql": "SELECT DATE_FROM_PARTS(1977, 8, 7)", + "formatted": "SELECT DATE_FROM_PARTS(1977, 8, 7)" + }, + { + "dialect": "snowflake", + "sql": "SELECT GET_PATH(v, 'attr[0].name') FROM vartab", + "formatted": "SELECT GET_PATH(v, 'attr[0].name')\nFROM vartab" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_ARRAY(CAST(x AS ARRAY))", + "formatted": "SELECT TO_ARRAY(CAST(x AS ARRAY))" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_ARRAY(CAST(['test'] AS VARIANT))", + "formatted": "SELECT TO_ARRAY(CAST(['test'] AS VARIANT))" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAY_UNIQUE_AGG(x)", + "formatted": "SELECT ARRAY_UNIQUE_AGG(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAY_APPEND([1, 2, 3], 4)", + "formatted": "SELECT ARRAY_APPEND([1, 2, 3], 4)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAY_CAT([1, 2], [3, 4])", + "formatted": "SELECT ARRAY_CAT([1, 2], [3, 4])" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAY_PREPEND([2, 3, 4], 1)", + "formatted": "SELECT ARRAY_PREPEND([2, 3, 4], 1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAY_REMOVE([1, 2, 3], 2)", + "formatted": "SELECT ARRAY_REMOVE([1, 2, 3], 2)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAYS_ZIP([1, 2, 3])", + "formatted": "SELECT ARRAYS_ZIP([1, 2, 3])" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAYS_ZIP([1, 2, 3], ['a', 'b', 'c'], [10, 20, 30])", + "formatted": "SELECT\n ARRAYS_ZIP(\n [1, 2, 3],\n ['a', 'b', 'c'],\n [10, 20, 30]\n )" + }, + { + "dialect": "snowflake", + "sql": "SELECT AI_AGG(review, 'Summarize the reviews')", + "formatted": "SELECT AI_AGG(review, 'Summarize the reviews')" + }, + { + "dialect": "snowflake", + "sql": "SELECT AI_SUMMARIZE_AGG(review)", + "formatted": "SELECT AI_SUMMARIZE_AGG(review)" + }, + { + "dialect": "snowflake", + "sql": "SELECT AI_CLASSIFY('text', ['travel', 'cooking'])", + "formatted": "SELECT AI_CLASSIFY('text', ['travel', 'cooking'])" + }, + { + "dialect": "snowflake", + "sql": "SELECT OBJECT_CONSTRUCT()", + "formatted": "SELECT OBJECT_CONSTRUCT()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_ACCOUNT()", + "formatted": "SELECT CURRENT_ACCOUNT()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_ACCOUNT_NAME()", + "formatted": "SELECT CURRENT_ACCOUNT_NAME()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_AVAILABLE_ROLES()", + "formatted": "SELECT CURRENT_AVAILABLE_ROLES()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_CLIENT()", + "formatted": "SELECT CURRENT_CLIENT()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_IP_ADDRESS()", + "formatted": "SELECT CURRENT_IP_ADDRESS()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_DATABASE()", + "formatted": "SELECT CURRENT_DATABASE()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_SCHEMAS()", + "formatted": "SELECT CURRENT_SCHEMAS()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_SECONDARY_ROLES()", + "formatted": "SELECT CURRENT_SECONDARY_ROLES()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_SESSION()", + "formatted": "SELECT CURRENT_SESSION()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_STATEMENT()", + "formatted": "SELECT CURRENT_STATEMENT()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_VERSION()", + "formatted": "SELECT CURRENT_VERSION()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_TRANSACTION()", + "formatted": "SELECT CURRENT_TRANSACTION()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_WAREHOUSE()", + "formatted": "SELECT CURRENT_WAREHOUSE()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_ORGANIZATION_USER()", + "formatted": "SELECT CURRENT_ORGANIZATION_USER()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_REGION()", + "formatted": "SELECT CURRENT_REGION()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_ROLE()", + "formatted": "SELECT CURRENT_ROLE()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_ROLE_TYPE()", + "formatted": "SELECT CURRENT_ROLE_TYPE()" + }, + { + "dialect": "snowflake", + "sql": "SELECT DAY(CURRENT_TIMESTAMP())", + "formatted": "SELECT DAY(CURRENT_TIMESTAMP())" + }, + { + "dialect": "snowflake", + "sql": "SELECT DAYOFMONTH(CURRENT_TIMESTAMP())", + "formatted": "SELECT DAYOFMONTH(CURRENT_TIMESTAMP())" + }, + { + "dialect": "snowflake", + "sql": "SELECT DAYOFYEAR(CURRENT_TIMESTAMP())", + "formatted": "SELECT DAYOFYEAR(CURRENT_TIMESTAMP())" + }, + { + "dialect": "snowflake", + "sql": "SELECT MONTH(CURRENT_TIMESTAMP())", + "formatted": "SELECT MONTH(CURRENT_TIMESTAMP())" + }, + { + "dialect": "snowflake", + "sql": "SELECT QUARTER(CURRENT_TIMESTAMP())", + "formatted": "SELECT QUARTER(CURRENT_TIMESTAMP())" + }, + { + "dialect": "snowflake", + "sql": "SELECT WEEK(CURRENT_TIMESTAMP())", + "formatted": "SELECT WEEK(CURRENT_TIMESTAMP())" + }, + { + "dialect": "snowflake", + "sql": "SELECT WEEKISO(CURRENT_TIMESTAMP())", + "formatted": "SELECT WEEKISO(CURRENT_TIMESTAMP())" + }, + { + "dialect": "snowflake", + "sql": "SELECT YEAR(CURRENT_TIMESTAMP())", + "formatted": "SELECT YEAR(CURRENT_TIMESTAMP())" + }, + { + "dialect": "snowflake", + "sql": "SELECT YEAROFWEEK(CURRENT_TIMESTAMP())", + "formatted": "SELECT YEAROFWEEK(CURRENT_TIMESTAMP())" + }, + { + "dialect": "snowflake", + "sql": "SELECT YEAROFWEEKISO(CURRENT_TIMESTAMP())", + "formatted": "SELECT YEAROFWEEKISO(CURRENT_TIMESTAMP())" + }, + { + "dialect": "snowflake", + "sql": "SELECT SUM(amount) FROM mytable GROUP BY ALL", + "formatted": "SELECT SUM(amount)\nFROM mytable\nGROUP BY ALL" + }, + { + "dialect": "snowflake", + "sql": "SELECT STDDEV(x)", + "formatted": "SELECT STDDEV(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT STDDEV(x) OVER (PARTITION BY 1)", + "formatted": "SELECT STDDEV(x) OVER (PARTITION BY 1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT STDDEV_POP(x)", + "formatted": "SELECT STDDEV_POP(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT STDDEV_POP(x) OVER (PARTITION BY 1)", + "formatted": "SELECT STDDEV_POP(x) OVER (PARTITION BY 1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT STDDEV_SAMP(x)", + "formatted": "SELECT STDDEV_SAMP(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT STDDEV_SAMP(x) OVER (PARTITION BY 1)", + "formatted": "SELECT STDDEV_SAMP(x) OVER (PARTITION BY 1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT KURTOSIS(x)", + "formatted": "SELECT KURTOSIS(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT KURTOSIS(x) OVER (PARTITION BY 1)", + "formatted": "SELECT KURTOSIS(x) OVER (PARTITION BY 1)" + }, + { + "dialect": "snowflake", + "sql": "WITH x AS (SELECT 1 AS foo) SELECT foo FROM IDENTIFIER('x')", + "formatted": "WITH x AS (\n SELECT 1 AS foo\n)\n\nSELECT foo\nFROM IDENTIFIER('x')" + }, + { + "dialect": "snowflake", + "sql": "WITH x AS (SELECT 1 AS foo) SELECT IDENTIFIER('foo') FROM x", + "formatted": "WITH x AS (\n SELECT 1 AS foo\n)\n\nSELECT IDENTIFIER('foo')\nFROM x" + }, + { + "dialect": "snowflake", + "sql": "SELECT IDENTIFIER($my_function_name)()", + "formatted": "SELECT IDENTIFIER($my_function_name)()" + }, + { + "dialect": "snowflake", + "sql": "SELECT IDENTIFIER('speed_of_light')()", + "formatted": "SELECT IDENTIFIER('speed_of_light')()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CAST('2021-01-01' AS DATE) + INTERVAL '1 DAY'", + "formatted": "SELECT CAST('2021-01-01' AS DATE) + INTERVAL '1 DAY'" + }, + { + "dialect": "snowflake", + "sql": "SELECT HLL(*)", + "formatted": "SELECT HLL(*)" + }, + { + "dialect": "snowflake", + "sql": "SELECT HLL(a)", + "formatted": "SELECT HLL(a)" + }, + { + "dialect": "snowflake", + "sql": "SELECT HLL(DISTINCT t.a)", + "formatted": "SELECT HLL(DISTINCT t.a)" + }, + { + "dialect": "snowflake", + "sql": "SELECT HLL(a, b, c)", + "formatted": "SELECT HLL(a, b, c)" + }, + { + "dialect": "snowflake", + "sql": "SELECT HLL(DISTINCT a, b, c)", + "formatted": "SELECT HLL(DISTINCT a, b, c)" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGEXP_LIKE(a, b, c)", + "formatted": "SELECT REGEXP_LIKE(a, b, c)" + }, + { + "dialect": "snowflake", + "sql": "SELECT CONVERT_TIMEZONE('UTC', 'America/Los_Angeles', col)", + "formatted": "SELECT CONVERT_TIMEZONE('UTC', 'America/Los_Angeles', col)" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_ORGANIZATION_NAME()", + "formatted": "SELECT CURRENT_ORGANIZATION_NAME()" + }, + { + "dialect": "snowflake", + "sql": "SELECT MATCH_CONDITION", + "formatted": "SELECT MATCH_CONDITION" + }, + { + "dialect": "snowflake", + "sql": "SELECT OBJECT_AGG(key, value) FROM tbl", + "formatted": "SELECT OBJECT_AGG(key, value)\nFROM tbl" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_CHAR(CAST('12:05:05' AS TIME))", + "formatted": "SELECT TO_CHAR(CAST('12:05:05' AS TIME))" + }, + { + "dialect": "snowflake", + "sql": "SELECT TRIM(COALESCE(TO_CHAR(CAST(c AS TIME)), '')) FROM t", + "formatted": "SELECT TRIM(COALESCE(TO_CHAR(CAST(c AS TIME)), ''))\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT GET_PATH(PARSE_JSON(foo), 'bar')", + "formatted": "SELECT GET_PATH(PARSE_JSON(foo), 'bar')" + }, + { + "dialect": "snowflake", + "sql": "SELECT PARSE_IP('192.168.1.1', 'INET')", + "formatted": "SELECT PARSE_IP('192.168.1.1', 'INET')" + }, + { + "dialect": "snowflake", + "sql": "SELECT PARSE_IP('192.168.1.1', 'INET', 0)", + "formatted": "SELECT PARSE_IP('192.168.1.1', 'INET', 0)" + }, + { + "dialect": "snowflake", + "sql": "SELECT GET_PATH(foo, 'bar')", + "formatted": "SELECT GET_PATH(foo, 'bar')" + }, + { + "dialect": "snowflake", + "sql": "SELECT a, exclude, b FROM xxx", + "formatted": "SELECT\n a,\n exclude,\n b\nFROM xxx" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAY_SORT(x, TRUE, FALSE)", + "formatted": "SELECT ARRAY_SORT(x, TRUE, FALSE)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BOOLXOR_AGG(col) FROM tbl", + "formatted": "SELECT BOOLXOR_AGG(col)\nFROM tbl" + }, + { + "dialect": "snowflake", + "sql": "SELECT PERCENTILE_DISC(0.9) WITHIN GROUP (ORDER BY col) OVER (PARTITION BY category)", + "formatted": "SELECT PERCENTILE_DISC(0.9) WITHIN GROUP (ORDER BY col) OVER (PARTITION BY category)" + }, + { + "dialect": "snowflake", + "sql": "SELECT AI_CLASSIFY('text', ['travel', 'cooking'], OBJECT_CONSTRUCT('output_mode', 'multi'))", + "formatted": "SELECT\n AI_CLASSIFY(\n 'text',\n ['travel', 'cooking'],\n OBJECT_CONSTRUCT('output_mode', 'multi')\n )" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM table AT (TIMESTAMP => '2024-07-24') UNPIVOT(a FOR b IN (c)) AS pivot_table", + "formatted": "SELECT *\nFROM table AT (TIMESTAMP => '2024-07-24') UNPIVOT(a FOR b IN (c)) AS pivot_table" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN ('2023_Q1', '2023_Q2', '2023_Q3', '2023_Q4', '2024_Q1') DEFAULT ON NULL (0)) ORDER BY empid", + "formatted": "SELECT *\nFROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN ('2023_Q1', '2023_Q2', '2023_Q3', '2023_Q4', '2024_Q1') DEFAULT ON NULL (0))\nORDER BY\n empid" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN (SELECT DISTINCT quarter FROM ad_campaign_types_by_quarter WHERE television = TRUE ORDER BY quarter)) ORDER BY empid", + "formatted": "SELECT *\nFROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN (SELECT DISTINCT\n quarter\nFROM ad_campaign_types_by_quarter\nWHERE television = TRUE\nORDER BY\n quarter))\nORDER BY\n empid" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN (ANY ORDER BY quarter)) ORDER BY empid", + "formatted": "SELECT *\nFROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN (ANY ORDER BY quarter))\nORDER BY\n empid" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN (ANY)) ORDER BY empid", + "formatted": "SELECT *\nFROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN (ANY))\nORDER BY\n empid" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM DATA AS DATA_L ASOF JOIN DATA AS DATA_R MATCH_CONDITION (DATA_L.VAL > DATA_R.VAL) ON DATA_L.ID = DATA_R.ID", + "formatted": "SELECT *\nFROM DATA AS DATA_L\nASOF JOIN DATA AS DATA_R MATCH_CONDITION (DATA_L.VAL > DATA_R.VAL)\n ON DATA_L.ID = DATA_R.ID" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_TIMESTAMP('2025-01-16T14:45:30.123+0500', 'yyyy-mm-DDThh24:mi:ss.ff9tzhtzm')", + "formatted": "SELECT TO_TIMESTAMP('2025-01-16T14:45:30.123+0500', 'yyyy-mm-DDThh24:mi:ss.ff9tzhtzm')" + }, + { + "dialect": "snowflake", + "sql": "SELECT * REPLACE (CAST(col AS TEXT) AS scol) FROM t", + "formatted": "SELECT * REPLACE (CAST(col AS TEXT) AS scol)\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT 1 put", + "formatted": "SELECT 1 put", + "previously_formatted": "SELECT\n 1 AS put" + }, + { + "dialect": "snowflake", + "sql": "SELECT 1 get", + "formatted": "SELECT 1 get", + "previously_formatted": "SELECT\n 1 AS get" + }, + { + "dialect": "snowflake", + "sql": "WITH t (SELECT 1 AS c) SELECT c FROM t", + "formatted": "WITH t (\n SELECT 1 AS c\n)\nSELECT c\nFROM t", + "previously_formatted": "WITH t AS (\n SELECT\n 1 AS c\n)\nSELECT\n c\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM s WHERE c NOT IN (1, 2, 3)", + "formatted": "SELECT *\nFROM s\nWHERE c NOT IN (1, 2, 3)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM s WHERE c NOT IN (SELECT * FROM t)", + "formatted": "SELECT *\nFROM s\nWHERE\n c NOT IN (\n SELECT *\n FROM t\n )" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM t1 INNER JOIN t2 USING (t1.col)", + "formatted": "SELECT *\nFROM t1\nINNER JOIN t2\n USING (t1.col)" + }, + { + "dialect": "snowflake", + "sql": "SELECT a:from::STRING, a:from || ' test' ", + "formatted": "SELECT\n a:from::STRING,\n a:from || ' test'" + }, + { + "dialect": "snowflake", + "sql": "SELECT a:select", + "formatted": "SELECT a:select" + }, + { + "dialect": "snowflake", + "sql": "SELECT GET_PATH(PARSE_JSON('{\"y\": [{\"z\": 1}]}'), 'y[0]:z')", + "formatted": "SELECT GET_PATH(PARSE_JSON('{\"y\": [{\"z\": 1}]}'), 'y[0]:z')" + }, + { + "dialect": "snowflake", + "sql": "SELECT p FROM t WHERE p:val NOT IN ('2')", + "formatted": "SELECT p\nFROM t\nWHERE p:val NOT IN ('2')" + }, + { + "dialect": "snowflake", + "sql": "SELECT PARSE_JSON('{\"x\": \"hello\"}'):x LIKE 'hello'", + "formatted": "SELECT PARSE_JSON('{\"x\": \"hello\"}'):x LIKE 'hello'" + }, + { + "dialect": "snowflake", + "sql": "SELECT data:x LIKE 'hello' FROM some_table", + "formatted": "SELECT data:x LIKE 'hello'\nFROM some_table" + }, + { + "dialect": "snowflake", + "sql": "SELECT SUM({ fn CONVERT(123, SQL_DOUBLE) })", + "formatted": "SELECT SUM({ fn CONVERT(123, SQL_DOUBLE) })" + }, + { + "dialect": "snowflake", + "sql": "SELECT SUM({ fn CONVERT(123, SQL_VARCHAR) })", + "formatted": "SELECT SUM({ fn CONVERT(123, SQL_VARCHAR) })" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMESTAMPFROMPARTS(d, t)", + "formatted": "SELECT TIMESTAMPFROMPARTS(d, t)" + }, + { + "dialect": "snowflake", + "sql": "SELECT v:attr[0].name FROM vartab", + "formatted": "SELECT v:attr[0].name\nFROM vartab" + }, + { + "dialect": "snowflake", + "sql": "SELECT v:\"fruit\" FROM vartab", + "formatted": "SELECT v:\"fruit\"\nFROM vartab" + }, + { + "dialect": "snowflake", + "sql": "SELECT PARSE_JSON('{\"food\":{\"fruit\":\"banana\"}}'):food.fruit::VARCHAR", + "formatted": "SELECT PARSE_JSON('{\"food\":{\"fruit\":\"banana\"}}'):food.fruit::VARCHAR" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM t, UNNEST(x) WITH ORDINALITY", + "formatted": "SELECT *\nFROM t, UNNEST(x) WITH ORDINALITY" + }, + { + "dialect": "snowflake", + "sql": "SELECT state, city, SUM(retail_price * quantity) AS gross_revenue FROM sales GROUP BY ALL", + "formatted": "SELECT\n state,\n city,\n SUM(retail_price * quantity) AS gross_revenue\nFROM sales\nGROUP BY ALL" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM foo window", + "formatted": "SELECT *\nFROM foo window", + "previously_formatted": "SELECT\n *\nFROM foo AS window" + }, + { + "dialect": "snowflake", + "sql": "SELECT RLIKE(a, $$regular expression with \\ characters: \\d{2}-\\d{3}-\\d{4}$$, 'i') FROM log_source", + "formatted": "SELECT RLIKE(a, $$regular expression with \\ characters: \\d{2}-\\d{3}-\\d{4}$$, 'i')\nFROM log_source" + }, + { + "dialect": "snowflake", + "sql": "SELECT $$a ' \\ \\t \\x21 z $ $$", + "formatted": "SELECT $$a ' \\ \\t \\x21 z $ $$" + }, + { + "dialect": "snowflake", + "sql": "SELECT $$a\\$b$$", + "formatted": "SELECT $$a\\$b$$" + }, + { + "dialect": "snowflake", + "sql": "SELECT $$a\\$$", + "formatted": "SELECT $$a\\$$" + }, + { + "dialect": "snowflake", + "sql": "SELECT {'test': 'best'}::VARIANT", + "formatted": "SELECT {'test': 'best'}::VARIANT" + }, + { + "dialect": "snowflake", + "sql": "SELECT {fn DAYNAME('2022-5-13')}", + "formatted": "SELECT {fn\n DAYNAME('2022-5-13') }" + }, + { + "dialect": "snowflake", + "sql": "SELECT {fn LOG(5)}", + "formatted": "SELECT {fn\n LOG(5) }" + }, + { + "dialect": "snowflake", + "sql": "SELECT {fn CEILING(5.3)}", + "formatted": "SELECT {fn\n CEILING(5.3) }" + }, + { + "dialect": "snowflake", + "sql": "SELECT CEIL(3.14)", + "formatted": "SELECT CEIL(3.14)" + }, + { + "dialect": "snowflake", + "sql": "SELECT CEIL(3.14, 1)", + "formatted": "SELECT CEIL(3.14, 1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT DAYOFWEEK('2016-01-02T23:39:20.123-07:00'::TIMESTAMP)", + "formatted": "SELECT DAYOFWEEK('2016-01-02T23:39:20.123-07:00'::TIMESTAMP)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM xxx WHERE col ilike '%Don''t%'", + "formatted": "SELECT *\nFROM xxx\nWHERE col ILIKE '%Don''t%'" + }, + { + "dialect": "snowflake", + "sql": "SELECT * EXCLUDE a, b FROM xxx", + "formatted": "SELECT\n * EXCLUDE a,\n b\nFROM xxx" + }, + { + "dialect": "snowflake", + "sql": "SELECT * RENAME a AS b, c AS d FROM xxx", + "formatted": "SELECT\n * RENAME a AS b,\n c AS d\nFROM xxx" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM xxx, yyy, zzz,", + "formatted": "SELECT *\nFROM xxx, yyy, zzz," + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM xxx, yyy, zzz, WHERE foo = bar", + "formatted": "SELECT *\nFROM xxx, yyy, zzz,\nWHERE foo = bar" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM xxx, yyy, zzz", + "formatted": "SELECT *\nFROM xxx, yyy, zzz" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMESTAMPNTZFROMPARTS(2013, 4, 5, 12, 00, 00)", + "formatted": "SELECT TIMESTAMPNTZFROMPARTS(2013, 4, 5, 12, 00, 00)" + }, + { + "dialect": "snowflake", + "sql": "SELECT NTH_VALUE(a, 2) FROM t", + "formatted": "SELECT NTH_VALUE(a, 2)\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAY_CONSTRUCT('foo')::VARIANT[0]", + "formatted": "SELECT ARRAY_CONSTRUCT('foo')::VARIANT[0]" + }, + { + "dialect": "snowflake", + "sql": "SELECT UUID_STRING(), UUID_STRING('fe971b24-9572-4005-b22f-351e9c09274d', 'foo')", + "formatted": "SELECT\n UUID_STRING(),\n UUID_STRING('fe971b24-9572-4005-b22f-351e9c09274d', 'foo')" + }, + { + "dialect": "snowflake", + "sql": "SELECT BITNOT(a)", + "formatted": "SELECT BITNOT(a)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BIT_NOT(a)", + "formatted": "SELECT BIT_NOT(a)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BITAND(a, b)", + "formatted": "SELECT BITAND(a, b)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BITAND(a, b, 'LEFT')", + "formatted": "SELECT BITAND(a, b, 'LEFT')" + }, + { + "dialect": "snowflake", + "sql": "SELECT BIT_AND(a, b)", + "formatted": "SELECT BIT_AND(a, b)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BIT_AND(a, b, 'LEFT')", + "formatted": "SELECT BIT_AND(a, b, 'LEFT')" + }, + { + "dialect": "snowflake", + "sql": "SELECT BITOR(a, b)", + "formatted": "SELECT BITOR(a, b)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BITOR(a, b, 'LEFT')", + "formatted": "SELECT BITOR(a, b, 'LEFT')" + }, + { + "dialect": "snowflake", + "sql": "SELECT BIT_OR(a, b)", + "formatted": "SELECT BIT_OR(a, b)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BIT_OR(a, b, 'RIGHT')", + "formatted": "SELECT BIT_OR(a, b, 'RIGHT')" + }, + { + "dialect": "snowflake", + "sql": "SELECT BITXOR(a, b)", + "formatted": "SELECT BITXOR(a, b)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BITXOR(a, b, 'LEFT')", + "formatted": "SELECT BITXOR(a, b, 'LEFT')" + }, + { + "dialect": "snowflake", + "sql": "SELECT BIT_XOR(a, b)", + "formatted": "SELECT BIT_XOR(a, b)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BIT_XOR(a, b, 'LEFT')", + "formatted": "SELECT BIT_XOR(a, b, 'LEFT')" + }, + { + "dialect": "snowflake", + "sql": "SELECT HOUR(CAST('08:50:57' AS TIME))", + "formatted": "SELECT HOUR(CAST('08:50:57' AS TIME))" + }, + { + "dialect": "snowflake", + "sql": "SELECT MINUTE(CAST('08:50:57' AS TIME))", + "formatted": "SELECT MINUTE(CAST('08:50:57' AS TIME))" + }, + { + "dialect": "snowflake", + "sql": "SELECT SECOND(CAST('08:50:57' AS TIME))", + "formatted": "SELECT SECOND(CAST('08:50:57' AS TIME))" + }, + { + "dialect": "snowflake", + "sql": "SELECT HOUR(CAST('2024-05-09 08:50:57' AS TIMESTAMP))", + "formatted": "SELECT HOUR(CAST('2024-05-09 08:50:57' AS TIMESTAMP))" + }, + { + "dialect": "snowflake", + "sql": "SELECT MONTHNAME(CAST('2024-05-09' AS DATE))", + "formatted": "SELECT MONTHNAME(CAST('2024-05-09' AS DATE))" + }, + { + "dialect": "snowflake", + "sql": "SELECT PREVIOUS_DAY(CAST('2024-05-09' AS DATE), 'MONDAY')", + "formatted": "SELECT PREVIOUS_DAY(CAST('2024-05-09' AS DATE), 'MONDAY')" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIME_FROM_PARTS(14, 30, 45)", + "formatted": "SELECT TIME_FROM_PARTS(14, 30, 45)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIME_FROM_PARTS(14, 30, 45, 123)", + "formatted": "SELECT TIME_FROM_PARTS(14, 30, 45, 123)" + }, + { + "dialect": "snowflake", + "sql": "SELECT MONTHS_BETWEEN(CAST('2019-03-15' AS DATE), CAST('2019-02-15' AS DATE))", + "formatted": "SELECT MONTHS_BETWEEN(CAST('2019-03-15' AS DATE), CAST('2019-02-15' AS DATE))" + }, + { + "dialect": "snowflake", + "sql": "SELECT MONTHS_BETWEEN(CAST('2019-03-01 02:00:00' AS TIMESTAMP), CAST('2019-02-15 01:00:00' AS TIMESTAMP))", + "formatted": "SELECT MONTHS_BETWEEN(CAST('2019-03-01 02:00:00' AS TIMESTAMP), CAST('2019-02-15 01:00:00' AS TIMESTAMP))" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIME_SLICE(CAST('2024-05-09 08:50:57.891' AS TIMESTAMP), 15, 'MINUTE')", + "formatted": "SELECT TIME_SLICE(CAST('2024-05-09 08:50:57.891' AS TIMESTAMP), 15, 'MINUTE')" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIME_SLICE(CAST('2024-05-09' AS DATE), 1, 'DAY')", + "formatted": "SELECT TIME_SLICE(CAST('2024-05-09' AS DATE), 1, 'DAY')" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIME_SLICE(CAST('2024-05-09 08:50:57.891' AS TIMESTAMP), 1, 'HOUR', 'start')", + "formatted": "SELECT\n TIME_SLICE(\n CAST('2024-05-09 08:50:57.891' AS TIMESTAMP),\n 1,\n 'HOUR',\n 'start'\n )" + }, + { + "dialect": "snowflake", + "sql": "SELECT * EXCLUDE foo RENAME bar AS baz FROM tbl", + "formatted": "SELECT * EXCLUDE foo RENAME bar AS baz\nFROM tbl" + }, + { + "dialect": "snowflake", + "sql": "SELECT LIKE(col, 'pattern')", + "formatted": "SELECT LIKE(col, 'pattern')" + }, + { + "dialect": "snowflake", + "sql": "SELECT ILIKE(col, 'pattern')", + "formatted": "SELECT ILIKE(col, 'pattern')" + }, + { + "dialect": "snowflake", + "sql": "SELECT LIKE(col, 'pattern', '\\\\')", + "formatted": "SELECT LIKE(col, 'pattern', '\\\\')" + }, + { + "dialect": "snowflake", + "sql": "SELECT ILIKE(col, 'pattern', '\\\\')", + "formatted": "SELECT ILIKE(col, 'pattern', '\\\\')" + }, + { + "dialect": "snowflake", + "sql": "SELECT LIKE(col, 'pattern', '!')", + "formatted": "SELECT LIKE(col, 'pattern', '!')" + }, + { + "dialect": "snowflake", + "sql": "SELECT ILIKE(col, 'pattern', '!')", + "formatted": "SELECT ILIKE(col, 'pattern', '!')" + }, + { + "dialect": "snowflake", + "sql": "SELECT BASE64_ENCODE('Hello World')", + "formatted": "SELECT BASE64_ENCODE('Hello World')" + }, + { + "dialect": "snowflake", + "sql": "SELECT TRY_HEX_DECODE_BINARY('48656C6C6F')", + "formatted": "SELECT TRY_HEX_DECODE_BINARY('48656C6C6F')" + }, + { + "dialect": "snowflake", + "sql": "SELECT TRY_HEX_DECODE_STRING('48656C6C6F')", + "formatted": "SELECT TRY_HEX_DECODE_STRING('48656C6C6F')" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAY_CONTAINS(1, [1])", + "formatted": "SELECT ARRAY_CONTAINS(1, [1])" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAY_DISTINCT(['A', 'B', 'A'])", + "formatted": "SELECT ARRAY_DISTINCT(['A', 'B', 'A'])" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_DATABASE()", + "formatted": "SELECT CURRENT_DATABASE()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_SCHEMA()", + "formatted": "SELECT CURRENT_SCHEMA()" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @\"mystage\"", + "formatted": "SELECT *\nFROM @\"mystage\"" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @\"myschema\".\"mystage\"/file.gz", + "formatted": "SELECT *\nFROM @\"myschema\".\"mystage\"/file.gz" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @\"my_DB\".\"schEMA1\".mystage/file.gz", + "formatted": "SELECT *\nFROM @\"my_DB\".\"schEMA1\".mystage/file.gz" + }, + { + "dialect": "snowflake", + "sql": "SELECT metadata$filename FROM @s1/", + "formatted": "SELECT metadata$filename\nFROM @s1/" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @~", + "formatted": "SELECT *\nFROM @~" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @~/some/path/to/file.csv", + "formatted": "SELECT *\nFROM @~/some/path/to/file.csv" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @mystage", + "formatted": "SELECT *\nFROM @mystage" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM '@mystage'", + "formatted": "SELECT *\nFROM '@mystage'" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @namespace.mystage/path/to/file.json.gz", + "formatted": "SELECT *\nFROM @namespace.mystage/path/to/file.json.gz" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @namespace.%table_name/path/to/file.json.gz", + "formatted": "SELECT *\nFROM @namespace.%table_name/path/to/file.json.gz" + }, + { + "dialect": "snowflake", + "sql": "SELECT $1, $2, metadata$filename FROM @mystage (PATTERN => '.*data-100.*')", + "formatted": "SELECT\n $1,\n $2,\n metadata$filename\nFROM @mystage (PATTERN => '.*data-100.*')" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM '@external/location' (FILE_FORMAT => 'path.to.csv')", + "formatted": null + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM (SELECT a FROM @foo)", + "formatted": "SELECT *\nFROM (\n SELECT a\n FROM @foo\n)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM (SELECT * FROM '@external/location' (FILE_FORMAT => 'path.to.csv'))", + "formatted": null + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @foo/bar (FILE_FORMAT => ds_sandbox.test.my_csv_format, PATTERN => 'test') AS bla", + "formatted": "SELECT *\nFROM @foo/bar (FILE_FORMAT => ds_sandbox.test.my_csv_format, PATTERN => 'test') AS bla" + }, + { + "dialect": "snowflake", + "sql": "SELECT t.$1, t.$2 FROM @mystage1 (FILE_FORMAT => 'myformat', PATTERN => '.*data.*[.]csv.gz') AS t", + "formatted": "SELECT\n t.$1,\n t.$2\nFROM @mystage1 (FILE_FORMAT => 'myformat', PATTERN => '.*data.*[.]csv.gz') AS t" + }, + { + "dialect": "snowflake", + "sql": "SELECT parse_json($1):a.b FROM @mystage2/data1.json.gz", + "formatted": "SELECT parse_json($1):a.b\nFROM @mystage2/data1.json.gz" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @mystage t (c1)", + "formatted": "SELECT *\nFROM @mystage t(c1)", + "previously_formatted": "SELECT\n *\nFROM @mystage AS t(c1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @foo/bar (PATTERN => 'test', FILE_FORMAT => ds_sandbox.test.my_csv_format) AS bla", + "formatted": "SELECT *\nFROM @foo/bar (PATTERN => 'test', FILE_FORMAT => ds_sandbox.test.my_csv_format) AS bla" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @test.public.thing/location/somefile.csv( FILE_FORMAT => 'fmt' )", + "formatted": "SELECT *\nFROM @test.public.thing/location/somefile.csv (FILE_FORMAT => 'fmt')" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM testtable TABLESAMPLE BERNOULLI (20.3)", + "formatted": "SELECT *\nFROM testtable TABLESAMPLE BERNOULLI (20.3)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM testtable TABLESAMPLE SYSTEM (3) SEED (82)", + "formatted": "SELECT *\nFROM testtable TABLESAMPLE SYSTEM (3) SEED (82)" + }, + { + "dialect": "snowflake", + "sql": "SELECT a FROM test PIVOT(SUM(x) FOR y IN ('z', 'q')) AS x TABLESAMPLE BERNOULLI (0.1)", + "formatted": "SELECT a\nFROM test PIVOT(SUM(x) FOR y IN ('z', 'q')) AS x TABLESAMPLE BERNOULLI (0.1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT i, j FROM table1 AS t1 INNER JOIN table2 AS t2 TABLESAMPLE BERNOULLI (50) WHERE t2.j = t1.i", + "formatted": "SELECT\n i,\n j\nFROM table1 AS t1\nINNER JOIN table2 AS t2 TABLESAMPLE BERNOULLI (50)\nWHERE t2.j = t1.i" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM (SELECT * FROM t1 JOIN t2 ON t1.a = t2.c) TABLESAMPLE BERNOULLI (1)", + "formatted": "SELECT *\nFROM (\n SELECT *\n FROM t1\n JOIN t2\n ON t1.a = t2.c\n) TABLESAMPLE BERNOULLI (1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM testtable TABLESAMPLE (10 ROWS)", + "formatted": "SELECT *\nFROM testtable TABLESAMPLE (10 ROWS)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM testtable TABLESAMPLE (100)", + "formatted": "SELECT *\nFROM testtable TABLESAMPLE (100)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM testtable SAMPLE (10)", + "formatted": "SELECT *\nFROM testtable SAMPLE (10)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM testtable SAMPLE ROW (0)", + "formatted": "SELECT *\nFROM testtable SAMPLE ROW (0)" + }, + { + "dialect": "snowflake", + "sql": "SELECT a FROM test SAMPLE BLOCK (0.5) SEED (42)", + "formatted": "SELECT a\nFROM test SAMPLE BLOCK (0.5) SEED (42)" + }, + { + "dialect": "snowflake", + "sql": "SELECT user_id, value FROM table_name SAMPLE BERNOULLI ($s) SEED (0)", + "formatted": "SELECT\n user_id,\n value\nFROM table_name SAMPLE BERNOULLI ($s) SEED (0)" + }, + { + "dialect": "snowflake", + "sql": "SELECT CAST('12:00:00' AS TIME)", + "formatted": "SELECT CAST('12:00:00' AS TIME)" + }, + { + "dialect": "snowflake", + "sql": "SELECT DATE_PART(month, a)", + "formatted": "SELECT DATE_PART(month, a)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_TIME(x) FROM t", + "formatted": "SELECT TO_TIME(x)\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_DATE('2019-02-28') + INTERVAL '1 day, 1 year'", + "formatted": "SELECT TO_DATE('2019-02-28') + INTERVAL '1 day, 1 year'" + }, + { + "dialect": "snowflake", + "sql": "SELECT CAST(a AS VARIANT)", + "formatted": "SELECT CAST(a AS VARIANT)" + }, + { + "dialect": "snowflake", + "sql": "SELECT CAST(a AS ARRAY)", + "formatted": "SELECT CAST(a AS ARRAY)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM my_table AT (STATEMENT => $query_id_var)", + "formatted": "SELECT *\nFROM my_table AT (STATEMENT => $query_id_var)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM my_table AT (OFFSET => -60 * 5)", + "formatted": "SELECT *\nFROM my_table AT (OFFSET => -60 * 5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM my_table BEFORE (STATEMENT => $query_id_var)", + "formatted": "SELECT *\nFROM my_table BEFORE (STATEMENT => $query_id_var)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM my_table BEFORE (OFFSET => -60 * 5)", + "formatted": "SELECT *\nFROM my_table BEFORE (OFFSET => -60 * 5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM my_table AT (TIMESTAMP => TO_TIMESTAMP(1432669154242, 3))", + "formatted": "SELECT *\nFROM my_table AT (TIMESTAMP => TO_TIMESTAMP(1432669154242, 3))" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM my_table AT (OFFSET => -60 * 5) AS T WHERE T.flag = 'valid'", + "formatted": "SELECT *\nFROM my_table AT (OFFSET => -60 * 5) AS T\nWHERE T.flag = 'valid'" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM my_table AT (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726')", + "formatted": "SELECT *\nFROM my_table AT (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726')" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM my_table BEFORE (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726')", + "formatted": "SELECT *\nFROM my_table BEFORE (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726')" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM my_table AT (TIMESTAMP => 'Fri, 01 May 2015 16:20:00 -0700'::timestamp)", + "formatted": "SELECT *\nFROM my_table AT (TIMESTAMP => 'Fri, 01 May 2015 16:20:00 -0700'::timestamp)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM my_table AT(TIMESTAMP => 'Fri, 01 May 2015 16:20:00 -0700'::timestamp_tz)", + "formatted": "SELECT *\nFROM my_table AT (TIMESTAMP => 'Fri, 01 May 2015 16:20:00 -0700'::timestamp_tz)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM my_table BEFORE (TIMESTAMP => 'Fri, 01 May 2015 16:20:00 -0700'::timestamp_tz);", + "formatted": "SELECT *\nFROM my_table BEFORE (TIMESTAMP => 'Fri, 01 May 2015 16:20:00 -0700'::timestamp_tz);", + "previously_formatted": "SELECT\n *\nFROM my_table BEFORE (TIMESTAMP => 'Fri, 01 May 2015 16:20:00 -0700'::timestamp_tz)" + }, + { + "dialect": "snowflake", + "sql": "\n SELECT oldt.* , newt.*\n FROM my_table BEFORE(STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726') AS oldt\n FULL OUTER JOIN my_table AT(STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726') AS newt\n ON oldt.id = newt.id\n WHERE oldt.id IS NULL OR newt.id IS NULL;\n ", + "formatted": "SELECT\n oldt.*,\n newt.*\nFROM my_table BEFORE (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726') AS oldt\nFULL OUTER JOIN my_table AT (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726') AS newt\n ON oldt.id = newt.id\nWHERE\n oldt.id IS NULL\n OR newt.id IS NULL;", + "previously_formatted": "SELECT\n oldt.*,\n newt.*\nFROM my_table BEFORE (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726') AS oldt\nFULL OUTER JOIN my_table AT (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726') AS newt\n ON oldt.id = newt.id\nWHERE\n oldt.id IS NULL\n OR newt.id IS NULL" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE('MYTABLE')", + "formatted": "SELECT *\nFROM TABLE('MYTABLE')" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE($MYVAR)", + "formatted": "SELECT *\nFROM TABLE($MYVAR)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE(?)", + "formatted": "SELECT *\nFROM TABLE(?)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE(:BINDING)", + "formatted": null + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE($MYVAR) WHERE COL1 = 10", + "formatted": "SELECT *\nFROM TABLE($MYVAR)\nWHERE COL1 = 10" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE('t1') AS f", + "formatted": "SELECT *\nFROM TABLE('t1') AS f" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM (TABLE('t1') CROSS JOIN TABLE('t2'))", + "formatted": "SELECT *\nFROM (TABLE('t1')\nCROSS JOIN TABLE('t2'))" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE('t1'), LATERAL (SELECT * FROM t2)", + "formatted": "SELECT *\nFROM TABLE('t1'), LATERAL (\n SELECT *\n FROM t2\n)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE('t1') UNION ALL SELECT * FROM TABLE('t2')", + "formatted": "SELECT *\nFROM TABLE('t1')\nUNION ALL\nSELECT *\nFROM TABLE('t2')" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE('t1') TABLESAMPLE BERNOULLI (20.3)", + "formatted": "SELECT *\nFROM TABLE('t1') TABLESAMPLE BERNOULLI (20.3)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE('MYDB.\"MYSCHEMA\".\"MYTABLE\"')", + "formatted": "SELECT *\nFROM TABLE('MYDB.\"MYSCHEMA\".\"MYTABLE\"')" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE($$MYDB. \"MYSCHEMA\".\"MYTABLE\"$$)", + "formatted": "SELECT *\nFROM TABLE($$MYDB. \"MYSCHEMA\".\"MYTABLE\"$$)" + }, + { + "dialect": "snowflake", + "sql": "SELECT value FROM TABLE(FLATTEN(input => SELECT PARSE_JSON('[1, 2]')))", + "formatted": "SELECT value\nFROM TABLE(FLATTEN(input => SELECT\n PARSE_JSON('[1, 2]')))" + }, + { + "dialect": "snowflake", + "sql": "SELECT SEARCH((play, line), 'dream')", + "formatted": "SELECT SEARCH((play, line), 'dream')" + }, + { + "dialect": "snowflake", + "sql": "SELECT SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER')", + "formatted": "SELECT SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER')" + }, + { + "dialect": "snowflake", + "sql": "SELECT SEARCH(character, 'king queen', SEARCH_MODE => 'AND')", + "formatted": "SELECT SEARCH(character, 'king queen', SEARCH_MODE => 'AND')" + }, + { + "dialect": "snowflake", + "sql": "SELECT SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER', SEARCH_MODE => 'OR')", + "formatted": "SELECT\n SEARCH(\n line,\n 'king',\n ANALYZER => 'UNICODE_ANALYZER',\n SEARCH_MODE => 'OR'\n )" + }, + { + "dialect": "snowflake", + "sql": "SELECT SEARCH(line, 'king')", + "formatted": "SELECT SEARCH(line, 'king')" + }, + { + "dialect": "snowflake", + "sql": "SELECT SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER')", + "formatted": "SELECT SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER')" + }, + { + "dialect": "snowflake", + "sql": "SELECT SEARCH(character, 'king queen', SEARCH_MODE => 'AND')", + "formatted": "SELECT SEARCH(character, 'king queen', SEARCH_MODE => 'AND')" + }, + { + "dialect": "snowflake", + "sql": "SELECT SEARCH(line, 'king', SEARCH_MODE => 'AND', ANALYZER => 'PATTERN_ANALYZER')", + "formatted": "SELECT\n SEARCH(\n line,\n 'king',\n SEARCH_MODE => 'AND',\n ANALYZER => 'PATTERN_ANALYZER'\n )" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGEXP_COUNT('hello world', 'l ')", + "formatted": "SELECT REGEXP_COUNT('hello world', 'l ')" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGEXP_COUNT('hello world', 'l', 1)", + "formatted": "SELECT REGEXP_COUNT('hello world', 'l', 1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGEXP_COUNT('hello world', 'l', 1, 'i')", + "formatted": "SELECT REGEXP_COUNT('hello world', 'l', 1, 'i')" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGEXP_INSTR('abc', 'a')", + "formatted": "SELECT REGEXP_INSTR('abc', 'a')" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGEXP_INSTR('abc', 'a', 1, 1, 0, 'i')", + "formatted": "SELECT REGEXP_INSTR('abc', 'a', 1, 1, 0, 'i')" + }, + { + "dialect": "snowflake", + "sql": "SELECT TRY_CAST(x AS DOUBLE)", + "formatted": "SELECT TRY_CAST(x AS DOUBLE)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TRY_CAST(FOO() AS TEXT)", + "formatted": "SELECT TRY_CAST(FOO() AS TEXT)" + }, + { + "dialect": "snowflake", + "sql": "SELECT $1", + "formatted": "SELECT $1" + }, + { + "dialect": "snowflake", + "sql": "SELECT $1.elem", + "formatted": "SELECT $1.elem" + }, + { + "dialect": "snowflake", + "sql": "SELECT $1:a.b", + "formatted": "SELECT $1:a.b" + }, + { + "dialect": "snowflake", + "sql": "SELECT t.$23:a.b", + "formatted": "SELECT t.$23:a.b" + }, + { + "dialect": "snowflake", + "sql": "SELECT t.$17:a[0].b[0].c", + "formatted": "SELECT t.$17:a[0].b[0].c" + }, + { + "dialect": "snowflake", + "sql": "SELECT C1 FROM t1 CHANGES (INFORMATION => APPEND_ONLY) AT (STREAM => 's1') END (TIMESTAMP => $ts2)", + "formatted": "SELECT C1\nFROM t1 CHANGES (INFORMATION => APPEND_ONLY) AT (STREAM => 's1') END (TIMESTAMP => $ts2)" + }, + { + "dialect": "snowflake", + "sql": "SELECT C1 FROM t1 CHANGES (INFORMATION => APPEND_ONLY) BEFORE (STATEMENT => 'STMT_ID') END (TIMESTAMP => $ts2)", + "formatted": "SELECT C1\nFROM t1 CHANGES (INFORMATION => APPEND_ONLY) BEFORE (STATEMENT => 'STMT_ID') END (TIMESTAMP => $ts2)" + }, + { + "dialect": "snowflake", + "sql": "SELECT 1 FROM some_table CHANGES (INFORMATION => APPEND_ONLY) AT (TIMESTAMP => TO_TIMESTAMP_TZ('2024-07-01 00:00:00+00:00')) END (TIMESTAMP => TO_TIMESTAMP_TZ('2024-07-01 14:28:59.999999+00:00'))", + "formatted": "SELECT 1\nFROM some_table CHANGES (INFORMATION => APPEND_ONLY) AT (TIMESTAMP => TO_TIMESTAMP_TZ('2024-07-01 00:00:00+00:00')) END (TIMESTAMP => TO_TIMESTAMP_TZ('2024-07-01 14:28:59.999999+00:00'))" + }, + { + "dialect": "snowflake", + "sql": "SELECT :1", + "formatted": null + }, + { + "dialect": "snowflake", + "sql": "SELECT :1, :2", + "formatted": null + }, + { + "dialect": "snowflake", + "sql": "SELECT :1 + :2", + "formatted": null + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM SEMANTIC_VIEW(foo METRICS a.b, a.c DIMENSIONS a.b, a.c WHERE a.b > '1995-01-01')", + "formatted": "SELECT *\nFROM SEMANTIC_VIEW(\n foo\n METRICS a.b, a.c\n DIMENSIONS a.b, a.c\n WHERE a.b > '1995-01-01'\n)" + }, + { + "dialect": "snowflake", + "sql": "SELECT col1, col2, metric1 FROM SEMANTIC_VIEW(mydb.myschema.my_semantic_view METRICS metric1 DIMENSIONS col1, DATE_TRUNC('MONTH', timestamp_col) AS col2) ORDER BY col1, col2 DESC", + "formatted": "SELECT\n col1,\n col2,\n metric1\nFROM SEMANTIC_VIEW(\n mydb.myschema.my_semantic_view\n METRICS metric1\n DIMENSIONS col1, DATE_TRUNC('MONTH', timestamp_col) AS col2\n)\nORDER BY\n col1,\n col2 DESC" + }, + { + "dialect": "snowflake", + "sql": "SELECT model!mladmin", + "formatted": "SELECT model!mladmin" + }, + { + "dialect": "snowflake", + "sql": "SELECT model!PREDICT(1)", + "formatted": "SELECT model!PREDICT(1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT m!PREDICT(INPUT_DATA => {*}) AS p FROM tbl", + "formatted": "SELECT m!PREDICT(INPUT_DATA => {*}) AS p\nFROM tbl" + }, + { + "dialect": "snowflake", + "sql": "SELECT m!PREDICT(INPUT_DATA => {tbl.*}) AS p FROM tbl", + "formatted": "SELECT m!PREDICT(INPUT_DATA => {tbl.*}) AS p\nFROM tbl" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE(model_trained_with_labeled_data!DETECT_ANOMALIES(INPUT_DATA => TABLE(view_with_data_to_analyze), TIMESTAMP_COLNAME => 'date', TARGET_COLNAME => 'sales', CONFIG_OBJECT => OBJECT_CONSTRUCT('prediction_interval', 0.99)))", + "formatted": "SELECT *\nFROM TABLE(model_trained_with_labeled_data!DETECT_ANOMALIES(\n INPUT_DATA => TABLE(view_with_data_to_analyze),\n TIMESTAMP_COLNAME => 'date',\n TARGET_COLNAME => 'sales',\n CONFIG_OBJECT => OBJECT_CONSTRUCT('prediction_interval', 0.99)\n))" + }, + { + "dialect": "snowflake", + "sql": "SELECT 1 FROM TABLE(GENERATOR(ROWCOUNT => 10))", + "formatted": "SELECT 1\nFROM TABLE(GENERATOR(ROWCOUNT => 10))" + }, + { + "dialect": "snowflake", + "sql": "SELECT 1 FROM TABLE(GENERATOR(TIMELIMIT => 5))", + "formatted": "SELECT 1\nFROM TABLE(GENERATOR(TIMELIMIT => 5))" + }, + { + "dialect": "snowflake", + "sql": "SELECT 1 FROM TABLE(GENERATOR(ROWCOUNT => 10, TIMELIMIT => 5))", + "formatted": "SELECT 1\nFROM TABLE(GENERATOR(ROWCOUNT => 10, TIMELIMIT => 5))" + }, + { + "dialect": "snowflake", + "sql": "SELECT 1 FROM TABLE(GENERATOR(10))", + "formatted": "SELECT 1\nFROM TABLE(GENERATOR(10))" + }, + { + "dialect": "snowflake", + "sql": "SELECT 1 FROM TABLE(GENERATOR(10, 5))", + "formatted": "SELECT 1\nFROM TABLE(GENERATOR(10, 5))" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM a CROSS DIRECTED JOIN b USING (id)", + "formatted": "SELECT *\nFROM a\nCROSS DIRECTED JOIN b\n USING (id)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM a INNER DIRECTED JOIN b USING (id)", + "formatted": "SELECT *\nFROM a\nINNER DIRECTED JOIN b\n USING (id)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM a NATURAL INNER DIRECTED JOIN b USING (id)", + "formatted": "SELECT *\nFROM a\nNATURAL INNER DIRECTED JOIN b\n USING (id)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TRY_PARSE_JSON('{\"x: 1}')", + "formatted": "SELECT TRY_PARSE_JSON('{\"x: 1}')" + }, + { + "dialect": "snowflake", + "sql": "SELECT number", + "formatted": "SELECT number" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_TIMESTAMP(123.4)", + "formatted": "SELECT TO_TIMESTAMP(123.4)" + }, + { + "dialect": "snowflake", + "sql": "SELECT DATE_PART(year FROM CAST('2024-04-08' AS DATE))", + "formatted": "SELECT DATE_PART(year FROM CAST('2024-04-08' AS DATE))" + }, + { + "dialect": "snowflake", + "sql": "SELECT DATE_PART('month' FROM CAST('2024-04-08' AS DATE))", + "formatted": "SELECT DATE_PART('month' FROM CAST('2024-04-08' AS DATE))" + }, + { + "dialect": "snowflake", + "sql": "SELECT DATE_PART(day FROM a)", + "formatted": "SELECT DATE_PART(day FROM a)" + }, + { + "dialect": "snowflake", + "sql": "SELECT SEARCH_IP(col, '192.168.0.0')", + "formatted": "SELECT SEARCH_IP(col, '192.168.0.0')" + }, + { + "dialect": "snowflake", + "sql": "SELECT DATEADD(DAY, -7, DATEADD(t.m, 1, CAST('2023-01-03' AS DATE))) FROM (SELECT 'month' AS m) AS t", + "formatted": "SELECT DATEADD(DAY, -7, DATEADD(t.m, 1, CAST('2023-01-03' AS DATE)))\nFROM (\n SELECT 'month' AS m\n) AS t" + }, + { + "dialect": "snowflake", + "sql": "SELECT FILE_URL FROM DIRECTORY(@mystage) WHERE SIZE > 100000", + "formatted": "SELECT FILE_URL\nFROM DIRECTORY(@mystage)\nWHERE SIZE > 100000" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM a..b", + "formatted": "SELECT *\nFROM a..b" + }, + { + "dialect": "tsql", + "sql": "SELECT TOP (SELECT 1) * FROM t", + "formatted": "SELECT TOP (\n SELECT 1\n)\n *\nFROM t" + }, + { + "dialect": "tsql", + "sql": "SELECT ATN2(x, y)", + "formatted": "SELECT ATN2(x, y)" + }, + { + "dialect": "tsql", + "sql": "SELECT EXP(1)", + "formatted": "SELECT EXP(1)" + }, + { + "dialect": "tsql", + "sql": "SELECT SYSDATETIMEOFFSET()", + "formatted": "SELECT SYSDATETIMEOFFSET()" + }, + { + "dialect": "tsql", + "sql": "SELECT COMPRESS('Hello World')", + "formatted": "SELECT COMPRESS('Hello World')" + }, + { + "dialect": "tsql", + "sql": "SELECT TRIM(' test ') AS Result", + "formatted": "SELECT TRIM(' test ') AS Result" + }, + { + "dialect": "tsql", + "sql": "SELECT TRIM('.,! ' FROM ' # test .') AS Result", + "formatted": "SELECT TRIM('.,! ' FROM ' # test .') AS Result" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t TABLESAMPLE (10 PERCENT)", + "formatted": "SELECT *\nFROM t TABLESAMPLE (10 PERCENT)" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t TABLESAMPLE (20 ROWS)", + "formatted": "SELECT *\nFROM t TABLESAMPLE (20 ROWS)" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t TABLESAMPLE (10 PERCENT) REPEATABLE (123)", + "formatted": "SELECT *\nFROM t TABLESAMPLE (10 PERCENT) REPEATABLE (123)" + }, + { + "dialect": "tsql", + "sql": "SELECT CONCAT(column1, column2)", + "formatted": "SELECT CONCAT(column1, column2)" + }, + { + "dialect": "tsql", + "sql": "SELECT TestSpecialChar.Test# FROM TestSpecialChar", + "formatted": "SELECT TestSpecialChar.Test#\nFROM TestSpecialChar" + }, + { + "dialect": "tsql", + "sql": "SELECT TestSpecialChar.Test@ FROM TestSpecialChar", + "formatted": "SELECT TestSpecialChar.Test@\nFROM TestSpecialChar" + }, + { + "dialect": "tsql", + "sql": "SELECT TestSpecialChar.Test$ FROM TestSpecialChar", + "formatted": "SELECT TestSpecialChar.Test$\nFROM TestSpecialChar" + }, + { + "dialect": "tsql", + "sql": "SELECT TestSpecialChar.Test_ FROM TestSpecialChar", + "formatted": "SELECT TestSpecialChar.Test_\nFROM TestSpecialChar" + }, + { + "dialect": "tsql", + "sql": "SELECT TOP (2 + 1) 1", + "formatted": "SELECT TOP (2 + 1) 1" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t WHERE NOT c", + "formatted": "SELECT *\nFROM t\nWHERE NOT c" + }, + { + "dialect": "tsql", + "sql": "WITH t1 AS (SELECT 1 AS a), t2 AS (SELECT 1 AS a) SELECT TOP 10 a FROM t1 UNION ALL SELECT TOP 10 a FROM t2", + "formatted": "WITH t1 AS (\n SELECT 1 AS a\n),\n\nt2 AS (\n SELECT 1 AS a\n)\n\nSELECT TOP 10 a\nFROM t1\nUNION ALL\nSELECT TOP 10 a\nFROM t2" + }, + { + "dialect": "tsql", + "sql": "SELECT TOP 10 s.RECORDID, n.c.VALUE('(/*:FORM_ROOT/*:SOME_TAG)[1]', 'float') AS SOME_TAG_VALUE FROM source_table.dbo.source_data AS s(nolock) CROSS APPLY FormContent.nodes('/*:FORM_ROOT') AS N(C)", + "formatted": "SELECT TOP 10\n s.RECORDID,\n n.c.VALUE('(/*:FORM_ROOT/*:SOME_TAG)[1]', 'float') AS SOME_TAG_VALUE\nFROM source_table.dbo.source_data AS s(nolock)\nCROSS APPLY FormContent.nodes('/*:FORM_ROOT') AS N(C)" + }, + { + "dialect": "tsql", + "sql": "WITH t1 AS (SELECT 1 AS a), t2 AS (SELECT 1 AS a) SELECT TOP 10 a FROM t1 UNION ALL SELECT TOP 10 a FROM t2 ORDER BY a DESC", + "formatted": "WITH t1 AS (\n SELECT 1 AS a\n),\n\nt2 AS (\n SELECT 1 AS a\n)\n\nSELECT TOP 10 a\nFROM t1\nUNION ALL\nSELECT TOP 10 a\nFROM t2\nORDER BY a DESC" + }, + { + "dialect": "tsql", + "sql": "WITH t1 AS (SELECT 1 AS a), t2 AS (SELECT 1 AS a) SELECT COUNT(*) FROM (SELECT TOP 10 a FROM t1 UNION ALL SELECT TOP 10 a FROM t2 ORDER BY a DESC) AS t", + "formatted": "WITH t1 AS (\n SELECT 1 AS a\n),\n\nt2 AS (\n SELECT 1 AS a\n)\n\nSELECT COUNT(*)\nFROM (\n SELECT TOP 10 a\n FROM t1\n UNION ALL\n SELECT TOP 10 a\n FROM t2\nORDER BY a DESC\n) AS t" + }, + { + "dialect": "tsql", + "sql": "SELECT 1 AS \"[x]\"", + "formatted": "SELECT 1 AS \"[x]\"" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t WITH (TABLOCK, INDEX(myindex))", + "formatted": "SELECT *\nFROM t WITH (TABLOCK, INDEX(myindex))" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t WITH (NOWAIT)", + "formatted": "SELECT *\nFROM t WITH (NOWAIT)" + }, + { + "dialect": "tsql", + "sql": "SELECT CASE WHEN a > 1 THEN b END", + "formatted": "SELECT CASE WHEN a > 1 THEN b END" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM taxi ORDER BY 1 OFFSET 0 ROWS FETCH NEXT 3 ROWS ONLY", + "formatted": "SELECT *\nFROM taxi\nORDER BY\n 1\nOFFSET 0 ROWS\nFETCH NEXT 3 ROWS ONLY" + }, + { + "dialect": "tsql", + "sql": "SELECT Employee_ID, Department_ID FROM @MyTableVar", + "formatted": "SELECT\n Employee_ID,\n Department_ID\nFROM @MyTableVar" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM #foo", + "formatted": "SELECT *\nFROM #foo" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM ##foo", + "formatted": "SELECT *\nFROM ##foo" + }, + { + "dialect": "tsql", + "sql": "SELECT a = 1", + "formatted": "SELECT a = 1" + }, + { + "dialect": "tsql", + "sql": "SELECT a = 1 UNION ALL SELECT a = b", + "formatted": "SELECT a = 1\nUNION ALL\nSELECT a = b" + }, + { + "dialect": "tsql", + "sql": "SELECT x FROM @MyTableVar AS m JOIN Employee ON m.EmployeeID = Employee.EmployeeID", + "formatted": "SELECT x\nFROM @MyTableVar AS m\nJOIN Employee\n ON m.EmployeeID = Employee.EmployeeID" + }, + { + "dialect": "tsql", + "sql": "SELECT DISTINCT DepartmentName, PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY BaseRate) OVER (PARTITION BY DepartmentName) AS MedianCont FROM dbo.DimEmployee", + "formatted": "SELECT DISTINCT\n DepartmentName,\n PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY BaseRate) OVER (PARTITION BY DepartmentName) AS MedianCont\nFROM dbo.DimEmployee" + }, + { + "dialect": "tsql", + "sql": "SELECT \"x\".\"y\" FROM foo", + "formatted": "SELECT \"x\".\"y\"\nFROM foo" + }, + { + "dialect": "tsql", + "sql": "SELECT val FROM (VALUES ((TRUE), (FALSE), (NULL))) AS t(val)", + "formatted": "SELECT val\nFROM (\n VALUES\n ((TRUE\n), (FALSE\n), (\n NULL\n))\n) AS t(val)" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t FOR XML PATH, BINARY BASE64, ELEMENTS XSINIL", + "formatted": "SELECT *\nFROM t\nFOR XML\n PATH,\n BINARY BASE64,\n ELEMENTS XSINIL" + }, + { + "dialect": "tsql", + "sql": "SELECT (SELECT TOP 5 a, b FROM t ORDER BY b DESC FOR JSON PATH) AS j FROM x", + "formatted": "SELECT\n (\n SELECT TOP 5\n a,\n b\n FROM t\n ORDER BY\n b DESC\n FOR JSON\n PATH\n ) AS j\nFROM x" + }, + { + "dialect": "tsql", + "sql": "SELECT j FROM (SELECT a AS j FROM t FOR JSON PATH) AS x", + "formatted": "SELECT j\nFROM (\n SELECT a AS j\n FROM t\n FOR JSON\n PATH\n) AS x" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t FOR JSON PATH, ROOT('Root'), INCLUDE_NULL_VALUES", + "formatted": "SELECT *\nFROM t\nFOR JSON\n PATH,\n ROOT('Root'),\n INCLUDE_NULL_VALUES" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t FOR BROWSE", + "formatted": "SELECT *\nFROM t FOR BROWSE" + }, + { + "dialect": "tsql", + "sql": "SELECT CAST(SUBSTRING('ABCD~1234', CHARINDEX('~', 'ABCD~1234') + 1, LEN('ABCD~1234')) AS BIGINT)", + "formatted": "SELECT CAST(SUBSTRING('ABCD~1234', CHARINDEX('~', 'ABCD~1234') + 1, LEN('ABCD~1234')) AS BIGINT)" + }, + { + "dialect": "tsql", + "sql": "SELECT DATEPART(\"dd\", x)", + "formatted": "SELECT DATEPART(\"dd\", x)" + }, + { + "dialect": "tsql", + "sql": "SELECT DATEADD(YEAR, 1, '2017/08/25')", + "formatted": "SELECT DATEADD(YEAR, 1, '2017/08/25')" + }, + { + "dialect": "tsql", + "sql": "SELECT DATEDIFF(HOUR, 1.5, '2021-01-01')", + "formatted": "SELECT DATEDIFF(HOUR, 1.5, '2021-01-01')" + }, + { + "dialect": "tsql", + "sql": "SELECT DATEDIFF_BIG(HOUR, 1.5, '2021-01-01')", + "formatted": "SELECT DATEDIFF_BIG(HOUR, 1.5, '2021-01-01')" + }, + { + "dialect": "tsql", + "sql": "SELECT TOP 10 PERCENT", + "formatted": "SELECT TOP 10 PERCENT" + }, + { + "dialect": "tsql", + "sql": "SELECT TOP 10 PERCENT WITH TIES", + "formatted": "SELECT TOP 10 PERCENT WITH TIES" + }, + { + "dialect": "tsql", + "sql": "SELECT FORMAT(foo, 'dddd', 'de-CH')", + "formatted": "SELECT FORMAT(foo, 'dddd', 'de-CH')" + }, + { + "dialect": "tsql", + "sql": "SELECT FORMAT(EndOfDayRate, 'N', 'en-us')", + "formatted": "SELECT FORMAT(EndOfDayRate, 'N', 'en-us')" + }, + { + "dialect": "tsql", + "sql": "SELECT FORMAT('01-01-1991', 'd.mm.yyyy')", + "formatted": "SELECT FORMAT('01-01-1991', 'd.mm.yyyy')" + }, + { + "dialect": "tsql", + "sql": "SELECT FORMAT(12345, '###.###.###')", + "formatted": "SELECT FORMAT(12345, '###.###.###')" + }, + { + "dialect": "tsql", + "sql": "SELECT FORMAT(1234567, 'f')", + "formatted": "SELECT FORMAT(1234567, 'f')" + }, + { + "dialect": "tsql", + "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME AS OF 'foo'", + "formatted": "SELECT [x]\nFROM [a].[b] FOR SYSTEM_TIME AS OF 'foo'" + }, + { + "dialect": "tsql", + "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME AS OF 'foo' AS alias", + "formatted": "SELECT [x]\nFROM [a].[b] FOR SYSTEM_TIME AS OF 'foo' AS alias" + }, + { + "dialect": "tsql", + "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME FROM c TO d", + "formatted": "SELECT [x]\nFROM [a].[b] FOR SYSTEM_TIME FROM c TO d" + }, + { + "dialect": "tsql", + "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME BETWEEN c AND d", + "formatted": "SELECT [x]\nFROM [a].[b] FOR SYSTEM_TIME BETWEEN c AND d" + }, + { + "dialect": "tsql", + "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME CONTAINED IN (c, d)", + "formatted": "SELECT [x]\nFROM [a].[b] FOR SYSTEM_TIME CONTAINED IN (c, d)" + }, + { + "dialect": "tsql", + "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME ALL AS alias", + "formatted": "SELECT [x]\nFROM [a].[b] FOR SYSTEM_TIME ALL AS alias" + }, + { + "dialect": "tsql", + "sql": "SELECT x FROM a INNER LOOP JOIN b ON b.id = a.id", + "formatted": "SELECT x\nFROM a\nINNER LOOP JOIN b\n ON b.id = a.id" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM OPENJSON(@json)", + "formatted": "SELECT *\nFROM OPENJSON(@json)" + }, + { + "dialect": "tsql", + "sql": "WITH t AS (SELECT 1) SELECT * FROM t", + "formatted": "WITH t AS (\n SELECT 1\n)\n\nSELECT *\nFROM t" + }, + { + "dialect": "tsql", + "sql": "WITH t AS (SELECT \"c\") SELECT * FROM t", + "formatted": "WITH t AS (\n SELECT \"c\"\n)\n\nSELECT *\nFROM t" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM (SELECT 1) AS subq", + "formatted": "SELECT *\nFROM (\n SELECT 1\n) AS subq" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM (SELECT \"c\") AS subq", + "formatted": "SELECT *\nFROM (\n SELECT \"c\"\n) AS subq" + }, + { + "dialect": "tsql", + "sql": "SELECT NEXT VALUE FOR db.schema.sequence_name OVER (ORDER BY foo), col", + "formatted": "SELECT\n NEXT VALUE FOR db.schema.sequence_name OVER (ORDER BY foo),\n col" + }, + { + "dialect": "tsql", + "sql": "SELECT COUNT(1) FROM x", + "formatted": "SELECT COUNT(1)\nFROM x" + }, + { + "dialect": "tsql", + "sql": "SELECT COUNT_BIG(1) FROM x", + "formatted": "SELECT COUNT_BIG(1)\nFROM x" + }, + { + "dialect": "tsql", + "sql": "SELECT 1; SELECT 2", + "formatted": "SELECT 1;\nSELECT 2" + }, + { + "dialect": "tsql", + "sql": "SELECT go", + "formatted": "SELECT go" + }, + { + "dialect": "tsql", + "sql": "SELECT 1 WHERE EXISTS(SELECT 1)", + "formatted": "SELECT 1\nWHERE\n EXISTS(\n SELECT 1\n )" + }, + { + "dialect": "tsql", + "sql": "SELECT a = 1", + "formatted": "SELECT a = 1" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM [#temp_table]", + "formatted": "SELECT *\nFROM [#temp_table]" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM [##temp_table]", + "formatted": "SELECT *\nFROM [##temp_table]" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM @x", + "formatted": "SELECT *\nFROM @x" + } +] diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/helpers.rs b/crates/sqlbuild-rules-native/src/sql_lint/tests/helpers.rs index 6f77d9c5fa..43e3886f44 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/tests/helpers.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/tests/helpers.rs @@ -1,6 +1,15 @@ +use std::str::FromStr; + +use polyglot_sql::{Dialect, DialectType}; use serde_json::{Value, json}; +use crate::sql_lint::main::batch_formatter::format_batch_json; use crate::sql_lint::main::engine::lint_json; +use crate::sql_lint::main::formatter::format_json; +use crate::sql_tokens::_helpers::builtin_functions::builtin_function_names; +use crate::sql_tokens::constants::CALL_SYNTAX_FUNCTIONS; +use crate::sql_tokens::main::canonical_tokens::canonical_tokens; +use crate::sql_tokens::main::query_fingerprint::query_fingerprint; pub(crate) fn nested_function_sql(depth: usize) -> String { format!("SELECT {}1{}", "F(".repeat(depth), ")".repeat(depth)) @@ -44,3 +53,203 @@ fn diagnostic_values(response: &str) -> Result, String> { .cloned() .ok_or_else(|| "diagnostics should be an array".to_string()) } + +/// A narrow width so that wrapping is exercised on most corpus queries. +const CORPUS_LINE_WIDTH: usize = 40; + +/// Format every corpus query of one dialect and return the formatted and refused counts. +pub(crate) fn check_dialect_corpus( + corpus: &[Value], + dialect: &str, +) -> Result<(usize, usize), String> { + let mut outcomes: Vec = Vec::new(); + for entry in corpus.iter().filter(|entry| entry["dialect"] == dialect) { + let sql = entry["sql"].as_str().unwrap_or_default(); + let response = format_once(sql, dialect, None)?; + let formatted = response["formatted"] == true; + let actual = response + .get("sql") + .filter(|_| formatted) + .cloned() + .unwrap_or(Value::Null); + let expected = entry["formatted"].as_str().map_or(Value::Null, Value::from); + assert_eq!(actual, expected, "{dialect}: {sql}"); + let printed = actual.as_str().unwrap_or(sql); + assert!( + only_keyword_case_differs(sql, printed, dialect)?, + "{dialect} fingerprint changed beyond keyword case: {sql}" + ); + let previous_layout: Option = entry["previously_formatted"] + .as_str() + .map(|previous| relaid_previous_layout(previous, dialect)) + .transpose()?; + let layouts = [ + entry["formatted"].as_str().map(str::to_string), + previous_layout, + ]; + for stable in layouts.iter().flatten() { + let stable = stable.as_str(); + let again = format_once(stable, dialect, None)?; + assert_eq!(again["sql"], stable, "{dialect} layout drifted: {stable}"); + assert_eq!( + again["changed"], false, + "{dialect} layout drifted: {stable}" + ); + assert_eq!( + query_fingerprint(again["sql"].as_str().unwrap_or_default(), dialect)?, + query_fingerprint(stable, dialect)?, + "{dialect} formatting changed the fingerprint of formatted SQL: {stable}" + ); + } + let wrapped = format_once(sql, dialect, Some(CORPUS_LINE_WIDTH))?; + assert_eq!(wrapped["formatted"], formatted, "{dialect} wrapped: {sql}"); + let wrapped_sql = wrapped["sql"].as_str().unwrap_or_default(); + let rewrapped = format_once(wrapped_sql, dialect, Some(CORPUS_LINE_WIDTH))?; + assert_eq!( + rewrapped["sql"], wrapped_sql, + "{dialect} wrapping drifted: {sql}" + ); + outcomes.push(formatted); + } + let formatted = outcomes.iter().filter(|outcome| **outcome).count(); + Ok((formatted, outcomes.len() - formatted)) +} + +/// Whether `printed` has the fingerprint tokens of `sql` except for words upper-cased by format. +fn only_keyword_case_differs(sql: &str, printed: &str, dialect: &str) -> Result { + let parser = Dialect::get(DialectType::from_str(dialect).map_err(|error| error.to_string())?); + let left_tokens = parser.tokenize(sql).map_err(|error| error.to_string())?; + let right_tokens = parser + .tokenize(printed) + .map_err(|error| error.to_string())?; + let left = canonical_tokens(sql, &left_tokens, &parser)?; + let right = canonical_tokens(printed, &right_tokens, &parser)?; + Ok(left.len() == right.len() + && left.iter().zip(&right).all(|(before, after)| { + before == after + || (before.text.eq_ignore_ascii_case(&after.text) + && after.text == after.text.to_ascii_uppercase()) + })) +} + +/// Re-format a layout of an earlier formatter release, which must keep its fingerprint. +fn relaid_previous_layout(previous: &str, dialect: &str) -> Result { + let relaid = format_once(previous, dialect, None)?; + let sql = relaid["sql"].as_str().unwrap_or_default().to_string(); + assert_eq!( + query_fingerprint(&sql, dialect)?, + query_fingerprint(previous, dialect)?, + "{dialect} formatting changed the fingerprint of a previous layout: {previous}" + ); + Ok(sql) +} + +pub(crate) fn format_at_width( + sql: &str, + dialect: &str, + line_width: usize, +) -> Result { + format_once(sql, dialect, Some(line_width)) +} + +fn format_once(sql: &str, dialect: &str, line_width: Option) -> Result { + let request = json!({"version": 1, "sql": sql, "dialect": dialect, "line_width": line_width}); + let response = format_json(&request.to_string())?; + serde_json::from_str(&response).map_err(|error| error.to_string()) +} + +/// Call names the formatter treats as built-ins in `dialect`, plus keyword-like call names. +const KEYWORD_LIKE_CALLS: [&str; 11] = [ + "if", + "iff", + "left", + "right", + "replace", + "insert", + "format", + "filter", + "date", + "time", + "timestamp", +]; + +/// Return names no 2/1/0-argument call could format, and calls printed with misplaced spaces. +pub(crate) fn function_call_spacing( + dialect: &str, + line_width: usize, + argument: &str, +) -> Result<(Vec, Vec), String> { + let dialect_type = DialectType::from_str(dialect).map_err(|error| error.to_string())?; + let mut names: Vec = builtin_function_names(dialect_type) + .into_iter() + .chain(KEYWORD_LIKE_CALLS.iter().map(|name| (*name).to_string())) + .filter(|name| name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')) + .filter(|name| !CALL_SYNTAX_FUNCTIONS.contains(&name.to_ascii_uppercase().as_str())) + .collect(); + names.sort_unstable(); + names.dedup(); + let mut misplaced: Vec = Vec::new(); + for arguments in [ + format!("{argument}, 1"), + argument.to_string(), + String::new(), + ] { + let responses = format_calls(&names, &arguments, dialect, line_width)?; + misplaced.extend( + names + .iter() + .zip(&responses) + .filter(|(_, response)| response["formatted"] == true) + .filter(|(name, response)| { + !is_tight_call( + name, + response["sql"].as_str().unwrap_or_default(), + &arguments, + ) + }) + .map(|(name, response)| format!("{name}: {}", response["sql"])), + ); + names = names + .into_iter() + .zip(&responses) + .filter(|(_, response)| response["formatted"] != true) + .map(|(name, _)| name) + .collect(); + } + Ok((names, misplaced)) +} + +fn format_calls( + names: &[String], + arguments: &str, + dialect: &str, + line_width: usize, +) -> Result, String> { + let requests: Vec = names + .iter() + .map(|name| { + json!({ + "version": 1, + "sql": format!("SELECT {name}({arguments}) AS x FROM t"), + "dialect": dialect, + "line_width": line_width, + }) + }) + .collect(); + serde_json::from_str(&format_batch_json(&Value::from(requests).to_string())?) + .map_err(|error| error.to_string()) +} + +/// Whether `sql` prints `name(` in authored or upper case with no space inside the call. +fn is_tight_call(name: &str, sql: &str, arguments: &str) -> bool { + [name.to_string(), name.to_ascii_uppercase()] + .iter() + .any(|printed| { + sql.contains(&format!("{printed}({arguments})")) + || sql.contains(&format!("{printed}(\n")) + }) + && !sql.contains("( ") + && !sql + .to_ascii_uppercase() + .contains(&format!("{} (", name.to_ascii_uppercase())) +} diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_authored_tokens.rs b/crates/sqlbuild-rules-native/src/sql_lint/tests/test_authored_tokens.rs index 879a18978f..05b8c4601d 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_authored_tokens.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/tests/test_authored_tokens.rs @@ -1,6 +1,8 @@ use serde_json::{Value, json}; -use crate::sql_lint::_helpers::formatter::preserve_authored_tokens; +use std::collections::BTreeMap; + +use crate::sql_lint::_helpers::token_layout::{KeywordRecase, verify_token_invariant}; use crate::sql_lint::main::formatter::format_json; use crate::sql_lint::tests::test_types; @@ -11,19 +13,19 @@ fn given_generator_respellings_when_formatting_then_authored_spellings_are_kept( test_types::FormatTestCase { description: "Snowflake STARTSWITH keeps its authored name", sql: "select startswith(order_code, 'EU') as is_eu_order from orders", - expected_sql: "SELECT\n startswith(order_code, 'EU') AS is_eu_order\nFROM orders", + expected_sql: "SELECT startswith(order_code, 'EU') AS is_eu_order\nFROM orders", expected_changed: true, }, test_types::FormatTestCase { description: "function synonyms keep their authored names", sql: "select SUBSTR(order_code, 1, 2) as region, TIMESTAMPDIFF(day, ordered_at, shipped_at) as days_to_ship, TRY_TO_DECIMAL(amount_text, 10, 2) as amount from orders", - expected_sql: "SELECT\n SUBSTR(order_code, 1, 2) AS region,\n TIMESTAMPDIFF(DAY, ordered_at, shipped_at) AS days_to_ship,\n TRY_TO_DECIMAL(amount_text, 10, 2) AS amount\nFROM orders", + expected_sql: "SELECT\n SUBSTR(order_code, 1, 2) AS region,\n TIMESTAMPDIFF(day, ordered_at, shipped_at) AS days_to_ship,\n TRY_TO_DECIMAL(amount_text, 10, 2) AS amount\nFROM orders", expected_changed: true, }, test_types::FormatTestCase { description: "operator and function spellings stay authored", sql: "select IFNULL(status, 'open') as status from orders where quantity != 0", - expected_sql: "SELECT\n IFNULL(status, 'open') AS status\nFROM orders\nWHERE\n quantity != 0", + expected_sql: "SELECT IFNULL(status, 'open') AS status\nFROM orders\nWHERE quantity != 0", expected_changed: true, }, ]; @@ -47,49 +49,343 @@ fn given_generator_respellings_when_formatting_then_authored_spellings_are_kept( } #[test] -fn given_generated_token_changes_when_preserving_authored_tokens_then_only_layout_survives() { - let dialect = polyglot_sql::Dialect::get(polyglot_sql::DialectType::Snowflake); +fn given_generator_restructuring_when_formatting_then_authored_tokens_are_laid_out() +-> Result<(), String> { let test_cases = [ - test_types::AuthoredTokenTestCase { - description: "renamed function is restored to its authored spelling", - authored: "select startswith(code, 'EU') from orders", - generated: "SELECT\n STARTS_WITH(code, 'EU')\nFROM orders", - expected_sql: Some("SELECT\n startswith(code, 'EU')\nFROM orders"), - }, - test_types::AuthoredTokenTestCase { - description: "swapped arguments are restored to authored order", - authored: "select datediff(day, a, b) from orders", - generated: "SELECT\n DATEDIFF(DAY, b, a)\nFROM orders", - expected_sql: Some("SELECT\n DATEDIFF(DAY, a, b)\nFROM orders"), - }, - test_types::AuthoredTokenTestCase { - description: "an explicit alias keyword may be added", - authored: "select a b from orders", - generated: "SELECT\n a AS b\nFROM orders", - expected_sql: Some("SELECT\n a AS b\nFROM orders"), - }, - test_types::AuthoredTokenTestCase { - description: "a structural change refuses the format", - authored: "select a from orders where a <> 1", - generated: "SELECT\n a\nFROM orders\nWHERE\n NOT a = 1", - expected_sql: None, - }, - test_types::AuthoredTokenTestCase { - description: "a dropped token refuses the format", - authored: "select a from orders left outer join items using (id)", - generated: "SELECT\n a\nFROM orders\nLEFT JOIN items USING (id)", - expected_sql: None, + test_types::DialectFormatTestCase { + description: "PostgreSQL comma SUBSTRING arguments are not rewritten to FROM/FOR", + dialect: "postgres", + sql: "select substring(code, 1, 2) as region from orders", + expected_sql: "SELECT SUBSTRING(code, 1, 2) AS region\nFROM orders", + }, + test_types::DialectFormatTestCase { + description: "DuckDB SELECT ALL keeps ALL", + dialect: "duckdb", + sql: "select all order_id from orders", + expected_sql: "SELECT ALL order_id\nFROM orders", + }, + test_types::DialectFormatTestCase { + description: "Snowflake SELECT ALL keeps ALL", + dialect: "snowflake", + sql: "select all order_id from orders", + expected_sql: "SELECT ALL order_id\nFROM orders", + }, + test_types::DialectFormatTestCase { + description: "PostgreSQL SELECT ALL keeps ALL", + dialect: "postgres", + sql: "select all order_id from orders", + expected_sql: "SELECT ALL order_id\nFROM orders", + }, + test_types::DialectFormatTestCase { + description: "BigQuery SELECT ALL keeps ALL", + dialect: "bigquery", + sql: "select all order_id from orders", + expected_sql: "SELECT ALL order_id\nFROM orders", + }, + test_types::DialectFormatTestCase { + description: "Databricks SELECT ALL keeps ALL", + dialect: "databricks", + sql: "select all order_id from orders", + expected_sql: "SELECT all order_id\nFROM orders", + }, + test_types::DialectFormatTestCase { + description: "T-SQL SELECT ALL keeps ALL", + dialect: "tsql", + sql: "select all order_id from orders", + expected_sql: "SELECT ALL order_id\nFROM orders", + }, + test_types::DialectFormatTestCase { + description: "implicit aliases stay implicit", + dialect: "duckdb", + sql: "select order_id id, amount from orders o", + expected_sql: "SELECT\n order_id id,\n amount\nFROM orders o", + }, + test_types::DialectFormatTestCase { + description: "a trailing select comma and statement terminator are kept", + dialect: "duckdb", + sql: "select order_id, amount, from orders;", + expected_sql: "SELECT\n order_id,\n amount,\nFROM orders;", + }, + test_types::DialectFormatTestCase { + description: "adjacent authored tokens are never split", + dialect: "duckdb", + sql: "select $name as v from orders where status != 1", + expected_sql: "SELECT $name AS v\nFROM orders\nWHERE status != 1", + }, + test_types::DialectFormatTestCase { + description: "BigQuery numeric-prefixed path parts stay joined", + dialect: "bigquery", + sql: "select * from orders.archive.25_", + expected_sql: "SELECT *\nFROM orders.archive.25_", }, ]; - for test_case in test_cases { - let actual = preserve_authored_tokens( - test_case.authored, - test_case.generated.to_string(), - &dialect, + for test_case in &test_cases { + let response = format_json( + &json!({"version": 1, "sql": test_case.sql, "dialect": test_case.dialect}).to_string(), + )?; + let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; + assert_eq!( + payload["sql"], test_case.expected_sql, + "{}", + test_case.description + ); + let again = format_json( + &json!({"version": 1, "sql": test_case.expected_sql, "dialect": test_case.dialect}) + .to_string(), + )?; + let repeated: Value = serde_json::from_str(&again).map_err(|error| error.to_string())?; + assert_eq!(repeated["changed"], false, "{}", test_case.description); + } + Ok(()) +} + +#[test] +fn given_case_or_line_sensitive_tokens_when_formatting_then_they_are_kept_exactly() +-> Result<(), String> { + let test_cases = [ + test_types::DialectFormatTestCase { + description: "DuckDB keyword-named aliases and qualified names keep their case", + dialect: "duckdb", + sql: "select pos as index, sku as replace, t.view from orders t", + expected_sql: "SELECT\n pos AS index,\n sku AS replace,\n t.view\nFROM orders t", + }, + test_types::DialectFormatTestCase { + description: "BigQuery keyword-named table names keep their case", + dialect: "bigquery", + sql: "select * from inventory.view", + expected_sql: "SELECT *\nFROM inventory.view", + }, + test_types::DialectFormatTestCase { + description: "Snowflake path keys keep their case", + dialect: "snowflake", + sql: "select payload:Customer.Name as name, payload:Date::string as day from orders", + expected_sql: "SELECT\n payload:Customer.Name AS name,\n payload:Date::string AS day\nFROM orders", + }, + test_types::DialectFormatTestCase { + description: "user-defined functions keep their case, built-ins are uppercased", + dialect: "bigquery", + sql: "select MyFunc(a), ds.MyUdf(b), coalesce(a, b) from orders", + expected_sql: "SELECT\n MyFunc(a),\n ds.MyUdf(b),\n COALESCE(a, b)\nFROM orders", + }, + test_types::DialectFormatTestCase { + description: "DuckDB non-reserved keyword operands keep their case before any operator", + dialect: "duckdb", + sql: "select a, Rows << 1, Rows == 1, Rows ^ 2 from t", + expected_sql: "SELECT\n a,\n Rows << 1,\n Rows == 1,\n Rows ^ 2\nFROM t", + }, + test_types::DialectFormatTestCase { + description: "BigQuery comma-joined keyword-named tables keep their case", + dialect: "bigquery", + sql: "select * from ds.a, View v", + expected_sql: "SELECT *\nFROM ds.a, View v", + }, + test_types::DialectFormatTestCase { + description: "DuckDB adjacent string literals keep their line break", + dialect: "duckdb", + sql: "select 'north'\n'east' as region from orders", + expected_sql: "SELECT\n 'north'\n 'east' AS region\nFROM orders", + }, + test_types::DialectFormatTestCase { + description: "PostgreSQL adjacent string literals keep their line break", + dialect: "postgres", + sql: "select 'north'\n 'east' as region from orders", + expected_sql: "SELECT\n 'north'\n 'east' AS region\nFROM orders", + }, + test_types::DialectFormatTestCase { + description: "Snowflake double-slash comments are kept", + dialect: "snowflake", + sql: "select order_id, // keep this note\n customer_id from orders", + expected_sql: "SELECT\n order_id, // keep this note\n customer_id\nFROM orders", + }, + test_types::DialectFormatTestCase { + description: "BigQuery hash comments are kept", + dialect: "bigquery", + sql: "select order_id, # keep this note\n customer_id from orders", + expected_sql: "SELECT\n order_id, # keep this note\n customer_id\nFROM orders", + }, + test_types::DialectFormatTestCase { + description: "T-SQL temporary table hashes are not comments", + dialect: "tsql", + sql: "select #orders.order_id from #orders", + expected_sql: "SELECT #orders.order_id\nFROM #orders", + }, + test_types::DialectFormatTestCase { + description: "DuckDB integer division and slashes inside strings are not comments", + dialect: "duckdb", + sql: "select 7 // 2 as half, 'https://example.com/a' as url from orders", + expected_sql: "SELECT\n 7 // 2 AS half,\n 'https://example.com/a' AS url\nFROM orders", + }, + test_types::DialectFormatTestCase { + description: "Snowflake double slashes inside strings are not comments", + dialect: "snowflake", + sql: "select 'https://example.com//a' as url from orders", + expected_sql: "SELECT 'https://example.com//a' AS url\nFROM orders", + }, + ]; + for test_case in &test_cases { + let response = format_json( + &json!({"version": 1, "sql": test_case.sql, "dialect": test_case.dialect}).to_string(), + )?; + let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; + assert_eq!( + payload["sql"], test_case.expected_sql, + "{}", + test_case.description ); + let again = format_json( + &json!({"version": 1, "sql": test_case.expected_sql, "dialect": test_case.dialect}) + .to_string(), + )?; + let repeated: Value = serde_json::from_str(&again).map_err(|error| error.to_string())?; + assert_eq!(repeated["changed"], false, "{}", test_case.description); + } + Ok(()) +} + +#[test] +fn given_formatted_candidates_when_checking_token_invariant_then_only_layout_may_differ() { + let test_cases = [ + test_types::TokenInvariantTestCase { + description: "whitespace and word case may change", + before: "select a,b from orders -- note", + after: "SELECT\n a,\n b\nFROM orders -- note", + recases: &[], + expected_error: None, + }, + test_types::TokenInvariantTestCase { + description: "blank lines between CTEs keep comments on their tokens", + before: "WITH orders AS (SELECT 1), -- note\n-- paid only\npaid AS (SELECT 2) SELECT * FROM paid", + after: "WITH orders AS (\n SELECT 1\n), -- note\n\n-- paid only\npaid AS (\n SELECT 2\n)\n\nSELECT *\nFROM paid", + recases: &[], + expected_error: None, + }, + test_types::TokenInvariantTestCase { + description: "a comment moved across a CTE blank line is refused", + before: "WITH orders AS (SELECT 1), -- note\npaid AS (SELECT 2) SELECT * FROM paid", + after: "WITH orders AS (\n SELECT 1\n)\n-- note\n,\n\npaid AS (\n SELECT 2\n)\n\nSELECT *\nFROM paid", + recases: &[], + expected_error: Some("native formatter could not preserve comment token attachments"), + }, + test_types::TokenInvariantTestCase { + description: "a renamed function is refused", + before: "select startswith(code, 'EU') from orders", + after: "SELECT STARTS_WITH(code, 'EU') FROM orders", + recases: &[], + expected_error: Some( + "native formatter would change authored SQL tokens, not only layout", + ), + }, + test_types::TokenInvariantTestCase { + description: "an added alias keyword is refused", + before: "select a b from orders", + after: "SELECT a AS b FROM orders", + recases: &[], + expected_error: Some( + "native formatter would change authored SQL tokens, not only layout", + ), + }, + test_types::TokenInvariantTestCase { + description: "string literal whitespace is refused", + before: "select 'a b' from orders", + after: "SELECT 'a b' FROM orders", + recases: &[], + expected_error: Some( + "native formatter would change authored SQL tokens, not only layout", + ), + }, + test_types::TokenInvariantTestCase { + description: "a recorded keyword recase from the layout guide is allowed", + before: "select a from orders order by a nulls last", + after: "SELECT a FROM orders ORDER BY a NULLS LAST", + recases: &[(7, "NULLS", true), (8, "LAST", true)], + expected_error: None, + }, + test_types::TokenInvariantTestCase { + description: "an unrecorded non-reserved keyword recase is refused", + before: "select a from orders order by a nulls last", + after: "SELECT a FROM orders ORDER BY a NULLS LAST", + recases: &[], + expected_error: Some( + "native formatter would change authored SQL tokens, not only layout", + ), + }, + test_types::TokenInvariantTestCase { + description: "a recase whose guide token is not a keyword is refused", + before: "select a from orders order by a nulls last", + after: "SELECT a FROM orders ORDER BY a NULLS LAST", + recases: &[(7, "NULLS", false), (8, "LAST", false)], + expected_error: Some( + "native formatter would change authored SQL tokens, not only layout", + ), + }, + test_types::TokenInvariantTestCase { + description: "an identifier case change is refused", + before: "select a from orders", + after: "SELECT a FROM Orders", + recases: &[], + expected_error: Some( + "native formatter would change authored SQL tokens, not only layout", + ), + }, + test_types::TokenInvariantTestCase { + description: "a path key case change is refused", + before: "select payload:customerId from orders", + after: "SELECT payload:CUSTOMERID FROM orders", + recases: &[], + expected_error: Some( + "native formatter would change authored SQL tokens, not only layout", + ), + }, + test_types::TokenInvariantTestCase { + description: "a built-in function may be recased", + before: "select coalesce(a, 0) from orders", + after: "SELECT COALESCE(a, 0) FROM orders", + recases: &[], + expected_error: None, + }, + test_types::TokenInvariantTestCase { + description: "joining adjacent string literals across a line break is refused", + before: "select 'a'\n'b' from orders", + after: "SELECT 'a' 'b' FROM orders", + recases: &[], + expected_error: Some( + "native formatter would join adjacent string literals across a line break", + ), + }, + test_types::TokenInvariantTestCase { + description: "a dropped double-slash comment is refused", + before: "select a // note\nfrom orders", + after: "SELECT a FROM orders", + recases: &[], + expected_error: Some("native formatter could not preserve comment token attachments"), + }, + test_types::TokenInvariantTestCase { + description: "a comment moved to another token is refused", + before: "select a, -- note\n b from orders", + after: "SELECT a, b -- note\nFROM orders", + recases: &[], + expected_error: Some("native formatter could not preserve comment token attachments"), + }, + ]; + let dialect = polyglot_sql::Dialect::get(polyglot_sql::DialectType::Snowflake); + for test_case in test_cases { + let recases: BTreeMap = test_case + .recases + .iter() + .map(|(index, generated, keyword)| { + ( + *index, + KeywordRecase { + generated: (*generated).to_string(), + keyword: *keyword, + }, + ) + }) + .collect(); + let actual = verify_token_invariant(test_case.before, test_case.after, &dialect, &recases); assert_eq!( - actual.ok().as_deref(), - test_case.expected_sql, + actual.err().as_deref(), + test_case.expected_error, "{}", test_case.description ); diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_comment_formatter.rs b/crates/sqlbuild-rules-native/src/sql_lint/tests/test_comment_formatter.rs index 940b1301ae..5519b0a18f 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_comment_formatter.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/tests/test_comment_formatter.rs @@ -10,25 +10,25 @@ fn given_leading_cte_comments_when_formatting_then_comments_stay_above_their_cte test_types::FormatTestCase { description: "leading comment before a middle CTE stays above it", sql: "WITH\nupstream AS (\n SELECT * FROM orders\n),\n\n-- Explains the next CTE: line one,\n-- line two.\ndistinct_rows AS (\n SELECT order_id FROM upstream GROUP BY ALL\n)\n\nSELECT order_id FROM distinct_rows", - expected_sql: "WITH upstream AS (\n SELECT\n *\n FROM orders\n),\n-- Explains the next CTE: line one,\n-- line two.\ndistinct_rows AS (\n SELECT\n order_id\n FROM upstream\n GROUP BY ALL\n)\nSELECT\n order_id\nFROM distinct_rows", + expected_sql: "WITH upstream AS (\n SELECT *\n FROM orders\n),\n\n-- Explains the next CTE: line one,\n-- line two.\ndistinct_rows AS (\n SELECT order_id\n FROM upstream\n GROUP BY ALL\n)\n\nSELECT order_id\nFROM distinct_rows", expected_changed: true, }, test_types::FormatTestCase { description: "formatted leading CTE comment is idempotent", - sql: "WITH upstream AS (\n SELECT\n *\n FROM orders\n),\n-- Explains the next CTE: line one,\n-- line two.\ndistinct_rows AS (\n SELECT\n order_id\n FROM upstream\n GROUP BY ALL\n)\nSELECT\n order_id\nFROM distinct_rows", - expected_sql: "WITH upstream AS (\n SELECT\n *\n FROM orders\n),\n-- Explains the next CTE: line one,\n-- line two.\ndistinct_rows AS (\n SELECT\n order_id\n FROM upstream\n GROUP BY ALL\n)\nSELECT\n order_id\nFROM distinct_rows", + sql: "WITH upstream AS (\n SELECT *\n FROM orders\n),\n\n-- Explains the next CTE: line one,\n-- line two.\ndistinct_rows AS (\n SELECT order_id\n FROM upstream\n GROUP BY ALL\n)\n\nSELECT order_id\nFROM distinct_rows", + expected_sql: "WITH upstream AS (\n SELECT *\n FROM orders\n),\n\n-- Explains the next CTE: line one,\n-- line two.\ndistinct_rows AS (\n SELECT order_id\n FROM upstream\n GROUP BY ALL\n)\n\nSELECT order_id\nFROM distinct_rows", expected_changed: false, }, test_types::FormatTestCase { description: "leading comment before the first CTE stays above it", sql: "WITH\n-- Explains the first CTE.\nupstream AS (SELECT * FROM orders)\nSELECT order_id FROM upstream", - expected_sql: "WITH\n-- Explains the first CTE.\nupstream AS (\n SELECT\n *\n FROM orders\n)\nSELECT\n order_id\nFROM upstream", + expected_sql: "WITH\n-- Explains the first CTE.\nupstream AS (\n SELECT *\n FROM orders\n)\n\nSELECT order_id\nFROM upstream", expected_changed: true, }, test_types::FormatTestCase { description: "leading comment before the last of three CTEs stays above it", sql: "WITH a AS (SELECT 1 AS x),\nb AS (SELECT x FROM a),\n /* Explains the last CTE. */\nc AS (SELECT x FROM b)\nSELECT x FROM c", - expected_sql: "WITH a AS (\n SELECT\n 1 AS x\n), b AS (\n SELECT\n x\n FROM a\n),\n/* Explains the last CTE. */\nc AS (\n SELECT\n x\n FROM b\n)\nSELECT\n x\nFROM c", + expected_sql: "WITH a AS (\n SELECT 1 AS x\n),\n\nb AS (\n SELECT x\n FROM a\n),\n\n/* Explains the last CTE. */\nc AS (\n SELECT x\n FROM b\n)\n\nSELECT x\nFROM c", expected_changed: true, }, ]; diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_format_corpus.rs b/crates/sqlbuild-rules-native/src/sql_lint/tests/test_format_corpus.rs new file mode 100644 index 0000000000..c40d2ff006 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_lint/tests/test_format_corpus.rs @@ -0,0 +1,61 @@ +use serde_json::Value; + +use crate::sql_lint::tests::helpers::check_dialect_corpus; +use crate::sql_lint::tests::test_types; + +const FORMAT_CORPUS: &str = include_str!("fixtures/format_corpus.json"); + +#[test] +fn given_dialect_corpus_when_formatting_then_output_layout_and_refusals_are_stable() +-> Result<(), String> { + let test_cases = [ + test_types::FormatCorpusTestCase { + description: "BigQuery identity corpus", + dialect: "bigquery", + expected_formatted: 168, + expected_refused: 0, + }, + test_types::FormatCorpusTestCase { + description: "Databricks identity corpus", + dialect: "databricks", + expected_formatted: 62, + expected_refused: 0, + }, + test_types::FormatCorpusTestCase { + description: "DuckDB identity corpus", + dialect: "duckdb", + expected_formatted: 178, + expected_refused: 11, + }, + test_types::FormatCorpusTestCase { + description: "PostgreSQL identity corpus", + dialect: "postgres", + expected_formatted: 132, + expected_refused: 5, + }, + test_types::FormatCorpusTestCase { + description: "Snowflake identity corpus", + dialect: "snowflake", + expected_formatted: 429, + expected_refused: 6, + }, + test_types::FormatCorpusTestCase { + description: "T-SQL identity corpus", + dialect: "tsql", + expected_formatted: 75, + expected_refused: 0, + }, + ]; + let corpus: Vec = + serde_json::from_str(FORMAT_CORPUS).map_err(|error| error.to_string())?; + for test_case in test_cases { + let (formatted, refused) = check_dialect_corpus(&corpus, test_case.dialect)?; + assert_eq!( + (formatted, refused), + (test_case.expected_formatted, test_case.expected_refused), + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_function_call_spacing.rs b/crates/sqlbuild-rules-native/src/sql_lint/tests/test_function_call_spacing.rs new file mode 100644 index 0000000000..4acd30cdbf --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_lint/tests/test_function_call_spacing.rs @@ -0,0 +1,117 @@ +use crate::sql_lint::tests::helpers::function_call_spacing; +use crate::sql_lint::tests::test_types; + +#[test] +fn given_builtin_call_names_when_formatting_then_no_space_enters_the_call() -> Result<(), String> { + let test_cases = [ + test_types::FunctionCallSpacingTestCase { + description: "duckdb compact calls", + dialect: "duckdb", + line_width: 100, + argument: "a", + expected_refused: &[], + }, + test_types::FunctionCallSpacingTestCase { + description: "duckdb breaking calls", + dialect: "duckdb", + line_width: 40, + argument: "order_amount_with_a_long_descriptive_name", + expected_refused: &[], + }, + test_types::FunctionCallSpacingTestCase { + description: "snowflake compact calls", + dialect: "snowflake", + line_width: 100, + argument: "a", + expected_refused: &[], + }, + test_types::FunctionCallSpacingTestCase { + description: "snowflake breaking calls", + dialect: "snowflake", + line_width: 40, + argument: "order_amount_with_a_long_descriptive_name", + expected_refused: &[], + }, + test_types::FunctionCallSpacingTestCase { + description: "bigquery compact calls", + dialect: "bigquery", + line_width: 100, + argument: "a", + expected_refused: &[], + }, + test_types::FunctionCallSpacingTestCase { + description: "bigquery breaking calls", + dialect: "bigquery", + line_width: 40, + argument: "order_amount_with_a_long_descriptive_name", + expected_refused: &[], + }, + test_types::FunctionCallSpacingTestCase { + description: "postgres compact calls", + dialect: "postgres", + line_width: 100, + argument: "a", + expected_refused: &[], + }, + test_types::FunctionCallSpacingTestCase { + description: "postgres breaking calls", + dialect: "postgres", + line_width: 40, + argument: "order_amount_with_a_long_descriptive_name", + expected_refused: &[], + }, + test_types::FunctionCallSpacingTestCase { + description: "tsql compact calls", + dialect: "tsql", + line_width: 100, + argument: "a", + expected_refused: &[], + }, + test_types::FunctionCallSpacingTestCase { + description: "tsql breaking calls", + dialect: "tsql", + line_width: 40, + argument: "order_amount_with_a_long_descriptive_name", + expected_refused: &[], + }, + test_types::FunctionCallSpacingTestCase { + description: "databricks compact calls", + dialect: "databricks", + line_width: 100, + argument: "a", + expected_refused: &[], + }, + test_types::FunctionCallSpacingTestCase { + description: "databricks breaking calls", + dialect: "databricks", + line_width: 40, + argument: "order_amount_with_a_long_descriptive_name", + expected_refused: &[], + }, + test_types::FunctionCallSpacingTestCase { + description: "generic compact calls", + dialect: "generic", + line_width: 100, + argument: "a", + expected_refused: &[], + }, + test_types::FunctionCallSpacingTestCase { + description: "generic breaking calls", + dialect: "generic", + line_width: 40, + argument: "order_amount_with_a_long_descriptive_name", + expected_refused: &[], + }, + ]; + for test_case in &test_cases { + let (refused, misplaced) = + function_call_spacing(test_case.dialect, test_case.line_width, test_case.argument)?; + assert_eq!( + refused, test_case.expected_refused, + "{}", + test_case.description + ); + assert_eq!(misplaced, Vec::::new(), "{}", test_case.description); + } + Ok(()) +} diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_layout_rules.rs b/crates/sqlbuild-rules-native/src/sql_lint/tests/test_layout_rules.rs new file mode 100644 index 0000000000..191db93296 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_lint/tests/test_layout_rules.rs @@ -0,0 +1,280 @@ +use serde_json::Value; + +use crate::sql_lint::tests::helpers::format_at_width; +use crate::sql_lint::tests::test_types; +use crate::sql_tokens::main::query_fingerprint::query_fingerprint; + +#[test] +fn given_clause_layout_rules_when_formatting_then_each_rule_applies_idempotently() +-> Result<(), String> { + let test_cases = [ + test_types::LayoutRuleTestCase { + description: "a long Snowflake FROM VALUES puts each unsplit row on its own line", + dialect: "snowflake", + sql: "SELECT * FROM VALUES (1, '2024-01-15', 'PAID', 'north warehouse', 'priority shipping'), (2, '2024-01-16', 'OPEN', 'south warehouse', 'standard shipping')", + line_width: 60, + expected_sql: "SELECT *\nFROM VALUES\n (1, '2024-01-15', 'PAID', 'north warehouse', 'priority shipping'),\n (2, '2024-01-16', 'OPEN', 'south warehouse', 'standard shipping')", + }, + test_types::LayoutRuleTestCase { + description: "a short VALUES list stays on one line", + dialect: "snowflake", + sql: "SELECT * FROM VALUES ('a', 'A'), ('b', 'B')", + line_width: 100, + expected_sql: "SELECT *\nFROM VALUES ('a', 'A'), ('b', 'B')", + }, + test_types::LayoutRuleTestCase { + description: "a long parenthesized PostgreSQL VALUES moves under its parenthesis", + dialect: "postgres", + sql: "SELECT order_id, status FROM (VALUES (1, 'paid and shipped to customer'), (2, 'cancelled by customer')) AS orders(order_id, status)", + line_width: 60, + expected_sql: "SELECT\n order_id,\n status\nFROM (\n VALUES\n (1, 'paid and shipped to customer'),\n (2, 'cancelled by customer')\n) AS orders(order_id, status)", + }, + test_types::LayoutRuleTestCase { + description: "a long T-SQL VALUES relation keeps rows whole", + dialect: "tsql", + sql: "SELECT order_id FROM (VALUES (1, 'paid and shipped to customer'), (2, 'cancelled by customer')) AS orders(order_id, status)", + line_width: 60, + expected_sql: "SELECT order_id\nFROM (\n VALUES\n (1, 'paid and shipped to customer'),\n (2, 'cancelled by customer')\n) AS orders(order_id, status)", + }, + test_types::LayoutRuleTestCase { + description: "DuckDB INSERT VALUES rows stay whole when a row is over the width", + dialect: "duckdb", + sql: "INSERT INTO orders VALUES (1, 'paid and shipped to the customer by priority courier'), (2, 'open')", + line_width: 40, + expected_sql: "INSERT INTO orders\nVALUES\n (1, 'paid and shipped to the customer by priority courier'),\n (2, 'open')", + }, + test_types::LayoutRuleTestCase { + description: "Snowflake CAST breaks around IFF, not inside NUMBER(38, 2)", + dialect: "snowflake", + sql: "SELECT CAST(IFF(amount > 100000000, amount * 1000000000000, amount / 100000000000000) AS NUMBER(38, 2)) AS total FROM orders", + line_width: 80, + expected_sql: "SELECT\n CAST(\n IFF(\n amount > 100000000,\n amount * 1000000000000,\n amount / 100000000000000\n ) AS NUMBER(38, 2)\n ) AS total\nFROM orders", + }, + test_types::LayoutRuleTestCase { + description: "a DuckDB :: cast keeps DECIMAL(18, 4) whole", + dialect: "duckdb", + sql: "SELECT COALESCE(first_order_amount, second_order_amount)::DECIMAL(18, 4) AS amount FROM orders", + line_width: 50, + expected_sql: "SELECT\n COALESCE(\n first_order_amount,\n second_order_amount\n )::DECIMAL(18, 4) AS amount\nFROM orders", + }, + test_types::LayoutRuleTestCase { + description: "a BigQuery NUMERIC(38, 9) cast keeps its parameters", + dialect: "bigquery", + sql: "SELECT CAST(COALESCE(first_order_amount, second_order_amount) AS NUMERIC(38, 9)) AS amount FROM orders", + line_width: 50, + expected_sql: "SELECT\n CAST(\n COALESCE(\n first_order_amount,\n second_order_amount\n ) AS NUMERIC(38, 9)\n ) AS amount\nFROM orders", + }, + test_types::LayoutRuleTestCase { + description: "a T-SQL VARCHAR(MAX) cast keeps its parameter", + dialect: "tsql", + sql: "SELECT CAST(COALESCE(first_order_note, second_order_note) AS VARCHAR(MAX)) AS note FROM orders", + line_width: 50, + expected_sql: "SELECT\n CAST(\n COALESCE(\n first_order_note,\n second_order_note\n ) AS VARCHAR(MAX)\n ) AS note\nFROM orders", + }, + test_types::LayoutRuleTestCase { + description: "a Databricks DECIMAL(10, 2) cast keeps its parameters", + dialect: "databricks", + sql: "SELECT CAST(COALESCE(first_order_amount, second_order_amount) AS DECIMAL(10, 2)) AS amount FROM orders", + line_width: 50, + expected_sql: "SELECT\n CAST(\n COALESCE(\n first_order_amount,\n second_order_amount\n ) AS DECIMAL(10, 2)\n ) AS amount\nFROM orders", + }, + test_types::LayoutRuleTestCase { + description: "a long Snowflake DECODE puts each pair and the default on its own line", + dialect: "snowflake", + sql: "SELECT DECODE(status, 1, 'new', 2, 'open', 3, 'paid', 4, 'shipped', 5, 'cancelled', 'unknown') AS status_name FROM orders", + line_width: 60, + expected_sql: "SELECT\n DECODE(\n status,\n 1, 'new',\n 2, 'open',\n 3, 'paid',\n 4, 'shipped',\n 5, 'cancelled',\n 'unknown'\n ) AS status_name\nFROM orders", + }, + test_types::LayoutRuleTestCase { + description: "a DECODE without a default ends with a pair", + dialect: "snowflake", + sql: "SELECT DECODE(status, 1, 'new', 2, 'open', 3, 'paid', 4, 'shipped', 5, 'cancelled') AS status_name FROM orders", + line_width: 60, + expected_sql: "SELECT\n DECODE(\n status,\n 1, 'new',\n 2, 'open',\n 3, 'paid',\n 4, 'shipped',\n 5, 'cancelled'\n ) AS status_name\nFROM orders", + }, + test_types::LayoutRuleTestCase { + description: "a DECODE the layout guide breaks is regrouped into pairs", + dialect: "snowflake", + sql: "SELECT DECODE(status, 1, 'new', 2, 'open', 3, 'paid', 4, 'shipped', 5, 'cancelled', 6, 'returned', 'unknown') AS status_name FROM orders", + line_width: 100, + expected_sql: "SELECT\n DECODE(\n status,\n 1, 'new',\n 2, 'open',\n 3, 'paid',\n 4, 'shipped',\n 5, 'cancelled',\n 6, 'returned',\n 'unknown'\n ) AS status_name\nFROM orders", + }, + test_types::LayoutRuleTestCase { + description: "a short DECODE stays on one line", + dialect: "snowflake", + sql: "SELECT DECODE(status, 1, 'new', 'other') AS status_name FROM orders", + line_width: 100, + expected_sql: "SELECT DECODE(status, 1, 'new', 'other') AS status_name\nFROM orders", + }, + test_types::LayoutRuleTestCase { + description: "an operator after a multi-line CASE starts the next line", + dialect: "duckdb", + sql: "SELECT CASE WHEN status = 'paid' THEN 'customer paid in full' WHEN status = 'open' THEN 'customer has not paid yet' ELSE 'other' END || CASE WHEN rush THEN '!' ELSE '' END AS label FROM orders", + line_width: 100, + expected_sql: "SELECT\n CASE\n WHEN status = 'paid'\n THEN 'customer paid in full'\n WHEN status = 'open'\n THEN 'customer has not paid yet'\n ELSE 'other'\n END\n || CASE WHEN rush THEN '!' ELSE '' END AS label\nFROM orders", + }, + test_types::LayoutRuleTestCase { + description: "AND after a multi-line subquery starts the next line", + dialect: "postgres", + sql: "SELECT order_id FROM orders WHERE customer_id IN (SELECT customer_id FROM customers WHERE region = 'north') AND EXISTS (SELECT 1 FROM shipments)", + line_width: 100, + expected_sql: "SELECT order_id\nFROM orders\nWHERE\n customer_id IN (\n SELECT customer_id\n FROM customers\n WHERE region = 'north'\n )\n AND EXISTS(\n SELECT 1\n FROM shipments\n )", + }, + test_types::LayoutRuleTestCase { + description: "BETWEEN bounds keep their AND", + dialect: "snowflake", + sql: "SELECT order_id FROM orders WHERE amount BETWEEN 1 AND 9", + line_width: 100, + expected_sql: "SELECT order_id\nFROM orders\nWHERE amount BETWEEN 1 AND 9", + }, + test_types::LayoutRuleTestCase { + description: "CTEs and the final statement are separated by one blank line", + dialect: "duckdb", + sql: "with orders as (select * from raw_orders),\n\n\n-- paid only\npaid as (select * from orders where amount > 0)\n\n\nselect * from paid", + line_width: 100, + expected_sql: "WITH orders AS (\n SELECT *\n FROM raw_orders\n),\n\n-- paid only\npaid AS (\n SELECT *\n FROM orders\n WHERE amount > 0\n)\n\nSELECT *\nFROM paid", + }, + test_types::LayoutRuleTestCase { + description: "a trailing comment after a CTE comma keeps its line", + dialect: "snowflake", + sql: "WITH orders AS (SELECT * FROM raw_orders), -- raw\npaid AS (SELECT * FROM orders) SELECT * FROM paid", + line_width: 100, + expected_sql: "WITH orders AS (\n SELECT *\n FROM raw_orders\n), -- raw\n\npaid AS (\n SELECT *\n FROM orders\n)\n\nSELECT *\nFROM paid", + }, + test_types::LayoutRuleTestCase { + description: "a nested WITH is separated at its own indentation", + dialect: "postgres", + sql: "SELECT * FROM (WITH a AS MATERIALIZED (SELECT 1 AS x), b AS (SELECT x FROM a) SELECT x FROM b) AS nested", + line_width: 100, + expected_sql: "SELECT *\nFROM (\n WITH a AS MATERIALIZED (\n SELECT 1 AS x\n ),\n\n b AS (\n SELECT x\n FROM a\n )\n\n SELECT x\n FROM b\n) AS nested", + }, + test_types::LayoutRuleTestCase { + description: "BigQuery recursive CTEs with column lists are separated", + dialect: "bigquery", + sql: "WITH RECURSIVE numbers (n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM numbers WHERE n < 3), doubled AS (SELECT n * 2 AS n FROM numbers) SELECT n FROM doubled", + line_width: 100, + expected_sql: "WITH RECURSIVE numbers (n) AS (\n SELECT 1\n UNION ALL\n SELECT n + 1\n FROM numbers\n WHERE n < 3\n),\n\ndoubled AS (\n SELECT n * 2 AS n\n FROM numbers\n)\n\nSELECT n\nFROM doubled", + }, + test_types::LayoutRuleTestCase { + description: "a T-SQL WITH is separated before INSERT", + dialect: "tsql", + sql: "WITH paid AS (SELECT order_id FROM orders) INSERT INTO paid_orders SELECT order_id FROM paid", + line_width: 100, + expected_sql: "WITH paid AS (\n SELECT order_id\n FROM orders\n)\n\nINSERT INTO paid_orders\nSELECT order_id\nFROM paid", + }, + test_types::LayoutRuleTestCase { + description: "a CTE macro placeholder is separated like a CTE", + dialect: "duckdb", + sql: "WITH __sqb_lint_0__, paid AS (SELECT * FROM orders) SELECT * FROM paid", + line_width: 100, + expected_sql: "WITH\n__sqb_lint_0__,\n\npaid AS (\n SELECT *\n FROM orders\n)\n\nSELECT *\nFROM paid", + }, + test_types::LayoutRuleTestCase { + description: "an alias and comma that push a call over the width break the call", + dialect: "snowflake", + sql: "SELECT ARRAY_CONTAINS('web'::VARIANT, COALESCE(o.order_channels, ARRAY_CONSTRUCT())) AS has_web_channel, o.order_id FROM orders o", + line_width: 80, + expected_sql: "SELECT\n ARRAY_CONTAINS(\n 'web'::VARIANT,\n COALESCE(o.order_channels, ARRAY_CONSTRUCT())\n ) AS has_web_channel,\n o.order_id\nFROM orders o", + }, + test_types::LayoutRuleTestCase { + description: "an AND chain inside a call argument joins its line, then wraps", + dialect: "snowflake", + sql: "SELECT CAST(MAX(IFF(line_number = 1 AND shipped_after_days = 960, order_amount, NULL)) AS NUMBER(38, 10)) AS first_line_amount_shipped_late FROM orders", + line_width: 100, + expected_sql: "SELECT\n CAST(\n MAX(IFF(line_number = 1 AND shipped_after_days = 960, order_amount, NULL)) AS NUMBER(38, 10)\n ) AS first_line_amount_shipped_late\nFROM orders", + }, + test_types::LayoutRuleTestCase { + description: "a one-line CASE over the width breaks at its branches", + dialect: "duckdb", + sql: "SELECT CASE WHEN o.shipped_at IS NULL THEN NULL WHEN o.refund_amount IS NULL THEN 0 ELSE 1 END AS has_refund FROM orders o", + line_width: 80, + expected_sql: "SELECT\n CASE\n WHEN o.shipped_at IS NULL\n THEN NULL\n WHEN o.refund_amount IS NULL\n THEN 0\n ELSE 1\n END AS has_refund\nFROM orders o", + }, + test_types::LayoutRuleTestCase { + description: "a WITHIN GROUP line breaks the aggregate call", + dialect: "snowflake", + sql: "SELECT customer_id, LISTAGG(DISTINCT LOWER(product_category_name), ', ') WITHIN GROUP (ORDER BY product_category_name) AS categories FROM orders GROUP BY customer_id", + line_width: 80, + expected_sql: "SELECT\n customer_id,\n LISTAGG(\n DISTINCT LOWER(product_category_name),\n ', '\n ) WITHIN GROUP (ORDER BY product_category_name) AS categories\nFROM orders\nGROUP BY\n customer_id", + }, + test_types::LayoutRuleTestCase { + description: "a window that continues on later lines breaks at its clauses", + dialect: "snowflake", + sql: "SELECT ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY CASE o.status WHEN 'paid' THEN 0 WHEN 'open' THEN 1 ELSE 2 END, o.ordered_at DESC) AS rn FROM orders o", + line_width: 60, + expected_sql: "SELECT\n ROW_NUMBER() OVER (\n PARTITION BY o.customer_id\n ORDER BY CASE o.status\n WHEN 'paid'\n THEN 0\n WHEN 'open'\n THEN 1\n ELSE 2\n END, o.ordered_at DESC\n ) AS rn\nFROM orders o", + }, + test_types::LayoutRuleTestCase { + description: "SELECT * and a single WHERE condition share their clause lines", + dialect: "duckdb", + sql: "select * from orders where amount > 0", + line_width: 100, + expected_sql: "SELECT *\nFROM orders\nWHERE amount > 0", + }, + test_types::LayoutRuleTestCase { + description: "SELECT DISTINCT keeps its single item, several items break", + dialect: "snowflake", + sql: "SELECT DISTINCT order_id FROM orders WHERE amount > 0 AND status = 'paid' UNION ALL SELECT order_id, customer_id FROM returns", + line_width: 100, + expected_sql: "SELECT DISTINCT order_id\nFROM orders\nWHERE\n amount > 0\n AND status = 'paid'\nUNION ALL\nSELECT\n order_id,\n customer_id\nFROM returns", + }, + test_types::LayoutRuleTestCase { + description: "T-SQL TOP, HAVING and Snowflake-style single items share lines", + dialect: "tsql", + sql: "SELECT TOP 10 COUNT(*) AS n FROM orders GROUP BY customer_id HAVING COUNT(*) > 1", + line_width: 100, + expected_sql: "SELECT TOP 10 COUNT(*) AS n\nFROM orders\nGROUP BY\n customer_id\nHAVING COUNT(*) > 1", + }, + test_types::LayoutRuleTestCase { + description: "QUALIFY with one condition shares its line", + dialect: "databricks", + sql: "SELECT order_id, ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at) AS rn FROM orders QUALIFY rn = 1", + line_width: 100, + expected_sql: "SELECT\n order_id,\n ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at) AS rn\nFROM orders\nQUALIFY rn = 1", + }, + test_types::LayoutRuleTestCase { + description: "a single item that does not fit stays under SELECT", + dialect: "bigquery", + sql: "SELECT COALESCE(first_order_amount, second_order_amount, third_order_amount) AS amount FROM orders WHERE first_order_amount + second_order_amount + third_order_amount > 100", + line_width: 60, + expected_sql: "SELECT\n COALESCE(\n first_order_amount,\n second_order_amount,\n third_order_amount\n ) AS amount\nFROM orders\nWHERE\n first_order_amount\n + second_order_amount\n + third_order_amount > 100", + }, + test_types::LayoutRuleTestCase { + description: "a short CASE item shares the SELECT line", + dialect: "duckdb", + sql: "SELECT CASE WHEN amount > 0 THEN 'paid' ELSE 'open' END AS status FROM orders", + line_width: 100, + expected_sql: "SELECT CASE WHEN amount > 0 THEN 'paid' ELSE 'open' END AS status\nFROM orders", + }, + test_types::LayoutRuleTestCase { + description: "a multi-line single item stays under SELECT", + dialect: "duckdb", + sql: "SELECT CASE WHEN amount > 1000 THEN 'large paid order' WHEN amount > 0 THEN 'paid order' ELSE 'open order' END AS status FROM orders", + line_width: 100, + expected_sql: "SELECT\n CASE\n WHEN amount > 1000\n THEN 'large paid order'\n WHEN amount > 0\n THEN 'paid order'\n ELSE 'open order'\n END AS status\nFROM orders", + }, + ]; + for test_case in &test_cases { + let response: Value = + format_at_width(test_case.sql, test_case.dialect, test_case.line_width)?; + assert_eq!(response["formatted"], true, "{}", test_case.description); + assert_eq!( + response["sql"], test_case.expected_sql, + "{}", + test_case.description + ); + assert_eq!( + query_fingerprint(test_case.expected_sql, test_case.dialect)?, + query_fingerprint(test_case.sql, test_case.dialect)?, + "{}: layout changed the fingerprint", + test_case.description + ); + let again: Value = format_at_width( + test_case.expected_sql, + test_case.dialect, + test_case.line_width, + )?; + assert_eq!(again["changed"], false, "{}", test_case.description); + } + Ok(()) +} diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_line_wrap.rs b/crates/sqlbuild-rules-native/src/sql_lint/tests/test_line_wrap.rs new file mode 100644 index 0000000000..3ae97ec83d --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_lint/tests/test_line_wrap.rs @@ -0,0 +1,107 @@ +use std::collections::HashMap; + +use serde_json::{Value, json}; + +use crate::sql_lint::main::formatter::format_json; +use crate::sql_lint::tests::test_types; + +#[test] +fn given_lines_over_the_width_when_formatting_then_each_wrap_rule_applies_idempotently() +-> Result<(), String> { + let test_cases = [ + test_types::LineWrapTestCase { + description: "function arguments go one per line", + sql: "select coalesce(first_order_amount, second_order_amount, third_order_amount, fallback_amount) as amount from orders", + line_width: 60, + token_widths: &[], + expected_sql: "SELECT\n COALESCE(\n first_order_amount,\n second_order_amount,\n third_order_amount,\n fallback_amount\n ) AS amount\nFROM orders", + }, + test_types::LineWrapTestCase { + description: "IN lists go one element per line", + sql: "select order_id from orders where region in ('north', 'south', 'east', 'west', 'central', 'remote')", + line_width: 50, + token_widths: &[], + expected_sql: "SELECT order_id\nFROM orders\nWHERE\n region IN (\n 'north',\n 'south',\n 'east',\n 'west',\n 'central',\n 'remote'\n )", + }, + test_types::LineWrapTestCase { + description: "window PARTITION BY and ORDER BY go on separate lines", + sql: "select row_number() over (partition by customer_id, region_code order by ordered_at desc, order_id desc) as rank_in_customer from orders", + line_width: 60, + token_widths: &[], + expected_sql: "SELECT\n ROW_NUMBER() OVER (\n PARTITION BY customer_id, region_code\n ORDER BY ordered_at DESC, order_id DESC\n ) AS rank_in_customer\nFROM orders", + }, + test_types::LineWrapTestCase { + description: "join conditions break before AND but not inside BETWEEN", + sql: "select o.order_id from orders o join customers c on o.customer_id = c.customer_id and o.amount between 1 and 9 and o.store = c.store", + line_width: 60, + token_widths: &[], + expected_sql: "SELECT o.order_id\nFROM orders o\nJOIN customers c\n ON o.customer_id = c.customer_id\n AND o.amount BETWEEN 1 AND 9\n AND o.store = c.store", + }, + test_types::LineWrapTestCase { + description: "arithmetic chains break before operators", + sql: "select first_amount + second_amount + third_amount + fourth_amount as total from orders", + line_width: 50, + token_widths: &[], + expected_sql: "SELECT\n first_amount\n + second_amount\n + third_amount\n + fourth_amount AS total\nFROM orders", + }, + test_types::LineWrapTestCase { + description: "a top-level AND chain breaks before a nested group", + sql: "select order_id from orders where cast(ordered_at as DATE) >= cast(@start as DATE) and cast(ordered_at as DATE) < cast(@end as DATE)", + line_width: 60, + token_widths: &[], + expected_sql: "SELECT order_id\nFROM orders\nWHERE\n CAST(ordered_at AS DATE) >= CAST(@start AS DATE)\n AND CAST(ordered_at AS DATE) < CAST(@end AS DATE)", + }, + test_types::LineWrapTestCase { + description: "a long string literal is never broken", + sql: "select 'a long literal that is wider than the configured line width' as note from orders", + line_width: 40, + token_widths: &[], + expected_sql: "SELECT\n 'a long literal that is wider than the configured line width' AS note\nFROM orders", + }, + test_types::LineWrapTestCase { + description: "a line with a comment is never broken", + sql: "select coalesce(first_amount, second_amount, third_amount) as amount -- keep\nfrom orders", + line_width: 40, + token_widths: &[], + expected_sql: "SELECT\n COALESCE(first_amount, second_amount, third_amount) AS amount -- keep\nFROM orders", + }, + test_types::LineWrapTestCase { + description: "a sentinel counts with the width of the text it stands for", + sql: "select coalesce(__sqb_lint_0__, b) as amount from orders", + line_width: 40, + token_widths: &[("__SQB_LINT_0__", 30)], + expected_sql: "SELECT\n COALESCE(\n __sqb_lint_0__,\n b\n ) AS amount\nFROM orders", + }, + test_types::LineWrapTestCase { + description: "lines within the width are unchanged", + sql: "select coalesce(first_amount, second_amount) as amount from orders", + line_width: 100, + token_widths: &[], + expected_sql: "SELECT COALESCE(first_amount, second_amount) AS amount\nFROM orders", + }, + ]; + for test_case in &test_cases { + let token_widths: HashMap<&str, usize> = test_case.token_widths.iter().copied().collect(); + let request = |sql: &str| { + json!({ + "version": 1, + "sql": sql, + "dialect": "duckdb", + "line_width": test_case.line_width, + "token_widths": token_widths, + }) + .to_string() + }; + let response: Value = serde_json::from_str(&format_json(&request(test_case.sql))?) + .map_err(|error| error.to_string())?; + assert_eq!( + response["sql"], test_case.expected_sql, + "{}", + test_case.description + ); + let again: Value = serde_json::from_str(&format_json(&request(test_case.expected_sql))?) + .map_err(|error| error.to_string())?; + assert_eq!(again["changed"], false, "{}", test_case.description); + } + Ok(()) +} diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_sql_lint.rs b/crates/sqlbuild-rules-native/src/sql_lint/tests/test_sql_lint.rs index 5501eeb3c9..971222a13d 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_sql_lint.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/tests/test_sql_lint.rs @@ -1812,13 +1812,13 @@ fn given_format_cases_when_formatting_then_output_matches() -> Result<(), String test_types::FormatTestCase { description: "comment-free canonical SQL", sql: "select a,b from items where a=1", - expected_sql: "SELECT\n a,\n b\nFROM items\nWHERE\n a = 1", + expected_sql: "SELECT\n a,\n b\nFROM items\nWHERE a = 1", expected_changed: true, }, test_types::FormatTestCase { description: "lowercase function canonicalization preserves structure", sql: "select count(1) from items", - expected_sql: "SELECT\n COUNT(1)\nFROM items", + expected_sql: "SELECT COUNT(1)\nFROM items", expected_changed: true, }, test_types::FormatTestCase { @@ -1830,7 +1830,7 @@ fn given_format_cases_when_formatting_then_output_matches() -> Result<(), String test_types::FormatTestCase { description: "line and leading comments", sql: "-- lead\nselect a,b from items -- tail\nwhere a=1", - expected_sql: "-- lead\nSELECT\n a,\n b\nFROM items -- tail\nWHERE\n a = 1", + expected_sql: "-- lead\nSELECT\n a,\n b\nFROM items -- tail\nWHERE a = 1", expected_changed: true, }, test_types::FormatTestCase { @@ -1842,19 +1842,19 @@ fn given_format_cases_when_formatting_then_output_matches() -> Result<(), String test_types::FormatTestCase { description: "comment marker string", sql: "select '-- not a comment' as value", - expected_sql: "SELECT\n '-- not a comment' AS value", + expected_sql: "SELECT '-- not a comment' AS value", expected_changed: true, }, test_types::FormatTestCase { description: "doubled apostrophe string remains compiler compatible", sql: "select 'Customer''s order' as value", - expected_sql: "SELECT\n 'Customer''s order' AS value", + expected_sql: "SELECT 'Customer''s order' AS value", expected_changed: true, }, test_types::FormatTestCase { description: "explicit null ordering remains explicit", sql: "select id from items order by category nulls last, created_at desc nulls first", - expected_sql: "SELECT\n id\nFROM items\nORDER BY\n category NULLS LAST,\n created_at DESC NULLS FIRST", + expected_sql: "SELECT id\nFROM items\nORDER BY\n category NULLS LAST,\n created_at DESC NULLS FIRST", expected_changed: true, }, test_types::FormatTestCase { @@ -1878,19 +1878,19 @@ fn given_format_cases_when_formatting_then_output_matches() -> Result<(), String test_types::FormatTestCase { description: "cast-free token expansion remains supported", sql: "select id label from items", - expected_sql: "SELECT\n id AS label\nFROM items", + expected_sql: "SELECT id label\nFROM items", expected_changed: true, }, test_types::FormatTestCase { description: "cast-free multiple statements remain supported", sql: "select 1; select 2", - expected_sql: "SELECT\n 1;\nSELECT\n 2", + expected_sql: "SELECT 1;\nSELECT 2", expected_changed: true, }, test_types::FormatTestCase { - description: "trailing comment", + description: "trailing comment after a kept statement terminator", sql: "select a from items; -- retained", - expected_sql: "SELECT\n a\nFROM items -- retained\n", + expected_sql: "SELECT a\nFROM items; -- retained\n", expected_changed: true, }, test_types::FormatTestCase { @@ -1927,13 +1927,13 @@ fn given_ordered_sql_batch_when_formatting_then_outputs_are_parallel_and_ordered test_types::FormatTestCase { description: "first order query", sql: "select order_id from orders where order_id=1", - expected_sql: "SELECT\n order_id\nFROM orders\nWHERE\n order_id = 1", + expected_sql: "SELECT order_id\nFROM orders\nWHERE order_id = 1", expected_changed: true, }, test_types::FormatTestCase { description: "second product query", sql: "select product_id from products where product_id=2", - expected_sql: "SELECT\n product_id\nFROM products\nWHERE\n product_id = 2", + expected_sql: "SELECT product_id\nFROM products\nWHERE product_id = 2", expected_changed: true, }, ]; diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_types.rs b/crates/sqlbuild-rules-native/src/sql_lint/tests/test_types.rs index bfc72df7c0..0235c87b20 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_types.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/tests/test_types.rs @@ -79,6 +79,52 @@ pub(crate) struct DialectLintRuleTestCase { pub expected_reason: &'static str, } +pub(crate) struct LineWrapTestCase { + pub description: &'static str, + pub sql: &'static str, + pub line_width: usize, + pub token_widths: &'static [(&'static str, usize)], + pub expected_sql: &'static str, +} + +pub(crate) struct FunctionCallSpacingTestCase { + pub description: &'static str, + pub dialect: &'static str, + pub line_width: usize, + pub argument: &'static str, + pub expected_refused: &'static [&'static str], +} + +pub(crate) struct LayoutRuleTestCase { + pub description: &'static str, + pub dialect: &'static str, + pub sql: &'static str, + pub line_width: usize, + pub expected_sql: &'static str, +} + +pub(crate) struct FormatCorpusTestCase { + pub description: &'static str, + pub dialect: &'static str, + pub expected_formatted: usize, + pub expected_refused: usize, +} + +pub(crate) struct DialectFormatTestCase { + pub description: &'static str, + pub dialect: &'static str, + pub sql: &'static str, + pub expected_sql: &'static str, +} + +pub(crate) struct TokenInvariantTestCase { + pub description: &'static str, + pub before: &'static str, + pub after: &'static str, + pub recases: &'static [(usize, &'static str, bool)], + pub expected_error: Option<&'static str>, +} + pub(crate) struct FormatTestCase { pub description: &'static str, pub sql: &'static str, @@ -104,11 +150,3 @@ pub(crate) struct FunctionDepthFailureTestCase { pub expected_code: &'static str, pub expected_limit: &'static str, } - -pub(crate) struct AuthoredTokenTestCase { - pub description: &'static str, - pub authored: &'static str, - pub generated: &'static str, - /// `None` when the formatter must refuse the change. - pub expected_sql: Option<&'static str>, -} diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_values_formatter.rs b/crates/sqlbuild-rules-native/src/sql_lint/tests/test_values_formatter.rs index b71fe80394..6963aadf4b 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_values_formatter.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/tests/test_values_formatter.rs @@ -10,25 +10,25 @@ fn given_mixed_values_relations_when_formatting_then_each_authored_form_is_prese test_types::FormatTestCase { description: "mixed values relations preserve parenthesized relation first", sql: "SELECT * FROM (VALUES (1, 2)) AS first_values(a, b) UNION ALL SELECT * FROM VALUES (3, 4)", - expected_sql: "SELECT\n *\nFROM (VALUES (1, 2)) AS first_values(a, b)\nUNION ALL\nSELECT\n *\nFROM VALUES (3, 4)", + expected_sql: "SELECT *\nFROM (VALUES (1, 2)) AS first_values(a, b)\nUNION ALL\nSELECT *\nFROM VALUES (3, 4)", expected_changed: true, }, test_types::FormatTestCase { description: "mixed values relations preserve unparenthesized relation first", sql: "SELECT * FROM VALUES (1, 2) UNION ALL SELECT * FROM (VALUES (3, 4)) AS second_values(a, b)", - expected_sql: "SELECT\n *\nFROM VALUES (1, 2)\nUNION ALL\nSELECT\n *\nFROM (VALUES (3, 4)) AS second_values(a, b)", + expected_sql: "SELECT *\nFROM VALUES (1, 2)\nUNION ALL\nSELECT *\nFROM (VALUES (3, 4)) AS second_values(a, b)", expected_changed: true, }, test_types::FormatTestCase { description: "mixed values relations inside joined subquery preserve authored forms", sql: "SELECT base.id FROM items AS base JOIN (SELECT * FROM (VALUES (1)) AS first_values(id) UNION ALL SELECT * FROM VALUES (2)) AS mixed ON base.id = mixed.id", - expected_sql: "SELECT\n base.id\nFROM items AS base\nJOIN (\n SELECT\n *\n FROM (VALUES (1)) AS first_values(id)\n UNION ALL\n SELECT\n *\n FROM VALUES (2)\n) AS mixed\n ON base.id = mixed.id", + expected_sql: "SELECT base.id\nFROM items AS base\nJOIN (\n SELECT *\n FROM (VALUES (1)) AS first_values(id)\n UNION ALL\n SELECT *\n FROM VALUES (2)\n) AS mixed\n ON base.id = mixed.id", expected_changed: true, }, test_types::FormatTestCase { description: "values text in string and comment is ignored", sql: "-- VALUES and FROM VALUES are examples\nSELECT 'VALUES' AS label FROM (VALUES (1)) AS first_values(id) UNION ALL SELECT * FROM VALUES (2)", - expected_sql: "-- VALUES and FROM VALUES are examples\nSELECT\n 'VALUES' AS label\nFROM (VALUES (1)) AS first_values(id)\nUNION ALL\nSELECT\n *\nFROM VALUES (2)", + expected_sql: "-- VALUES and FROM VALUES are examples\nSELECT 'VALUES' AS label\nFROM (VALUES (1)) AS first_values(id)\nUNION ALL\nSELECT *\nFROM VALUES (2)", expected_changed: true, }, ]; diff --git a/crates/sqlbuild-rules-native/src/sql_scan/models.rs b/crates/sqlbuild-rules-native/src/sql_scan/models.rs index bba4289068..ee967e3b9c 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/models.rs +++ b/crates/sqlbuild-rules-native/src/sql_scan/models.rs @@ -30,12 +30,12 @@ impl QuotePolicy { dollar_quotes: true, }; - /// SQL lint base policy: backslashes escape; lint enables backticks per dialect. + /// SQL lint policy: backslashes escape, backticks per dialect, dollar quotes as in the compiler. pub(crate) const SQL_LINT: Self = Self { backtick_identifiers: false, single_quote_backslash_escapes: true, double_quote_backslash_escapes: true, - dollar_quotes: false, + dollar_quotes: true, }; /// Return this policy with backtick-delimited identifiers enabled or disabled. diff --git a/crates/sqlbuild-rules-native/src/sql_scan/tests/test_sql_scan.rs b/crates/sqlbuild-rules-native/src/sql_scan/tests/test_sql_scan.rs index 8079314db4..371063385b 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/tests/test_sql_scan.rs +++ b/crates/sqlbuild-rules-native/src/sql_scan/tests/test_sql_scan.rs @@ -72,9 +72,15 @@ fn given_quote_policies_when_matching_parentheses_then_policy_decides_quote_boun MatchingParenPolicyTestCase { description: "dollar quotes are code when the policy disables them", sql: "($$ ) $$ x) y", - policy: QuotePolicy::SQL_LINT, + policy: QUOTES_ONLY, expected_close: Ok(4), }, + MatchingParenPolicyTestCase { + description: "SQL lint policy skips dollar-quoted text like the compiler", + sql: "($$ ) $$ x) y", + policy: QuotePolicy::SQL_LINT, + expected_close: Ok(10), + }, MatchingParenPolicyTestCase { description: "multibyte text keeps byte offsets on character boundaries", sql: "('é)ü' ñ x) y", diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/_helpers/builtin_functions.rs b/crates/sqlbuild-rules-native/src/sql_tokens/_helpers/builtin_functions.rs new file mode 100644 index 0000000000..d1ca266e10 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/_helpers/builtin_functions.rs @@ -0,0 +1,151 @@ +//! Function names a dialect recognises as built-ins, from the SQL library's own metadata. + +use std::collections::{HashMap, HashSet}; +use std::sync::LazyLock; + +use crate::sql_tokens::constants::CALL_SYNTAX_FUNCTIONS; +use polyglot_sql::{Dialect, DialectType, Expression}; +use polyglot_sql_function_catalogs::types::{builtin_arity, type_signature}; +use polyglot_sql_function_catalogs::{ + CatalogSink, FunctionNameCase, FunctionSignature, register_enabled_catalogs, +}; + +const DUCKDB_CATALOG: &str = "duckdb"; + +static CATALOG_FUNCTIONS: LazyLock>> = LazyLock::new(|| { + let mut sink = NameSink::default(); + register_enabled_catalogs(&mut sink); + sink.names +}); +static SNOWFLAKE_TYPED: LazyLock> = + LazyLock::new(|| parser_typed_names(DialectType::Snowflake)); +static BIGQUERY_TYPED: LazyLock> = + LazyLock::new(|| parser_typed_names(DialectType::BigQuery)); +static POSTGRES_TYPED: LazyLock> = + LazyLock::new(|| parser_typed_names(DialectType::PostgreSQL)); +static DATABRICKS_TYPED: LazyLock> = + LazyLock::new(|| parser_typed_names(DialectType::Databricks)); +static TSQL_TYPED: LazyLock> = + LazyLock::new(|| parser_typed_names(DialectType::TSQL)); +static GENERIC_TYPED: LazyLock> = + LazyLock::new(|| parser_typed_names(DialectType::Generic)); + +/// Return whether `upper_name(...)` calls a function the dialect defines. +pub(crate) fn is_builtin_function(dialect: DialectType, upper_name: &str) -> bool { + if CALL_SYNTAX_FUNCTIONS.contains(&upper_name) { + return true; + } + let lower_name = upper_name.to_ascii_lowercase(); + let typed = match dialect { + DialectType::DuckDB => return catalogued(DUCKDB_CATALOG, &lower_name), + DialectType::Snowflake => &SNOWFLAKE_TYPED, + DialectType::BigQuery => &BIGQUERY_TYPED, + DialectType::PostgreSQL => &POSTGRES_TYPED, + DialectType::Databricks => &DATABRICKS_TYPED, + DialectType::TSQL => &TSQL_TYPED, + _ => &GENERIC_TYPED, + }; + typed.contains(&lower_name) || catalogued(catalog_key(dialect), &lower_name) +} + +/// Every catalogued or parser-typed built-in function name of `dialect`, lower-cased. +#[cfg(test)] +pub(crate) fn builtin_function_names(dialect: DialectType) -> Vec { + let typed: &HashSet = match dialect { + DialectType::DuckDB => &HashSet::new(), + DialectType::Snowflake => &SNOWFLAKE_TYPED, + DialectType::BigQuery => &BIGQUERY_TYPED, + DialectType::PostgreSQL => &POSTGRES_TYPED, + DialectType::Databricks => &DATABRICKS_TYPED, + DialectType::TSQL => &TSQL_TYPED, + _ => &GENERIC_TYPED, + }; + let key = match dialect { + DialectType::DuckDB => DUCKDB_CATALOG, + _ => catalog_key(dialect), + }; + let catalogued = CATALOG_FUNCTIONS.get(key).into_iter().flatten(); + let mut names: Vec = typed.iter().chain(catalogued).cloned().collect(); + names.sort_unstable(); + names.dedup(); + names +} + +fn catalog_key(dialect: DialectType) -> &'static str { + match dialect { + DialectType::Snowflake => "snowflake", + DialectType::BigQuery => "bigquery", + DialectType::PostgreSQL => "postgres", + _ => "", + } +} + +fn catalogued(key: &str, lower_name: &str) -> bool { + CATALOG_FUNCTIONS + .get(key) + .is_some_and(|names| names.contains(lower_name)) + || builtin_arity(key, lower_name).is_some() + || type_signature(key, lower_name).is_some() +} + +/// Catalogued function names whose call the dialect's parser turns into a specific expression. +fn parser_typed_names(dialect: DialectType) -> HashSet { + let parser = Dialect::get(dialect); + let mut typed: HashSet = HashSet::new(); + for name in CATALOG_FUNCTIONS.get(DUCKDB_CATALOG).into_iter().flatten() { + if is_typed_call(&parser, name) { + typed.insert(name.clone()); + } + } + typed +} + +fn is_typed_call(parser: &Dialect, lower_name: &str) -> bool { + if !lower_name + .chars() + .all(|character| character.is_ascii_alphanumeric() || character == '_') + { + return false; + } + let Ok(statements) = parser.parse(&format!("SELECT {lower_name}(probe)")) else { + return false; + }; + let Some(Expression::Select(select)) = statements.first() else { + return false; + }; + select.expressions.first().is_some_and(|call| { + !matches!( + call, + Expression::Function(_) | Expression::Anonymous(_) | Expression::Column(_) + ) + }) +} + +#[derive(Default)] +struct NameSink { + names: HashMap<&'static str, HashSet>, +} + +impl CatalogSink for NameSink { + fn set_dialect_name_case(&mut self, _dialect: &'static str, _name_case: FunctionNameCase) {} + + fn set_function_name_case( + &mut self, + _dialect: &'static str, + _function_name: &str, + _name_case: FunctionNameCase, + ) { + } + + fn register( + &mut self, + dialect: &'static str, + function_name: &str, + _signatures: Vec, + ) { + self.names + .entry(dialect) + .or_default() + .insert(function_name.to_ascii_lowercase()); + } +} diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/_helpers/mod.rs b/crates/sqlbuild-rules-native/src/sql_tokens/_helpers/mod.rs new file mode 100644 index 0000000000..8e8d072e5b --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod builtin_functions; +pub(crate) mod reserved_keywords; +pub(crate) mod token_context; diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/_helpers/reserved_keywords.rs b/crates/sqlbuild-rules-native/src/sql_tokens/_helpers/reserved_keywords.rs new file mode 100644 index 0000000000..ccf9004d8d --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/_helpers/reserved_keywords.rs @@ -0,0 +1,43 @@ +//! Per-dialect reserved keywords, which can never be unquoted identifiers in that dialect. + +use std::collections::HashSet; +use std::sync::LazyLock; + +use polyglot_sql::DialectType; + +static DUCKDB: LazyLock> = + LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/duckdb.txt"))); +static POSTGRES: LazyLock> = + LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/postgres.txt"))); +static BIGQUERY: LazyLock> = + LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/bigquery.txt"))); +static SNOWFLAKE: LazyLock> = + LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/snowflake.txt"))); +static TSQL: LazyLock> = + LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/tsql.txt"))); +static DATABRICKS: LazyLock> = + LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/databricks.txt"))); +static GENERIC: LazyLock> = + LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/generic.txt"))); + +/// Return whether the upper-cased word is a reserved keyword of `dialect`. +pub(crate) fn is_reserved_keyword(dialect: DialectType, upper_word: &str) -> bool { + let keywords: &HashSet = match dialect { + DialectType::DuckDB => &DUCKDB, + DialectType::PostgreSQL => &POSTGRES, + DialectType::BigQuery => &BIGQUERY, + DialectType::Snowflake => &SNOWFLAKE, + DialectType::TSQL => &TSQL, + DialectType::Databricks => &DATABRICKS, + _ => &GENERIC, + }; + keywords.contains(upper_word) +} + +fn keyword_set(data: &str) -> HashSet { + data.lines() + .map(str::trim) + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .map(str::to_ascii_uppercase) + .collect() +} diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/_helpers/token_context.rs b/crates/sqlbuild-rules-native/src/sql_tokens/_helpers/token_context.rs new file mode 100644 index 0000000000..fabb747d27 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/_helpers/token_context.rs @@ -0,0 +1,30 @@ +//! Positional context of one authored token among its neighbours. + +use polyglot_sql::DialectType; + +use crate::sql_tokens::_helpers::builtin_functions::is_builtin_function; +use crate::sql_tokens::_helpers::reserved_keywords::is_reserved_keyword; +use crate::sql_tokens::constants::CALL_OPENER; +use crate::sql_tokens::main::is_unquoted_word::is_unquoted_word; + +pub(crate) struct TokenContext<'a> { + pub(crate) raws: &'a [String], + pub(crate) uppers: &'a [String], + pub(crate) index: usize, +} + +impl TokenContext<'_> { + /// Whether the token is a reserved keyword or a built-in function name at a call site. + pub(crate) fn is_foldable(&self, dialect: DialectType) -> bool { + if !is_unquoted_word(&self.raws[self.index]) { + return false; + } + let upper = self.uppers[self.index].as_str(); + is_reserved_keyword(dialect, upper) + || (self.next() == Some(CALL_OPENER) && is_builtin_function(dialect, upper)) + } + + fn next(&self) -> Option<&str> { + self.uppers.get(self.index + 1).map(String::as_str) + } +} diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/constants.rs b/crates/sqlbuild-rules-native/src/sql_tokens/constants.rs new file mode 100644 index 0000000000..8a2cb3012d --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/constants.rs @@ -0,0 +1,17 @@ +//! Token positions that decide which authored words may change letter case. + +/// Column aliases after `AS` may be any keyword in DuckDB and PostgreSQL. +pub(crate) const ALIAS_KEYWORD: &str = "AS"; +/// The token that turns a preceding name into a function call. +pub(crate) const CALL_OPENER: &str = "("; +/// Field names after `.`, path keys after `:`, and parameter names after `@` or `$`. +pub(crate) const NAME_PREFIXES: [&str; 4] = [".", ":", "@", "$"]; +/// Built-in calls with keyword syntax inside their parentheses, which catalogues do not list. +pub(crate) const CALL_SYNTAX_FUNCTIONS: [&str; 6] = [ + "CAST", + "TRY_CAST", + "SAFE_CAST", + "EXTRACT", + "CONVERT", + "TRY_CONVERT", +]; diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/main/canonical_tokens.rs b/crates/sqlbuild-rules-native/src/sql_tokens/main/canonical_tokens.rs new file mode 100644 index 0000000000..d921e2a80a --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/main/canonical_tokens.rs @@ -0,0 +1,25 @@ +//! Canonical token stream of SQL text under one dialect tokenizer. + +use polyglot_sql::Dialect; +use polyglot_sql::tokens::Token; + +use crate::sql_tokens::main::case_folding::foldable_tokens; +use crate::sql_tokens::main::token_texts::token_texts; +use crate::sql_tokens::models::CanonicalToken; + +/// Drop whitespace and comments from tokenized `sql` and upper-case case-insensitive tokens. +pub(crate) fn canonical_tokens( + sql: &str, + tokens: &[Token], + dialect: &Dialect, +) -> Result, String> { + let raws = token_texts(sql, tokens)?; + let foldable = foldable_tokens(&raws, tokens, dialect); + Ok(raws + .into_iter() + .zip(foldable) + .map(|(raw, fold)| CanonicalToken { + text: if fold { raw.to_ascii_uppercase() } else { raw }, + }) + .collect()) +} diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/main/case_folding.rs b/crates/sqlbuild-rules-native/src/sql_tokens/main/case_folding.rs new file mode 100644 index 0000000000..46645f9c08 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/main/case_folding.rs @@ -0,0 +1,23 @@ +//! Which authored tokens are case-insensitive keywords or built-in function names. + +use polyglot_sql::Dialect; +use polyglot_sql::tokens::Token; + +use crate::sql_tokens::_helpers::token_context::TokenContext; +use crate::sql_tokens::main::name_positions::name_positions; + +/// Return, for every token, whether the formatter may recase it and the fingerprint folds it. +pub(crate) fn foldable_tokens(raws: &[String], tokens: &[Token], dialect: &Dialect) -> Vec { + let uppers: Vec = raws.iter().map(|raw| raw.to_ascii_uppercase()).collect(); + let names = name_positions(&uppers); + let mut foldable = Vec::with_capacity(tokens.len()); + for (index, name) in names.iter().enumerate().take(tokens.len()) { + let context = TokenContext { + raws, + uppers: &uppers, + index, + }; + foldable.push(!name && context.is_foldable(dialect.dialect_type())); + } + foldable +} diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/main/is_unquoted_word.rs b/crates/sqlbuild-rules-native/src/sql_tokens/main/is_unquoted_word.rs new file mode 100644 index 0000000000..c1c40c63ad --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/main/is_unquoted_word.rs @@ -0,0 +1,11 @@ +//! Unquoted SQL word recognition shared by canonical tokens and the formatter. + +/// Return whether `text` is an unquoted ASCII word whose letter case carries no meaning. +pub(crate) fn is_unquoted_word(text: &str) -> bool { + let mut characters = text.chars(); + characters + .next() + .is_some_and(|first| first.is_ascii_alphabetic() || first == '_') + && characters + .all(|character| character.is_ascii_alphanumeric() || matches!(character, '_' | '$')) +} diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/main/mod.rs b/crates/sqlbuild-rules-native/src/sql_tokens/main/mod.rs new file mode 100644 index 0000000000..f7a6b602b6 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/main/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod canonical_tokens; +pub(crate) mod case_folding; +pub(crate) mod is_unquoted_word; +pub(crate) mod name_positions; +pub(crate) mod query_fingerprint; +pub(crate) mod token_texts; diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/main/name_positions.rs b/crates/sqlbuild-rules-native/src/sql_tokens/main/name_positions.rs new file mode 100644 index 0000000000..91027cae60 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/main/name_positions.rs @@ -0,0 +1,16 @@ +//! Token positions where some dialect accepts even a reserved word as an unquoted name. + +use crate::sql_tokens::constants::{ALIAS_KEYWORD, NAME_PREFIXES}; + +/// Return, for every upper-cased token, whether it follows `.`, `:`, `@`, `$` or `AS`. +pub(crate) fn name_positions(uppers: &[String]) -> Vec { + let mut positions = Vec::with_capacity(uppers.len()); + let mut previous: Option<&str> = None; + for upper in uppers { + positions.push( + previous.is_some_and(|word| word == ALIAS_KEYWORD || NAME_PREFIXES.contains(&word)), + ); + previous = Some(upper.as_str()); + } + positions +} diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/main/query_fingerprint.rs b/crates/sqlbuild-rules-native/src/sql_tokens/main/query_fingerprint.rs new file mode 100644 index 0000000000..d162134f48 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/main/query_fingerprint.rs @@ -0,0 +1,25 @@ +//! Whitespace-, comment- and keyword-case-insensitive SQL change fingerprints. + +use std::str::FromStr; + +use polyglot_sql::{Dialect, DialectType}; +use sha2::{Digest, Sha256}; + +use crate::sql_tokens::main::canonical_tokens::canonical_tokens; + +/// Return the SHA-256 hex digest of the canonical token stream of `sql` under `dialect_name`. +pub(crate) fn query_fingerprint(sql: &str, dialect_name: &str) -> Result { + let dialect_type = DialectType::from_str(dialect_name) + .map_err(|_| format!("unknown SQL fingerprint dialect '{dialect_name}'"))?; + let dialect = Dialect::get(dialect_type); + let mut digest = Sha256::new(); + let tokens = dialect + .tokenize(sql) + .map_err(|error| format!("SQL tokenization failed: {error}"))?; + for token in canonical_tokens(sql, &tokens, &dialect)? { + digest.update(token.text.len().to_string().as_bytes()); + digest.update(b":"); + digest.update(token.text.as_bytes()); + } + Ok(format!("{:x}", digest.finalize())) +} diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/main/token_texts.rs b/crates/sqlbuild-rules-native/src/sql_tokens/main/token_texts.rs new file mode 100644 index 0000000000..f0f29ada3a --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/main/token_texts.rs @@ -0,0 +1,16 @@ +//! Authored text of tokens, taken from their character spans. + +use polyglot_sql::tokens::Token; + +/// Return the authored text of every token in `sql`. +pub(crate) fn token_texts(sql: &str, tokens: &[Token]) -> Result, String> { + let characters: Vec = sql.chars().collect(); + let mut texts = Vec::with_capacity(tokens.len()); + for token in tokens { + let slice = characters + .get(token.span.start..token.span.end) + .ok_or_else(|| "SQL tokenizer returned an invalid token span".to_string())?; + texts.push(slice.iter().collect()); + } + Ok(texts) +} diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/mod.rs b/crates/sqlbuild-rules-native/src/sql_tokens/mod.rs new file mode 100644 index 0000000000..4e8cc6fdd9 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/mod.rs @@ -0,0 +1,9 @@ +//! SQLBuild's canonical SQL token stream: authored tokens without layout or comments. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub(crate) mod main; +pub(crate) mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/models.rs b/crates/sqlbuild-rules-native/src/sql_tokens/models.rs new file mode 100644 index 0000000000..c31021a7b3 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/models.rs @@ -0,0 +1,7 @@ +//! Canonical token values shared by the formatter invariant and change fingerprints. + +/// One authored token with layout removed: keywords and built-in function names upper-cased. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct CanonicalToken { + pub(crate) text: String, +} diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/bigquery.txt b/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/bigquery.txt new file mode 100644 index 0000000000..ded970a614 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/bigquery.txt @@ -0,0 +1,99 @@ +# BigQuery (GoogleSQL) reserved keywords: 'Reserved keywords' in +# https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical +all +and +any +array +as +asc +assert_rows_modified +at +between +by +case +cast +collate +contains +create +cross +cube +current +default +define +desc +distinct +else +end +enum +escape +except +exclude +exists +extract +false +fetch +following +for +from +full +graph_table +group +grouping +groups +hash +having +if +ignore +in +inner +intersect +interval +into +is +join +lateral +left +like +limit +lookup +merge +natural +new +no +not +null +nulls +of +on +or +order +outer +over +partition +preceding +proto +qualify +range +recursive +respect +right +rollup +rows +select +set +some +struct +tablesample +then +to +treat +true +unbounded +union +unnest +using +when +where +window +with +within diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/databricks.txt b/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/databricks.txt new file mode 100644 index 0000000000..dc43281979 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/databricks.txt @@ -0,0 +1,4 @@ +# Databricks reserves no keywords as identifiers: +# https://docs.databricks.com/sql/language-manual/sql-ref-reserved-words +# ('Databricks does not formally disallow any specific literals from being used as identifiers'). + diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/duckdb.txt b/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/duckdb.txt new file mode 100644 index 0000000000..4374c4e538 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/duckdb.txt @@ -0,0 +1,77 @@ +# DuckDB reserved keywords: SELECT keyword_name FROM duckdb_keywords() +# WHERE keyword_category = 'reserved' (DuckDB 1.5.2). +all +analyse +analyze +and +any +array +as +asc +asymmetric +both +case +cast +check +collate +column +constraint +create +default +deferrable +desc +describe +distinct +do +else +end +except +false +fetch +for +foreign +from +group +having +in +initially +intersect +into +lambda +lateral +leading +limit +not +null +offset +on +only +or +order +pivot +pivot_longer +pivot_wider +placing +primary +qualify +references +returning +select +show +some +summarize +symmetric +table +then +to +trailing +true +union +unique +unpivot +using +variadic +when +where +window +with diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/generic.txt b/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/generic.txt new file mode 100644 index 0000000000..05f8ee42cd --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/generic.txt @@ -0,0 +1,28 @@ +# Generic dialect: keywords reserved in every dialect file above except Databricks +# (DuckDB, PostgreSQL, BigQuery, Snowflake and T-SQL). +all +and +any +as +create +distinct +else +for +from +group +having +in +intersect +into +not +null +on +or +order +select +some +then +to +union +where +with diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/postgres.txt b/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/postgres.txt new file mode 100644 index 0000000000..7a57911cb5 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/postgres.txt @@ -0,0 +1,80 @@ +# PostgreSQL reserved keywords (non-function): RESERVED_KEYWORD entries of +# src/include/parser/kwlist.h (REL_17_STABLE); Appendix C 'reserved'. +all +analyse +analyze +and +any +array +as +asc +asymmetric +both +case +cast +check +collate +column +constraint +create +current_catalog +current_date +current_role +current_time +current_timestamp +current_user +default +deferrable +desc +distinct +do +else +end +except +false +fetch +for +foreign +from +grant +group +having +in +initially +intersect +into +lateral +leading +limit +localtime +localtimestamp +not +null +offset +on +only +or +order +placing +primary +references +returning +select +session_user +some +symmetric +system_user +table +then +to +trailing +true +union +unique +user +using +variadic +when +where +window +with diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/snowflake.txt b/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/snowflake.txt new file mode 100644 index 0000000000..511b64668b --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/snowflake.txt @@ -0,0 +1,65 @@ +# Snowflake reserved keywords: entries marked 'Reserved by ANSI', 'Reserved by Snowflake' +# or 'Reserved by Snowflake and others' in https://docs.snowflake.com/en/sql-reference/reserved-keywords. +# Limited keywords (usable as identifiers in some positions) are excluded. +all +alter +and +any +as +between +by +check +column +connect +create +current +delete +distinct +drop +else +exists +following +for +from +grant +group +having +ilike +in +increment +insert +intersect +into +is +like +minus +not +null +of +on +or +order +qualify +regexp +revoke +rlike +row +rows +sample +select +set +some +start +table +tablesample +then +to +trigger +union +unique +update +values +whenever +where +window +with diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/tsql.txt b/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/tsql.txt new file mode 100644 index 0000000000..b83a35b59d --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/tsql.txt @@ -0,0 +1,186 @@ +# T-SQL reserved keywords (SQL Server and Azure Synapse table) in +# https://learn.microsoft.com/sql/t-sql/language-elements/reserved-keywords-transact-sql +add +all +alter +and +any +as +asc +authorization +backup +begin +between +break +browse +bulk +by +cascade +case +check +checkpoint +close +clustered +coalesce +collate +column +commit +compute +constraint +contains +containstable +continue +convert +create +cross +current +current_date +current_time +current_timestamp +current_user +cursor +database +dbcc +deallocate +declare +default +delete +deny +desc +disk +distinct +distributed +double +drop +dump +else +end +errlvl +escape +except +exec +execute +exists +exit +external +fetch +file +fillfactor +for +foreign +freetext +freetexttable +from +full +function +goto +grant +group +having +holdlock +identity +identity_insert +identitycol +if +in +index +inner +insert +intersect +into +is +join +key +kill +left +like +lineno +load +merge +national +nocheck +nonclustered +not +null +nullif +of +off +offsets +on +open +opendatasource +openquery +openrowset +openxml +option +or +order +outer +over +percent +pivot +plan +precision +primary +print +proc +procedure +public +raiserror +read +readtext +reconfigure +references +replication +restore +restrict +return +revert +revoke +right +rollback +rowcount +rowguidcol +rule +save +schema +securityaudit +select +semantickeyphrasetable +semanticsimilaritydetailstable +semanticsimilaritytable +session_user +set +setuser +shutdown +some +statistics +system_user +table +tablesample +textsize +then +to +top +tran +transaction +trigger +truncate +try_convert +tsequal +union +unique +unpivot +update +updatetext +use +user +values +varying +view +waitfor +when +where +while +with +writetext diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/tests.rs b/crates/sqlbuild-rules-native/src/sql_tokens/tests.rs new file mode 100644 index 0000000000..5468fc5562 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_canonical_tokens.rs"] +mod canonical_tokens; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/tests/test_canonical_tokens.rs b/crates/sqlbuild-rules-native/src/sql_tokens/tests/test_canonical_tokens.rs new file mode 100644 index 0000000000..2dec3bf4cc --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/tests/test_canonical_tokens.rs @@ -0,0 +1,218 @@ +use polyglot_sql::{Dialect, DialectType}; + +use crate::sql_tokens::main::canonical_tokens::canonical_tokens; +use crate::sql_tokens::tests::test_types::CanonicalTokensTestCase; + +#[test] +fn given_sql_pairs_when_canonicalizing_then_only_layout_comments_and_word_case_are_ignored() +-> Result<(), String> { + let test_cases = [ + CanonicalTokensTestCase { + description: "whitespace between tokens is ignored", + dialect: DialectType::DuckDB, + left: "select a,b from orders", + right: "SELECT\n a, b\nFROM orders", + expected_equal: true, + }, + CanonicalTokensTestCase { + description: "comments are ignored", + dialect: DialectType::Snowflake, + left: "select a -- note\nfrom orders /* block */", + right: "select a from orders", + expected_equal: true, + }, + CanonicalTokensTestCase { + description: "keyword and built-in function case is ignored", + dialect: DialectType::Snowflake, + left: "select count(*) from orders", + right: "SELECT COUNT(*) FROM orders", + expected_equal: true, + }, + CanonicalTokensTestCase { + description: "unquoted identifier case is significant", + dialect: DialectType::Snowflake, + left: "select count(*) from orders", + right: "SELECT COUNT(*) FROM ORDERS", + expected_equal: false, + }, + CanonicalTokensTestCase { + description: "Snowflake path key case is significant", + dialect: DialectType::Snowflake, + left: "select payload:customerId from t", + right: "select payload:customerid from t", + expected_equal: false, + }, + CanonicalTokensTestCase { + description: "BigQuery qualified table case is significant", + dialect: DialectType::BigQuery, + left: "select * from ds.Orders", + right: "select * from ds.orders", + expected_equal: false, + }, + CanonicalTokensTestCase { + description: "DuckDB alias case is significant", + dialect: DialectType::DuckDB, + left: "select 1 as Foo", + right: "select 1 as foo", + expected_equal: false, + }, + CanonicalTokensTestCase { + description: "keyword-named qualified name case is significant", + dialect: DialectType::BigQuery, + left: "select * from inventory.view", + right: "select * from inventory.VIEW", + expected_equal: false, + }, + CanonicalTokensTestCase { + description: "keyword-named alias case is significant", + dialect: DialectType::DuckDB, + left: "select pos as index from t", + right: "select pos AS INDEX from t", + expected_equal: false, + }, + CanonicalTokensTestCase { + description: "user-defined function case is significant", + dialect: DialectType::DuckDB, + left: "select my_udf(a) from t", + right: "select MY_UDF(a) from t", + expected_equal: false, + }, + CanonicalTokensTestCase { + description: "built-in function case is ignored", + dialect: DialectType::DuckDB, + left: "select coalesce(a, 0) from t", + right: "select COALESCE(a, 0) from t", + expected_equal: true, + }, + CanonicalTokensTestCase { + description: "Snowflake double-slash comments are ignored", + dialect: DialectType::Snowflake, + left: "select a // note\nfrom t", + right: "select a from t", + expected_equal: true, + }, + CanonicalTokensTestCase { + description: "BigQuery hash comments are ignored", + dialect: DialectType::BigQuery, + left: "select a # note\nfrom t", + right: "select a from t", + expected_equal: true, + }, + CanonicalTokensTestCase { + description: "DuckDB struct-literal key case is significant", + dialect: DialectType::DuckDB, + left: "select {Key: 1}", + right: "select {key: 1}", + expected_equal: false, + }, + CanonicalTokensTestCase { + description: "a non-reserved keyword operand before an operator keeps its case", + dialect: DialectType::DuckDB, + left: "select a, Rows ^ 2 from t", + right: "select a, rows ^ 2 from t", + expected_equal: false, + }, + CanonicalTokensTestCase { + description: "a non-reserved keyword table in a comma join keeps its case", + dialect: DialectType::BigQuery, + left: "select * from ds.a, View v", + right: "select * from ds.a, view v", + expected_equal: false, + }, + CanonicalTokensTestCase { + description: "a reserved word after a dot is a field name", + dialect: DialectType::BigQuery, + left: "select abc.Group from abc", + right: "select abc.group from abc", + expected_equal: false, + }, + CanonicalTokensTestCase { + description: "a reserved keyword column alias keeps its case", + dialect: DialectType::DuckDB, + left: "select 1 as Select", + right: "select 1 as select", + expected_equal: false, + }, + CanonicalTokensTestCase { + description: "reserved keyword case is ignored", + dialect: DialectType::DuckDB, + left: "select a from t where a in (1) limit 1", + right: "SELECT a FROM t WHERE a IN (1) LIMIT 1", + expected_equal: true, + }, + CanonicalTokensTestCase { + description: "Databricks reserves no keywords", + dialect: DialectType::Databricks, + left: "select a from t", + right: "SELECT a FROM t", + expected_equal: false, + }, + CanonicalTokensTestCase { + description: "whitespace inside a string literal is significant", + dialect: DialectType::DuckDB, + left: "select 'a b' as code", + right: "select 'a b' as code", + expected_equal: false, + }, + CanonicalTokensTestCase { + description: "string literal case is significant", + dialect: DialectType::DuckDB, + left: "select 'Open' as status", + right: "select 'OPEN' as status", + expected_equal: false, + }, + CanonicalTokensTestCase { + description: "quoted identifier case is significant", + dialect: DialectType::Snowflake, + left: "select \"Order\" from orders", + right: "select \"ORDER\" from orders", + expected_equal: false, + }, + CanonicalTokensTestCase { + description: "dollar-quoted literal content is significant", + dialect: DialectType::PostgreSQL, + left: "select $$a b$$ as note", + right: "select $$a b$$ as note", + expected_equal: false, + }, + CanonicalTokensTestCase { + description: "number spelling is significant", + dialect: DialectType::DuckDB, + left: "select 1.0 as amount", + right: "select 1.00 as amount", + expected_equal: false, + }, + CanonicalTokensTestCase { + description: "a renamed function is significant", + dialect: DialectType::Snowflake, + left: "select startswith(code, 'EU')", + right: "select starts_with(code, 'EU')", + expected_equal: false, + }, + CanonicalTokensTestCase { + description: "an added alias keyword is significant", + dialect: DialectType::DuckDB, + left: "select a b from orders", + right: "select a as b from orders", + expected_equal: false, + }, + ]; + for test_case in test_cases { + let dialect = Dialect::get(test_case.dialect); + let left_tokens = dialect + .tokenize(test_case.left) + .map_err(|error| error.to_string())?; + let right_tokens = dialect + .tokenize(test_case.right) + .map_err(|error| error.to_string())?; + let left = canonical_tokens(test_case.left, &left_tokens, &dialect)?; + let right = canonical_tokens(test_case.right, &right_tokens, &dialect)?; + assert_eq!( + left == right, + test_case.expected_equal, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/tests/test_types.rs b/crates/sqlbuild-rules-native/src/sql_tokens/tests/test_types.rs new file mode 100644 index 0000000000..2e7671dfac --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/tests/test_types.rs @@ -0,0 +1,7 @@ +pub(crate) struct CanonicalTokensTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: polyglot_sql::DialectType, + pub(crate) left: &'static str, + pub(crate) right: &'static str, + pub(crate) expected_equal: bool, +} diff --git a/scripts/cold_compile_performance/_helpers/diagnostic_measurement.py b/scripts/cold_compile_performance/_helpers/diagnostic_measurement.py index d333d5c1e3..c84e192236 100644 --- a/scripts/cold_compile_performance/_helpers/diagnostic_measurement.py +++ b/scripts/cold_compile_performance/_helpers/diagnostic_measurement.py @@ -5,13 +5,15 @@ import sys from pathlib import Path from time import perf_counter -from typing import Any +from typing import Any, cast +import sqlbuild._native as _native from scripts.cold_compile_performance.exceptions import CompileBenchmarkFixtureError from sqlbuild.compiler.compile._helpers.assembly.binding_positions import ( get_authored_binding_location, ) from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic +from sqlbuild.compiler.sql_analysis.types import NativeDiagnosticRow, NativePositionsModule from sqlbuild.spec.contracts.models import SourceLocation @@ -75,3 +77,25 @@ def measure_position_mapping(*, count: int, lines: int = 4000) -> float: "mapping must preserve the authored comparison location" ) return perf_counter() - started + + +def measure_function_name_scan(*, calls: int) -> float: + """Measure locating many unsupported built-in spellings in one wide Snowflake query.""" + sql: str = ( + "SELECT\n" + + ",\n".join( + f" STARTS_WITH(sku, 'a{index}') AS unsupported_{index}, " + f"STARTSWITH(sku, 'a') AS accepted_{index}, normalize_sku(sku) AS unknown_{index}" + for index in range(calls) + ) + + "\nFROM products" + ) + native: NativePositionsModule = cast(NativePositionsModule, _native) + started: float = perf_counter() + rows: list[NativeDiagnosticRow] = native.binding_diagnostics( + sql=sql, dialect="snowflake", rows=[] + ) + elapsed: float = perf_counter() - started + if len(rows) != calls or rows[-1][2] != calls + 1: + raise CompileBenchmarkFixtureError("every unsupported spelling must keep its own location") + return elapsed diff --git a/scripts/cold_compile_performance/_helpers/diagnostic_project.py b/scripts/cold_compile_performance/_helpers/diagnostic_project.py index 3c83eed60f..4ef3339755 100644 --- a/scripts/cold_compile_performance/_helpers/diagnostic_project.py +++ b/scripts/cold_compile_performance/_helpers/diagnostic_project.py @@ -73,3 +73,28 @@ def write_cascade_project(*, project_dir: Path, depth: int, width: int = 50) -> + f' FROM __ref("{previous}")' ) previous = name + + +def write_function_name_project(*, project_dir: Path, model_count: int, calls: int) -> None: + """Generate wide Snowflake models mixing unsupported, accepted, and unknown function calls.""" + project_dir.mkdir(parents=True, exist_ok=True) + models: Path = project_dir / "models" + models.mkdir(exist_ok=True) + (project_dir / "sqlbuild_project.toml").write_text( + 'name = "function_name_orders"\nadapter = "snowflake"\n[rules]\nselect = []\n' + ) + for model_index in range(model_count): + projections: list[str] = [] + for index in range(calls): + projections.extend( + ( + f" STARTS_WITH(sku, 'a{index}') AS unsupported_{index:05}", + f" STARTSWITH(sku, 'a{index}') AS accepted_{index:05}", + f" normalize_sku(sku, {index}) AS unknown_{index:05}", + ) + ) + (models / f"wide_products_{model_index}.sql").write_text( + "MODEL (database warehouse, schema analytics);\nSELECT\n" + + ",\n".join(projections) + + "\nFROM (SELECT 'anvil' AS sku) AS products\nWHERE NOT (sku IN ('a', 'b'))\n" + ) diff --git a/scripts/format_performance/__init__.py b/scripts/format_performance/__init__.py new file mode 100644 index 0000000000..fe8e51a43c --- /dev/null +++ b/scripts/format_performance/__init__.py @@ -0,0 +1 @@ +"""Generated workloads for formatter performance guards.""" diff --git a/scripts/format_performance/_helpers/model_project.py b/scripts/format_performance/_helpers/model_project.py new file mode 100644 index 0000000000..a72c86aed4 --- /dev/null +++ b/scripts/format_performance/_helpers/model_project.py @@ -0,0 +1,81 @@ +"""Synthetic unformatted model projects for formatter performance guards.""" + +from pathlib import Path + +_SHAPE_CYCLE: int = 10 +_WIDE_SHAPE: int = 0 +_CTE_CHAIN_SHAPES: int = 4 +_WIDE_COLUMNS: int = 300 +_CTE_STEPS: int = 6 + + +def write_model_format_project( + *, project_dir: Path, model_count: int, large_model_lines: int = 0 +) -> None: + """Write deterministic unformatted models of mixed shape, width and length.""" + + project_dir.mkdir() + (project_dir / "sqlbuild_project.toml").write_text( + 'name = "format_models"\nadapter = "duckdb"\n', encoding="utf-8" + ) + models_dir: Path = project_dir / "models" + models_dir.mkdir() + for model_index in range(model_count): + models_dir.joinpath(f"orders_{model_index:05d}.sql").write_text( + 'MODEL (description "Generated order model");\n\n' + + _model_sql(model_index=model_index), + encoding="utf-8", + ) + if large_model_lines: + models_dir.joinpath("large_orders.sql").write_text( + 'MODEL (description "Generated large order model");\n\n' + + _large_model_sql(line_count=large_model_lines), + encoding="utf-8", + ) + + +def _model_sql(*, model_index: int) -> str: + upstream: str = ( + f'__ref("orders_{model_index - 1:05d}")' + if model_index + else "(select 1 as order_id, 2 as customer_id, 3 as amount)" + ) + shape: int = model_index % _SHAPE_CYCLE + if shape == _WIDE_SHAPE: + columns: str = ", ".join( + f"coalesce(o.amount, 0) * {column} as amount_{column}" + for column in range(_WIDE_COLUMNS) + ) + return f"select o.order_id, {columns} from {upstream} o\n" + if shape < _CTE_CHAIN_SHAPES: + ctes: str = ",\n".join( + f"step_{step} as (select order_id, customer_id, amount + {step} as amount " + f"from {'base' if step == 0 else f'step_{step - 1}'} " + f"where amount > {step} -- keep positive amounts\n)" + for step in range(_CTE_STEPS) + ) + return ( + f"with base as (select order_id, customer_id, amount from {upstream}),\n{ctes}\n" + "select order_id, customer_id, sum(amount) over (partition by customer_id " + "order by order_id rows between unbounded preceding and current row) as amount, " + "case when amount > 100 then 'large' when amount > 10 then 'medium' " + "else 'small' end as size_band from step_5\n" + ) + return ( + "/* orders slice */\n" + f"select o.order_id, o.customer_id,o.amount, 'order {model_index}' as label " + f"from {upstream} o where o.amount>{model_index} and o.customer_id is not null " + "or o.order_id in (1,2,3,4,5,6,7,8)\n" + ) + + +def _large_model_sql(*, line_count: int) -> str: + steps: str = ",\n".join( + f"step_{index} as (select order_id, customer_id, amount + {index} as amount " + f"from {'base' if index == 0 else f'step_{index - 1}'} -- step {index}\n)" + for index in range(line_count // 2) + ) + return ( + "with base as (select 1 as order_id, 2 as customer_id, 3 as amount),\n" + f"{steps}\nselect order_id, customer_id, amount from step_{line_count // 2 - 1}\n" + ) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md index c7775839a3..e00d270193 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md @@ -6,6 +6,13 @@ Online: https://sqlbuild.com/docs/cli/format/ +## Contents + +- Line width and descriptions +- What formatting preserves +- Rule fixes +- Unsafe bodies + Formats SQL files using SQLBuild's canonical, dialect-aware representation. ```bash @@ -16,6 +23,7 @@ sqb format [flags] |------|-------------| | `--check` | Fail when formatting changes are needed without writing | | `--diff` | Print the proposed diff without writing | +| `--fix` | Also apply compiler-verified, meaning-preserving Rule fixes before layout formatting | | `--json` | Print structured results | | `--select`, `-s` | Format selected models | | `--select-file` | Read selectors from a file | @@ -32,23 +40,101 @@ configured [Rules](../concepts/rules.md), which never mutate source. ## Line width and descriptions -Long model and scenario descriptions are reflowed deterministically. Ordinary authored line breaks -are normalized as spaces, while blank lines preserve paragraph boundaries. Configure the maximum -physical line width in `sqlbuild_project.toml` (the default is `100`): +Configure the maximum physical line width in `sqlbuild_project.toml` (the default is `100`): ```toml [format] line_width = 100 ``` +Long model and scenario descriptions are reflowed deterministically. Ordinary authored line breaks +are normalized as spaces, while blank lines preserve paragraph boundaries. + +Clause layout follows a few fixed rules: + +- A `SELECT` list with exactly one item stays on the `SELECT` line when the item fits (`SELECT *`, + `SELECT DISTINCT order_id`). Two or more items go one per line. +- `WHERE`, `QUALIFY` and `HAVING` with one top-level condition stay on the clause line when it + fits (`WHERE amount > 0`). Several `AND`/`OR` conditions go one per line. +- Each CTE closes with `),` followed by one blank line, and one blank line separates the last CTE + from the statement that uses it. Comments above a CTE stay directly above its name. +- A `VALUES` list that does not fit on its line puts each row on its own line under `VALUES`, and a + row is never split, even when it is longer than the width. +- When an operand of `+`, `-`, `*`, `/`, `%`, `||`, `AND` or `OR` spans several lines, the + operator starts the next line. +- A broken `DECODE` puts its first argument on its own line, then one search/result pair per line, + then any default. + +SQL lines longer than the width, counting any trailing alias and comma, are broken at the outermost +structure first: before top-level `AND` and `OR` (never inside `BETWEEN ... AND`), then before +arithmetic operators, then by putting the arguments of the widest outermost function call, `IN` +list, or array one per line, or the branches of a `CASE` on their own lines. Inside `OVER (...)`, +`PARTITION BY` and `ORDER BY` go on separate lines. Type parameters such as `NUMBER(38, 2)` or +`VARCHAR(MAX)` are never split; the enclosing call breaks instead. Header `audits [...]` lists put +each audit, and then each audit argument, on its own line. String literals, comments, and lines that +carry a comment are never broken, so a line can stay longer than the width when nothing on it can be +split. `sqb format --check` fails while a long line can still be shortened. + ## What formatting preserves +Formatting changes layout only: it prints the authored SQL tokens in their original order and +changes only the whitespace between them and the letter case of keywords and built-in function +names. A word is upper-cased only when the parse confirms it is a keyword at that position, such as +`ORDER BY`, `CASE WHEN`, `NULLS LAST`, `IS NULL` or `JOIN`, or when it names a built-in function at +a call site, such as `coalesce(`. Identifiers keep their authored case even when they are spelled +like keywords, such as `t.view`, `AS index` or a table named `view`, and so do aliases, qualified +names, JSON path keys and user-defined function names. + +Query fingerprints ignore the case of reserved keywords and built-in function names only, so +formatting lowercase SQL may change a model's fingerprint once, when a non-reserved keyword such as +`by` or `nulls` is upper-cased. Formatting already formatted SQL never changes its fingerprint. + +Comments in every dialect form are kept, including `//` in Snowflake and `#` in BigQuery, and +adjacent string literals split across lines stay on separate lines. It does not add `AS` to aliases +or remove statement terminators or trailing commas. Every result is checked against the authored +token stream and comment positions, and a body whose tokens would change is reported as +`format-unsafe` instead of being written. + Formatting preserves authored cast types, postfix casts, quoted literals, variant paths, typed lambda parameters, and supported SQL function spellings while applying canonical layout. SQLBuild calls keep their authored spelling, including zero-argument cursor and empty-fixture intrinsics. CTE-producing macros remain authored calls rather than expanded project SQL, with each call on its own CTE-list line and leading comments attached to the node they describe. +## Rule fixes + +`sqb format` on its own never changes what SQL means. `sqb format --fix` also applies Rule edits +that are known to preserve meaning, then formats the layout. `--fix --check` reports the proposed +changes without writing files, and `--fix --diff` prints their diff. + +Edits are planned in memory and repeated until no more apply; overlapping edits wait for the next +pass. Every pass recompiles the project and compares output columns (including types and +nullability), dependencies, and column lineage of each edited model and everything downstream of it +against the original compilation. When a pass cannot be proved, the edits of each file are checked +on their own: files whose edits cannot be proved keep their original contents and report +`format-fix-verification-failed`, while the other files are still fixed. A cycle or the 128-pass +limit fails without writing any edit. + +| Rule | Decision | Reason | +| --- | --- | --- | +| SQBRSQL001 NULL comparison | Refused | Replacing `= NULL` with `IS NULL` changes three-valued logic. | +| SQBRSQL002 implicit cartesian join | Applied | A comma-separated product becomes an explicit `CROSS JOIN`; mixed join precedence is excluded. | +| SQBRSQL003 conditionless join | Applied in Snowflake | Snowflake defines an unqualified `JOIN` without `ON` or `USING` as a cartesian product. Other dialects are refused because the original syntax may be invalid. | +| SQBRSQL005 unused CTE | Applied | Unreferenced `SELECT` CTEs contribute no rows. Mutation statements and commented CTEs are excluded. | +| SQBRSQL006 redundant DISTINCT | Applied | The projected and grouping expressions are proved to coincide. | +| SQBRSQL008 bare UNION | Applied | `UNION DISTINCT` states the existing duplicate elimination, in dialects that support it. | +| SQBRSQL013 parenthesized DISTINCT | Refused | Removing parentheses can change operator precedence. | +| SQBRSQL017 constant row count | Applied | `COUNT(1)` and `COUNT(*)` both count every row. | +| SQBRSQL023 unused table alias | Refused | Not yet reviewed as meaning-preserving. | +| SQBRSQL025 implicit inner join | Applied | A bare `JOIN ... ON` is an inner join. | +| SQBRSQL030 boolean CASE simplification | Refused | The `COALESCE` rewrite is not proved portable to every supported dialect. | + +Edits that touch macro or intrinsic expansions are refused, and so are edits outside model SQL +bodies, because recompiling cannot prove their output contract. Findings without a proposed edit +are reported as unavailable. Human output lists each applied, refused, and unavailable fix with its +reason; JSON output includes a `rule_fixes` array with `file`, `line`, `code`, `status`, and +`reason`. Refused and unavailable fixes still need a manual change. + ## Unsafe bodies `sqb format` reports a file-specific `format-unsafe` fault whenever a SQL body cannot be safely diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index cef8b9084c..f73dc774ed 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -411,7 +411,7 @@ default_audit_run_scope = "final" | Field | Default | Description | |-------|---------|-------------| | `sql_analysis` | `true` | Enable SQL syntax analysis, column binding, type inference, output inference, and semantic validation at compile time | -| `query_change_tracking` | `true` | Track query fingerprints for change detection | +| `query_change_tracking` | `true` | Track query fingerprints for change detection. A fingerprint covers the SQL tokens of the compiled query and ignores whitespace, comments, and the letter case of reserved keywords and built-in function names, so formatting-only edits do not rebuild. Every other word, including non-reserved keywords, identifiers, aliases, JSON path keys and user-defined function names, counts exactly, as do string literals, quoted identifiers, numbers, and optimizer hints. | | `column_contract_mode` | `implicit` | Controls whether column declarations on models without a `contract` declaration activate static shape/nullability validation. `implicit` preserves that validation; `explicit` treats columns as metadata and audit attachment unless the model declares `contract enforced`. Model-level `contract enforced` and `contract none` override this setting. Explicit type enforcement remains independent. See [Contracts](models/contracts.md). | | `concurrency` | `1` | Maximum parallel model execution (currently serial only) | | `microbatch_concurrency` | `false` | Explicitly permit models with `batch_concurrency > 1`; concurrent batches use immutable coordination facts | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md index f53d9d1d1f..d9525d4ab7 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md @@ -66,6 +66,19 @@ Snowflake's function catalogue is partial and deliberately does not reject unkno Function signatures and overload coverage remain partial; compilation is not a replacement for executing SQL unit tests. +Built-in function spellings are checked for Snowflake and DuckDB. The SQL parser treats alias +spellings such as `STARTS_WITH` and `STARTSWITH` as one built-in, so a spelling the warehouse +rejects would otherwise compile. SQLBuild compares each call's authored name, case-insensitively, +with the dialect's accepted spellings and reports a blocking B101 error naming the dialect's +spelling, for example `Unknown function 'STARTS_WITH' for dialect Snowflake; did you mean +STARTSWITH?`. Snowflake's accepted spellings come from its function reference and include +documented synonyms such as `SUBSTR` and `SUBSTRING`, `DATEDIFF` and `TIMESTAMPDIFF`, and +`TRY_TO_DECIMAL` and `TRY_TO_NUMBER`. DuckDB's come from its function catalogue. A name is +rejected only when the parser maps it to the same built-in as an accepted spelling, so project +UDFs called through `__udf`, schema-qualified and method calls, quoted names, and unknown +functions are not affected. There is no per-model suppression: use the suggested spelling, or +schema-qualify a call to an external UDF that shares an alias name. Other dialects are not checked. + Derived-table column aliases, UNPIVOT, and Snowflake GROUPING SETS are checked. DuckDB and Snowflake reject excess relation aliases; fewer aliases than output columns remain valid. Missing UNPIVOT input columns are errors. @@ -168,7 +181,7 @@ the notice never reduces them to a count alone. | B002 | E201 | Unknown column | | B003 | E221 | Ambiguous column | | B004–B005 | E222–E223 | Reference/scope errors | -| B101 | E202 | Unknown function | +| B101 | E202 or SQLBuild | Unknown function, or a built-in spelling the dialect does not accept | | B102 | E203 or project declaration | Invalid function arity | | B210–B219 | E210–E219 | Proven native type/shape errors on enabled dialects | | B211–B218 | W210–W216 | Implicit conversions; blocking, fixed by an explicit `CAST` or comparison | diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 37bd4c9921..51cd178294 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -14,6 +14,7 @@ def evaluate_json(request_json: str) -> str: ... def finalize_rule_findings_json(request_json: str) -> str: ... def format_sql_json(request_json: str) -> str: ... def format_sql_batch_json(request_json: str) -> str: ... +def query_fingerprint(sql: str, dialect: str) -> str: ... def lint_sql_json(request_json: str) -> str: ... def lint_sql_batch_json(request_json: str) -> str: ... def prepare_lint_sql( diff --git a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py index cdb2d20c55..74ccc3cf23 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py @@ -564,6 +564,8 @@ def _dispatch_lint_format_command( """Route the source-formatting command to its handler.""" if args.format_diff and args.json: raise CliUserError("format --diff cannot be combined with --json", code="C112") + if args.format_fix and args.format_fixtures_only: + raise CliUserError("format --fix cannot be combined with --fixtures-only", code="C112") path_selectors: tuple[str, ...] = format_path_selectors( paths=tuple(args.format_paths), project_dir=project_dir, cwd=Path.cwd() ) @@ -574,6 +576,7 @@ def _dispatch_lint_format_command( check=args.format_check, diff=args.format_diff, fixtures_only=args.format_fixtures_only, + fix=args.format_fix, json_output=args.json, no_color=args.no_color, ) diff --git a/src/sqlbuild/cli/commands/_helpers/entry/lazy_handlers.py b/src/sqlbuild/cli/commands/_helpers/entry/lazy_handlers.py index 386aa017f6..f43bc104f7 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/lazy_handlers.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/lazy_handlers.py @@ -207,7 +207,7 @@ def build_lazy_cli_handlers() -> CliEntrypointHandlers: targets=targets, force=force, ), - run_format=lambda project_dir, select, exclude, check, diff, fixtures_only, json_output, no_color: ( # noqa: E501 + run_format=lambda project_dir, select, exclude, check, diff, fixtures_only, fix, json_output, no_color: ( # noqa: E501 lazy["format"]( project_dir=project_dir, select=select, @@ -215,6 +215,7 @@ def build_lazy_cli_handlers() -> CliEntrypointHandlers: check=check, diff=diff, fixtures_only=fixtures_only, + fix=fix, json_output=json_output, no_color=no_color, ) diff --git a/src/sqlbuild/cli/commands/_helpers/entry/parser.py b/src/sqlbuild/cli/commands/_helpers/entry/parser.py index 0ede580a9d..0440bdde45 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/parser.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/parser.py @@ -342,6 +342,12 @@ def _add_quality_parsers( format_parser.add_argument("--json", action="store_true", default=False) format_parser.add_argument("--check", dest="format_check", action="store_true") format_parser.add_argument("--diff", dest="format_diff", action="store_true") + format_parser.add_argument( + "--fix", + dest="format_fix", + action="store_true", + help="apply compiler-verified meaning-preserving Rule fixes", + ) format_parser.add_argument( "--fixtures-only", dest="format_fixtures_only", diff --git a/src/sqlbuild/cli/commands/_helpers/lint/runs.py b/src/sqlbuild/cli/commands/_helpers/lint/runs.py index 0a7b14259e..d20982cd5d 100644 --- a/src/sqlbuild/cli/commands/_helpers/lint/runs.py +++ b/src/sqlbuild/cli/commands/_helpers/lint/runs.py @@ -89,6 +89,8 @@ def render_lint_result( if result.violations: print("\n\n".join(format_lint_diagnostics(result=result, root=root, use_color=use_color))) print() + for fix in result.rule_fixes: + print(f"{fix.status.upper()} {fix.code} {fix.file_path}:{fix.line}: {fix.reason}") summary: str = ( f"Completed. FAULT={len(result.faults)} WARN={len(result.warnings)} " f"FILES={result.files_checked}" @@ -105,6 +107,16 @@ def render_lint_result_json(*, result: LintRunResult) -> None: "faults": len(result.faults), "files_checked": result.files_checked, "formatted_files": [str(path) for path in result.formatted_files], + "rule_fixes": [ + { + "file": str(fix.file_path), + "code": fix.code, + "line": fix.line, + "status": fix.status, + "reason": fix.reason, + } + for fix in result.rule_fixes + ], "violations": [ { "code": violation.code, diff --git a/src/sqlbuild/cli/commands/classes/cli_namespace.py b/src/sqlbuild/cli/commands/classes/cli_namespace.py index 13d2eb55bf..11530e5d84 100644 --- a/src/sqlbuild/cli/commands/classes/cli_namespace.py +++ b/src/sqlbuild/cli/commands/classes/cli_namespace.py @@ -140,6 +140,7 @@ "format_check": False, "format_diff": False, "format_fixtures_only": False, + "format_fix": False, "format_paths": [], "scope_target": None, "scope_at": None, @@ -309,6 +310,7 @@ class CliNamespace: format_check: bool format_diff: bool format_fixtures_only: bool + format_fix: bool format_paths: list[str] scope_target: str | None scope_at: str | None diff --git a/src/sqlbuild/cli/commands/main/project/_format.py b/src/sqlbuild/cli/commands/main/project/_format.py index 588fcac330..1ff842a3de 100644 --- a/src/sqlbuild/cli/commands/main/project/_format.py +++ b/src/sqlbuild/cli/commands/main/project/_format.py @@ -16,6 +16,7 @@ ) from sqlbuild.cli.commands._helpers.lint.selection import resolve_lint_inputs from sqlbuild.compiler.discovery.constants import SQL_ANALYSIS_SETTING_KEY +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.lint.main.has_fixture_typed_null_candidates import ( has_fixture_typed_null_candidates, @@ -34,6 +35,7 @@ def run_format_command( check: bool = False, diff: bool = False, fixtures_only: bool = False, + fix: bool = False, json_output: bool = False, no_color: bool = False, ) -> int: @@ -41,8 +43,6 @@ def run_format_command( base_dir: Path = project_dir if project_dir is not None else Path.cwd() prepared: tuple[LintConfig, str | None] = prepare_lint_run(project_dir=base_dir) - if prepared[1] is not None: - print(f"WARN {prepared[1]}") config: LintConfig = prepared[0] value_renderer: BaseAdapter | None = None selected_paths: frozenset[Path] | None = None @@ -50,7 +50,7 @@ def run_format_command( has_fixture_candidates: bool = False if not select and not exclude: has_fixture_candidates = has_fixture_typed_null_candidates(project_dir=base_dir) - if select or exclude or has_fixture_candidates: + if select or exclude or has_fixture_candidates or fix: try: value_renderer, selected_paths, discovered_inputs = resolve_lint_inputs( project_dir=base_dir, @@ -58,7 +58,7 @@ def run_format_command( exclude=exclude, ) except Exception: - if select or exclude: + if select or exclude or fix: raise else: has_fixture_candidates = bool(discovered_inputs.test_files) @@ -67,6 +67,8 @@ def run_format_command( dialect=value_renderer.sql_analysis_dialect_name or "generic", ) fixture_inputs: DiscoveredProjectInputs | None = None + if fix: + discovered_inputs = discover_project_inputs(project_dir=base_dir) if discovered_inputs is not None and has_fixture_candidates: local_overrides: frozenset[str] = discovered_inputs.local_config.setting_overrides sql_analysis_enabled: bool = ( @@ -88,20 +90,16 @@ def run_format_command( config=config, value_renderer=value_renderer, selected_paths=selected_paths, - discovered_inputs=fixture_inputs, + discovered_inputs=discovered_inputs if fix else fixture_inputs, fixtures_only=fixtures_only, write=not (check or diff), + fix=fix, ) except Exception: format_status.error("Formatting SQL ERROR") raise elapsed_seconds: float = time.monotonic() - format_started_at - format_status.complete( - message=( - f"Formatting SQL OK ({elapsed_seconds:.2f}s; " - f"{result.files_checked} files checked, {len(result.formatted_files)} changed)" - ) - ) + _ = _complete_format_status(status=format_status, result=result, elapsed=elapsed_seconds) if diff: _ = _render_format_diff(result=result) if json_output: @@ -117,6 +115,20 @@ def run_format_command( return 1 if result.faults or (check and result.formatted_files) else 0 +def _complete_format_status( + *, status: TransientStatusReporter, result: LintRunResult, elapsed: float +) -> None: + if result.faults: + status.error(f"Formatting SQL ERROR ({elapsed:.2f}s; {len(result.faults)} faults)") + else: + status.complete( + message=( + f"Formatting SQL OK ({elapsed:.2f}s; " + f"{result.files_checked} files checked, {len(result.formatted_files)} changed)" + ) + ) + + def _render_format_diff(*, result: LintRunResult) -> None: """Print stable unified diffs for files the formatter would change.""" diff --git a/src/sqlbuild/cli/commands/types.py b/src/sqlbuild/cli/commands/types.py index f9adb82b2d..ac84926844 100644 --- a/src/sqlbuild/cli/commands/types.py +++ b/src/sqlbuild/cli/commands/types.py @@ -126,6 +126,7 @@ def __call__( check: bool, diff: bool, fixtures_only: bool, + fix: bool, json_output: bool, no_color: bool, ) -> int: ... diff --git a/src/sqlbuild/compiler/fingerprints/_helpers/query.py b/src/sqlbuild/compiler/fingerprints/_helpers/query.py index 1a7f89fd00..c45a691ca7 100644 --- a/src/sqlbuild/compiler/fingerprints/_helpers/query.py +++ b/src/sqlbuild/compiler/fingerprints/_helpers/query.py @@ -4,19 +4,49 @@ import hashlib import re +from functools import lru_cache + +import sqlbuild._native as _native +from sqlbuild.compiler.fingerprints.constants import ( + GENERIC_FINGERPRINT_DIALECT, + QUERY_FINGERPRINT_CACHE_SIZE, + SQL_FUNCTION_LANGUAGE, +) +from sqlbuild.compiler.fingerprints.exceptions import QueryFingerprintError +from sqlbuild.compiler.sql_analysis.constants import NATIVE_DIALECT_ALIASES _WHITESPACE_RUN: re.Pattern[str] = re.compile(r"\s+") def normalize_query_sql_impl(query_sql: str) -> str: - """Normalize query SQL while preserving persisted hash behavior.""" + """Normalize query SQL text by collapsing whitespace runs.""" stripped: str = query_sql.strip() return _WHITESPACE_RUN.sub(" ", stripped) -def compute_query_hash_impl(query_sql: str) -> str: - """Compute the persisted SHA-256 query fingerprint.""" +def compute_query_hash_impl(*, query_sql: str, dialect: str | None) -> str: + """Compute the persisted SHA-256 fingerprint of the query's canonical token stream.""" + + native_dialect: str = ( + NATIVE_DIALECT_ALIASES.get(dialect, dialect) if dialect else GENERIC_FINGERPRINT_DIALECT + ) + return _cached_query_hash(query_sql=query_sql, dialect=native_dialect) + + +def compute_function_definition_hash_impl( + *, fingerprint_sql: str, language: str, dialect: str | None +) -> str: + """Fingerprint SQL function definitions like queries and other languages exactly.""" + + if language == SQL_FUNCTION_LANGUAGE: + return compute_query_hash_impl(query_sql=fingerprint_sql, dialect=dialect) + return hashlib.sha256(fingerprint_sql.encode("utf-8")).hexdigest() + - normalized: str = normalize_query_sql_impl(query_sql) - return hashlib.sha256(normalized.encode("utf-8")).hexdigest() +@lru_cache(maxsize=QUERY_FINGERPRINT_CACHE_SIZE) +def _cached_query_hash(*, query_sql: str, dialect: str) -> str: + try: + return _native.query_fingerprint(query_sql, dialect) + except ValueError as error: + raise QueryFingerprintError(str(error)) from None diff --git a/src/sqlbuild/compiler/fingerprints/constants.py b/src/sqlbuild/compiler/fingerprints/constants.py index 7ca63166e8..b2e7b2d999 100644 --- a/src/sqlbuild/compiler/fingerprints/constants.py +++ b/src/sqlbuild/compiler/fingerprints/constants.py @@ -31,3 +31,6 @@ NODE_TYPE_QUERY_DIFF_ARTIFACT: str = "query_diff_artifact" AUDIT_GATE_METADATA_KEY: str = "audit_gate" +GENERIC_FINGERPRINT_DIALECT: str = "generic" +QUERY_FINGERPRINT_CACHE_SIZE: int = 16_384 +SQL_FUNCTION_LANGUAGE: str = "sql" diff --git a/src/sqlbuild/compiler/fingerprints/exceptions.py b/src/sqlbuild/compiler/fingerprints/exceptions.py index 31a1922b19..e715da351c 100644 --- a/src/sqlbuild/compiler/fingerprints/exceptions.py +++ b/src/sqlbuild/compiler/fingerprints/exceptions.py @@ -13,3 +13,7 @@ def __init__(self, message: str, *, code: str | None = None, help: str | None = self.message = message self.code = code if code is not None else self.code self.help = help + + +class QueryFingerprintError(ValueError): + """Raised when SQL cannot be tokenized into a change fingerprint.""" diff --git a/src/sqlbuild/compiler/fingerprints/main/compute_function_definition_hash.py b/src/sqlbuild/compiler/fingerprints/main/compute_function_definition_hash.py new file mode 100644 index 0000000000..eac0231898 --- /dev/null +++ b/src/sqlbuild/compiler/fingerprints/main/compute_function_definition_hash.py @@ -0,0 +1,13 @@ +"""Function definition fingerprint entrypoint.""" + +from sqlbuild.compiler.fingerprints._helpers.query import compute_function_definition_hash_impl + + +def compute_function_definition_hash( + *, fingerprint_sql: str, language: str, dialect: str | None +) -> str: + """Fingerprint a function definition; non-SQL bodies are hashed exactly.""" + + return compute_function_definition_hash_impl( + fingerprint_sql=fingerprint_sql, language=language, dialect=dialect + ) diff --git a/src/sqlbuild/compiler/fingerprints/main/compute_query_hash.py b/src/sqlbuild/compiler/fingerprints/main/compute_query_hash.py index 2fb8c1e3d2..f33d0ee9ec 100644 --- a/src/sqlbuild/compiler/fingerprints/main/compute_query_hash.py +++ b/src/sqlbuild/compiler/fingerprints/main/compute_query_hash.py @@ -3,7 +3,7 @@ from sqlbuild.compiler.fingerprints._helpers.query import compute_query_hash_impl -def compute_query_hash(query_sql: str) -> str: - """Compute the persisted SHA-256 query fingerprint.""" +def compute_query_hash(*, query_sql: str, dialect: str | None) -> str: + """Fingerprint query SQL, ignoring layout, comments and keyword case.""" - return compute_query_hash_impl(query_sql) + return compute_query_hash_impl(query_sql=query_sql, dialect=dialect) diff --git a/src/sqlbuild/compiler/manifest/_helpers/model_nodes.py b/src/sqlbuild/compiler/manifest/_helpers/model_nodes.py index 6acd78d8cd..19ef455999 100644 --- a/src/sqlbuild/compiler/manifest/_helpers/model_nodes.py +++ b/src/sqlbuild/compiler/manifest/_helpers/model_nodes.py @@ -24,6 +24,7 @@ def build_model_node( plan_entry: ModelPlanEntry | None, project_name: str, python_hook_metadata: dict[str, dict[str, object]] | None = None, + sql_analysis_dialect: str | None = None, ) -> dict[str, object]: """Build one dbt-compatible model node dict.""" @@ -31,7 +32,7 @@ def build_model_node( relative_path: Path = model.relative_path raw_code: str = model.query_sql compiled_code: str = plan_entry.resolved_sql if plan_entry is not None else raw_code - query_hash: str = compute_query_hash(raw_code) + query_hash: str = compute_query_hash(query_sql=raw_code, dialect=sql_analysis_dialect) return { "database": model.destination.database, diff --git a/src/sqlbuild/compiler/manifest/main/build.py b/src/sqlbuild/compiler/manifest/main/build.py index 49aaa323c7..82c0cd0240 100644 --- a/src/sqlbuild/compiler/manifest/main/build.py +++ b/src/sqlbuild/compiler/manifest/main/build.py @@ -72,6 +72,7 @@ def build_manifest( plan_entry=plan_entry, project_name=project_name, python_hook_metadata=python_hook_metadata, + sql_analysis_dialect=project.sql_analysis_dialect, ) source: CompiledSource diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py index 4912df0155..9401c5338f 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py @@ -14,7 +14,6 @@ InferredColumn, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.changes.metadata import ( changed_local_function_names, @@ -35,6 +34,7 @@ build_compiled_function_fingerprint_sql, detect_function_change, ) +from sqlbuild.compiler.planner._helpers.identity.hashing import model_definition_hash from sqlbuild.compiler.planner._helpers.identity.model_metadata import declared_columns_hash from sqlbuild.compiler.planner.constants import EMPTY_FINGERPRINT_METADATA_JSON from sqlbuild.compiler.planner.main.identity.version_identity_function_hashes import ( @@ -90,10 +90,12 @@ def detect_changes( function_changes[function.name] = FunctionChangeResult( fingerprint_sql=fingerprint_sql, reason=detect_function_change( + function=function, fingerprint_sql=fingerprint_sql, fingerprint=snapshot.fingerprints.functions.get(function.name), query_change_tracking=project.settings.query_change_tracking, full_refresh=full_refresh, + dialect=snapshot.column_dialect, ), ) @@ -118,7 +120,7 @@ def detect_model_changes_in_scope( else query_change_tracking ) function_local_hashes: dict[str, str] = build_function_local_hashes( - functions=project.functions, + functions=project.functions, dialect=project.sql_analysis_dialect ) hook_version_hashes: dict[str, str] = { name: identity.version_hash @@ -182,13 +184,20 @@ def detect_model_changes( relation_exists: bool = model_name in snapshot.existing_relations fingerprint: Fingerprint | None = snapshot.fingerprints.models.get(model_name) + compiled_query_hash: str | None = ( + model_definition_hash( + model_name=model_name, query_sql=model.query_sql, dialect=snapshot.column_dialect + ) + if query_change_tracking and fingerprint is not None + else None + ) if not relation_exists and model_name in snapshot.renamed_models: renamed_query_changed: bool = ( - query_change_tracking + compiled_query_hash is not None and fingerprint is not None and detect_query_change( - compiled_query_hash=compute_query_hash(model.query_sql), fingerprint=fingerprint + compiled_query_hash=compiled_query_hash, fingerprint=fingerprint ) ) return ChangeDetectionResult( @@ -244,10 +253,10 @@ def detect_model_changes( renames=snapshot.reference_renames, dialect=snapshot.column_dialect, ) - if query_change_tracking + if compiled_query_hash is not None and fingerprint is not None and snapshot.reference_renames - and compute_query_hash(model.query_sql) != fingerprint.definition_hash + and compiled_query_hash != fingerprint.definition_hash else None ) config_changed: bool = recorded_metadata_json is not None and identity_payload( @@ -256,9 +265,8 @@ def detect_model_changes( else metadata_json ) != identity_payload(recorded_metadata_json) replay_backfill: BackfillResult = BackfillResult(action=BackfillAction.FORWARD_ONLY) - if query_change_tracking and fingerprint is not None: + if compiled_query_hash is not None and fingerprint is not None: debug_logger: logging.Logger = logging.getLogger("sqlbuild.planner.changes") - compiled_query_hash: str = compute_query_hash(model.query_sql) query_changed = origin_names is None and detect_query_change( compiled_query_hash=compiled_query_hash, fingerprint=fingerprint, diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/recorded_renames.py b/src/sqlbuild/compiler/planner/_helpers/changes/recorded_renames.py index 1c9634cba5..8123e820a4 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/recorded_renames.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/recorded_renames.py @@ -3,8 +3,8 @@ from __future__ import annotations from sqlbuild.compiler.compile.models import CompiledModel -from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash from sqlbuild.compiler.fingerprints.models import Fingerprint +from sqlbuild.compiler.planner._helpers.identity.hashing import model_definition_hash from sqlbuild.compiler.planner.classes.migration_state_inspection import ( MigrationStateInspection, ) @@ -68,5 +68,8 @@ def _referencing_query_changed(*, model: CompiledModel, snapshot: WarehouseSnaps return ( recorded is not None and any(reference.ref_kind == SqlReferenceKind.REF for reference in model.references) - and compute_query_hash(model.query_sql) != recorded.definition_hash + and model_definition_hash( + model_name=model.name, query_sql=model.query_sql, dialect=snapshot.column_dialect + ) + != recorded.definition_hash ) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/reference_renames.py b/src/sqlbuild/compiler/planner/_helpers/changes/reference_renames.py index c5d439c829..6f028c37ab 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/reference_renames.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/reference_renames.py @@ -50,7 +50,8 @@ def origin_reference_names( name: origin for name, origin in zip(names, combination, strict=True) if origin != name } if mapping and compute_query_hash( - _substitute(query_sql=query_sql, spans=spans, mapping=mapping) + query_sql=_substitute(query_sql=query_sql, spans=spans, mapping=mapping), + dialect=dialect, ) == (recorded.definition_hash): return mapping return None diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/direct.py b/src/sqlbuild/compiler/planner/_helpers/identity/direct.py index c6774c1998..f67daea363 100644 --- a/src/sqlbuild/compiler/planner/_helpers/identity/direct.py +++ b/src/sqlbuild/compiler/planner/_helpers/identity/direct.py @@ -9,6 +9,7 @@ build_model_local_identity_hash, compose_native_graph_identity, graph_key_for_compiled_resource, + model_definition_hash, ) from sqlbuild.compiler.planner._helpers.identity.seed import build_seed_identity from sqlbuild.compiler.planner.main.identity._graph_identity import ( @@ -37,10 +38,13 @@ def build_direct_model_version_identities( scope: PlannerScope, source_version_hashes: dict[str, str] | None = None, hook_functions: tuple[DiscoveredHookFunction, ...] = (), + dialect: str | None, ) -> DirectModelVersionIdentities: """Compute current direct model identities from code and upstream identities.""" - function_local_hashes: dict[str, str] = build_function_local_hashes(functions=functions) + function_local_hashes: dict[str, str] = build_function_local_hashes( + functions=functions, dialect=dialect + ) hook_version_hashes: dict[str, str] = { name: identity.version_hash for name, identity in build_hook_identities(hook_functions).items() @@ -119,7 +123,9 @@ def build_direct_model_version_identities( ) model_metadata_jsons[model.name] = metadata_json local_hash: str = build_model_local_identity_hash( - query_sql=model.query_sql, + query_fingerprint=model_definition_hash( + model_name=model.name, query_sql=model.query_sql, dialect=dialect + ), metadata_json=metadata_json, ) model_local_hashes[model.name] = local_hash diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/functions.py b/src/sqlbuild/compiler/planner/_helpers/identity/functions.py index 2b81e411d7..496645b184 100644 --- a/src/sqlbuild/compiler/planner/_helpers/identity/functions.py +++ b/src/sqlbuild/compiler/planner/_helpers/identity/functions.py @@ -7,8 +7,8 @@ FunctionArgument, FunctionReturnColumn, ) -from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash from sqlbuild.compiler.fingerprints.models import Fingerprint +from sqlbuild.compiler.planner._helpers.identity.hashing import function_definition_hash from sqlbuild.compiler.planner.types import PlanReason @@ -29,10 +29,12 @@ def build_compiled_function_fingerprint_sql(function: CompiledFunction) -> str: def detect_function_change( *, + function: CompiledFunction, fingerprint_sql: str, fingerprint: Fingerprint | None, query_change_tracking: bool, full_refresh: bool, + dialect: str | None, ) -> PlanReason: """Resolve why a function is redeployed; callers decide their own replay.""" @@ -40,7 +42,15 @@ def detect_function_change( return PlanReason.FULL_REFRESH if fingerprint is None: return PlanReason.FIRST_RUN - if query_change_tracking and compute_query_hash(fingerprint_sql) != fingerprint.definition_hash: + if query_change_tracking and ( + function_definition_hash( + function_name=function.name, + fingerprint_sql=fingerprint_sql, + language=str(function.language), + dialect=dialect, + ) + != fingerprint.definition_hash + ): return PlanReason.QUERY_CHANGED return PlanReason.NO_CHANGE diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/hashing.py b/src/sqlbuild/compiler/planner/_helpers/identity/hashing.py index 62c2ab6156..3587dcbb6f 100644 --- a/src/sqlbuild/compiler/planner/_helpers/identity/hashing.py +++ b/src/sqlbuild/compiler/planner/_helpers/identity/hashing.py @@ -3,6 +3,8 @@ from __future__ import annotations import hashlib +from collections.abc import Callable +from functools import partial from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.fingerprints.constants import ( @@ -11,8 +13,20 @@ NODE_TYPE_TABLE_FN, NODE_TYPE_UDF, ) +from sqlbuild.compiler.fingerprints.exceptions import QueryFingerprintError +from sqlbuild.compiler.fingerprints.main.compute_function_definition_hash import ( + compute_function_definition_hash, +) +from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash +from sqlbuild.compiler.planner.constants import QUERY_FINGERPRINT_FAILED +from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import GraphNodeKey +_FINGERPRINT_HELP: str = ( + "SQLBuild fingerprints the dialect token stream of compiled SQL to detect changes. " + "Fix the SQL so the warehouse dialect tokenizer accepts it." +) + def stable_version_identity_hash(value: str) -> str: """Return the stable SHA-256 hash used for local and composed identities.""" @@ -20,10 +34,46 @@ def stable_version_identity_hash(value: str) -> str: return hashlib.sha256(value.encode("utf-8")).hexdigest() -def build_model_local_identity_hash(*, query_sql: str, metadata_json: str) -> str: - """Build a model's local identity hash from query SQL and non-query metadata.""" +def model_definition_hash(*, model_name: str, query_sql: str, dialect: str | None) -> str: + """Fingerprint a model query for change detection, naming the model when it cannot.""" + + return _planner_fingerprint( + subject=f"Model '{model_name}' SQL", + compute=partial(compute_query_hash, query_sql=query_sql, dialect=dialect), + ) + + +def function_definition_hash( + *, function_name: str, fingerprint_sql: str, language: str, dialect: str | None +) -> str: + """Fingerprint a function definition for change detection, naming it when it cannot.""" + + return _planner_fingerprint( + subject=f"Function '{function_name}'", + compute=partial( + compute_function_definition_hash, + fingerprint_sql=fingerprint_sql, + language=language, + dialect=dialect, + ), + ) + + +def _planner_fingerprint(*, subject: str, compute: Callable[[], str]) -> str: + try: + return compute() + except QueryFingerprintError as error: + raise PlannerInputError( + f"{subject} cannot be fingerprinted for change detection: {error}", + code=QUERY_FINGERPRINT_FAILED, + help=_FINGERPRINT_HELP, + ) from None + + +def build_model_local_identity_hash(*, query_fingerprint: str, metadata_json: str) -> str: + """Build a model's local identity hash from its query fingerprint and non-query metadata.""" - return stable_version_identity_hash("\n".join((query_sql, metadata_json))) + return stable_version_identity_hash("\n".join((query_fingerprint, metadata_json))) def graph_key_for_compiled_resource( diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py index 9500d43388..edf4edb742 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py @@ -8,7 +8,6 @@ from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import CompiledModel, CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.migrations.main._newest_column_event import ( newest_column_migration_event_mentioning, @@ -16,6 +15,7 @@ from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location from sqlbuild.compiler.migrations.models import ColumnMigrationEvent from sqlbuild.compiler.migrations.types import ColumnMigrationDecision, MigrationDiscovery +from sqlbuild.compiler.planner._helpers.identity.hashing import model_definition_hash from sqlbuild.compiler.planner._helpers.migrations.column_renames import ( identical_renames, rename_hints, @@ -195,8 +195,12 @@ def _candidate( if column.migrate_from is not None } fingerprint: Fingerprint | None = snapshot.fingerprints.models.get(model.name) - changed: bool = fingerprint is not None and fingerprint.definition_hash != compute_query_hash( - model.query_sql + changed: bool = ( + fingerprint is not None + and fingerprint.definition_hash + != model_definition_hash( + model_name=model.name, query_sql=model.query_sql, dialect=snapshot.column_dialect + ) ) if not warehouse or (not declared and not changed): return None @@ -441,7 +445,11 @@ def _overlay_snapshot( fingerprints[candidate.model.name] = replace( candidate.fingerprint, definition=candidate.model.query_sql, - definition_hash=compute_query_hash(candidate.model.query_sql), + definition_hash=model_definition_hash( + model_name=candidate.model.name, + query_sql=candidate.model.query_sql, + dialect=snapshot.column_dialect, + ), ) return replace( snapshot, diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py index d0a0c7daf2..6daf12837c 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py @@ -357,7 +357,7 @@ def _metadata_jsons( *, runtime: PlannerRuntime, models: tuple[CompiledModel, ...] ) -> dict[str, str]: function_hashes: dict[str, str] = build_function_local_hashes( - functions=runtime.project.functions + functions=runtime.project.functions, dialect=runtime.project.sql_analysis_dialect ) hook_hashes: dict[str, str] = { name: identity.version_hash diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index f04432235f..df0491ea04 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -218,6 +218,7 @@ def plan_model_migrations( handovers[request.model.name] = _equivalent_definition( model=request.model, handover=handover, + dialect=snapshot.column_dialect, destination_fingerprints=discovery.destination_fingerprints.get( request.model.name, () ), @@ -273,14 +274,18 @@ def _renamed_models( def _equivalent_definition( - *, model: CompiledModel, handover: Fingerprint | None, destination_fingerprints: tuple[str, ...] + *, + model: CompiledModel, + handover: Fingerprint | None, + destination_fingerprints: tuple[str, ...], + dialect: str | None, ) -> Fingerprint | None: if handover is None or stored_migration_fingerprint(handover) not in destination_fingerprints: return handover return replace( handover, definition=model.query_sql, - definition_hash=compute_query_hash(model.query_sql), + definition_hash=compute_query_hash(query_sql=model.query_sql, dialect=dialect), ) diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/identities.py b/src/sqlbuild/compiler/planner/_helpers/planning/identities.py index bac63e0d53..5b0595530e 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/identities.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/identities.py @@ -33,6 +33,7 @@ def build_planner_identity_context( seeds=project.seeds, scope=scopes.inspection_scope, hook_functions=project.hook_functions, + dialect=project.sql_analysis_dialect, ) return PlannerIdentityContext( version_identities=version_identities, @@ -42,6 +43,7 @@ def build_planner_identity_context( seeds=project.seeds, scope=scopes.stale_warning_scope, hook_functions=project.hook_functions, + dialect=project.sql_analysis_dialect, ) if include_stale_warning_identities else version_identities diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index 59f1ed55e5..4694c625d3 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -45,6 +45,7 @@ SCENARIO_HASH_PREFIX_LENGTH: int = 12 SCENARIO_ARTIFACT_KINDS: tuple[str, ...] = ("source", "ref", "seed", "dbt_ref", "model") SCENARIO_DEFAULT_IDENTIFIER_LIMIT: int = 63 +QUERY_FINGERPRINT_FAILED: str = "S025" SCENARIO_PLAN_INVALID_HASH_PREFIX: str = "S501" SCENARIO_PLAN_HASH_COLLISION: str = "S502" SCENARIO_PLAN_RELATION_COLLISION: str = "S503" diff --git a/src/sqlbuild/compiler/planner/main/identity/version_identity_function_hashes.py b/src/sqlbuild/compiler/planner/main/identity/version_identity_function_hashes.py index 8675c993a5..80886d310b 100644 --- a/src/sqlbuild/compiler/planner/main/identity/version_identity_function_hashes.py +++ b/src/sqlbuild/compiler/planner/main/identity/version_identity_function_hashes.py @@ -6,13 +6,15 @@ import json from sqlbuild.compiler.compile.models import CompiledFunction +from sqlbuild.compiler.planner._helpers.identity.hashing import function_definition_hash def build_function_local_hashes( *, functions: tuple[CompiledFunction, ...], + dialect: str | None, ) -> dict[str, str]: - """Derive local-only semantic hashes for functions.""" + """Derive local-only semantic hashes for functions; SQL bodies ignore layout and comments.""" hashes: dict[str, str] = {} for function in functions: @@ -28,7 +30,12 @@ def build_function_local_hashes( "arguments": arguments, "returns": function.returns, "return_columns": return_columns, - "body_sql": function.body_sql, + "body_sql": function_definition_hash( + function_name=function.name, + fingerprint_sql=function.body_sql, + language=function.language.value, + dialect=dialect, + ), "language": function.language.value, "runtime_version": function.runtime_version, "entry_point": function.entry_point, diff --git a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py index 094d1547cf..2796b40a5c 100644 --- a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py +++ b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py @@ -91,7 +91,9 @@ def materialize_query_diff_artifact( connection=connection, relation=artifact.relation, ) - definition_hash: str = compute_query_hash(sql) + definition_hash: str = compute_query_hash( + query_sql=sql, dialect=adapter.sql_analysis_dialect() + ) metadata_json: str = json.dumps( { "artifact_version": _ARTIFACT_VERSION, diff --git a/src/sqlbuild/executor/functions/main/_execute.py b/src/sqlbuild/executor/functions/main/_execute.py index 2c07cd3bc8..a71d7abb86 100644 --- a/src/sqlbuild/executor/functions/main/_execute.py +++ b/src/sqlbuild/executor/functions/main/_execute.py @@ -10,7 +10,9 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import FunctionDefinition from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash +from sqlbuild.compiler.fingerprints.main.compute_function_definition_hash import ( + compute_function_definition_hash, +) from sqlbuild.compiler.fingerprints.main.write import write_fingerprint from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import FunctionPlanEntry @@ -153,6 +155,11 @@ def _try_write_function_fingerprint( statement_recorder=statement_recorder, ) schema_fp: str = hashlib.sha256(b"").hexdigest() + definition_hash: str = compute_function_definition_hash( + fingerprint_sql=entry.fingerprint_query_sql, + language=str(entry.language), + dialect=adapter.sql_analysis_dialect(), + ) fingerprint: Fingerprint = Fingerprint( node_type=function_node_type(return_columns=entry.return_columns), node_name=entry.name, @@ -160,8 +167,8 @@ def _try_write_function_fingerprint( target_schema=entry.destination.schema, target_name=entry.destination.name, run_id=run_id, - definition_hash=compute_query_hash(entry.fingerprint_query_sql), - version_hash=compute_query_hash(entry.fingerprint_query_sql), + definition_hash=definition_hash, + version_hash=definition_hash, schema_fingerprint=schema_fp, definition=entry.fingerprint_query_sql, metadata_json="{}", diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index 6580032c99..1393bb9325 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -1733,7 +1733,7 @@ def _prepare_microbatch_history( cursor_type=context.entry.cursor_type or "", cursor_grain=context.entry.cursor_grain, model_version_hash=expected_version_hash, - definition_hash=compute_query_hash(context.entry.fingerprint_query_sql), + definition_hash=_definition_hash(context=context), fingerprint_status=MicrobatchFingerprintStatus.KNOWN, replay_requirement_id=requirement_id, required_model_version_hash=expected_version_hash, @@ -1907,7 +1907,7 @@ def _record_microbatch_completion( cursor_type=context.entry.cursor_type or "", cursor_grain=context.entry.cursor_grain, model_version_hash=_expected_model_version_hash(context=context), - definition_hash=compute_query_hash(context.entry.fingerprint_query_sql), + definition_hash=_definition_hash(context=context), fingerprint_status=MicrobatchFingerprintStatus.KNOWN, replay_requirement_id=( recovery_origin.replay_requirement_id @@ -1953,11 +1953,18 @@ def _microbatch_run_type(*, context: ModelMaterializationContext) -> MicrobatchR return MicrobatchRunType.NORMAL +def _definition_hash(*, context: ModelMaterializationContext) -> str: + return compute_query_hash( + query_sql=context.entry.fingerprint_query_sql, + dialect=context.adapter.sql_analysis_dialect(), + ) + + def _expected_model_version_hash(*, context: ModelMaterializationContext) -> str: return ( context.microbatch_model_version_hash or context.entry.fingerprint_version_hash - or compute_query_hash(context.entry.fingerprint_query_sql) + or _definition_hash(context=context) ) diff --git a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py index b823a163d4..0fe0b2a525 100644 --- a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py +++ b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py @@ -41,6 +41,9 @@ def try_write_fingerprint( ) try: schema_fp: str = hashlib.sha256(b"").hexdigest() + definition_hash: str = compute_query_hash( + query_sql=entry.fingerprint_query_sql, dialect=adapter.sql_analysis_dialect() + ) metadata_json: str = model_fingerprint_metadata_with_audit_gate( metadata_json=entry.fingerprint_metadata_json, model_audits=model_audits, @@ -57,9 +60,8 @@ def try_write_fingerprint( target_schema=entry.destination.schema, target_name=entry.destination.name, run_id=run_id, - definition_hash=compute_query_hash(entry.fingerprint_query_sql), - version_hash=entry.fingerprint_version_hash - or compute_query_hash(entry.fingerprint_query_sql), + definition_hash=definition_hash, + version_hash=entry.fingerprint_version_hash or definition_hash, schema_fingerprint=schema_fp, definition=entry.fingerprint_query_sql, metadata_json=metadata_json, diff --git a/src/sqlbuild/lint/_helpers/fixes.py b/src/sqlbuild/lint/_helpers/fixes.py index caf897b212..41ba1582f3 100644 --- a/src/sqlbuild/lint/_helpers/fixes.py +++ b/src/sqlbuild/lint/_helpers/fixes.py @@ -1,11 +1,241 @@ -"""Deterministic lint edit planning, validation, and atomic persistence.""" +"""Conservative, compiler-verified fixed-point Rule repair planning.""" from __future__ import annotations import os import tempfile +from dataclasses import dataclass, field, replace from pathlib import Path +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.compiler.compile.models import CompiledModel, CompiledObjectKey, CompiledProject +from sqlbuild.compiler.compile.types import TypedSqlValueRenderer +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.pipeline.main.project import compile_project +from sqlbuild.lint._helpers.native_format import with_newline_style +from sqlbuild.lint.constants import LINT_ENGINE_NATIVE, VIOLATION_SEVERITY_FAULT +from sqlbuild.lint.exceptions import ProjectCompileError +from sqlbuild.lint.main.run_lint import run_lint +from sqlbuild.lint.models import ( + FormatChange, + LintConfig, + LintEdit, + LintRunResult, + LintViolation, + RuleFixResult, +) +from sqlbuild.lint.types import RuleFixStatus + +_SAFE_RULES: frozenset[str] = frozenset( + { + "SQBRSQL002", + "SQBRSQL003", + "SQBRSQL005", + "SQBRSQL006", + "SQBRSQL008", + "SQBRSQL017", + "SQBRSQL025", + } +) +_MAX_PASSES: int = 128 +_APPLIED: str = "applied" +_CONDITIONLESS_JOIN_CODE: str = "SQBRSQL003" +_SNOWFLAKE_DIALECT: str = "snowflake" +_APPLIED_REASON: str = "Meaning-preserving rewrite; compiler invariants verified" +_VERIFICATION_FAILURE: str = ( + "Compilation could not prove unchanged output columns, types, dependencies and lineage" +) + + +@dataclass +class _FixSession: + """One fixed-point run that verifies each file's edits and reverts only failing files.""" + + project_dir: Path + original: dict[Path, str] + config: LintConfig + adapter: BaseAdapter + inputs: DiscoveredProjectInputs + reports: list[RuleFixResult] = field(default_factory=list) + blocked: set[Path] = field(default_factory=set) + baseline: CompiledProject | None = None + + def run(self) -> tuple[dict[Path, str], tuple[RuleFixResult, ...], list[LintViolation]]: + current: dict[Path, str] = dict(self.original) + model_paths: frozenset[Path] = frozenset( + model.file_path for model in self.inputs.model_files + ) + seen: set[tuple[tuple[Path, str], ...]] = set() + for _ in range(_MAX_PASSES): + state: tuple[tuple[Path, str], ...] = tuple(sorted(current.items())) + if state in seen: + return self._decline(reason="Rule fixes did not converge") + seen.add(state) + edits, pending, remaining = self._plan_pass(files=current, model_paths=model_paths) + if not edits: + return current, tuple(self.reports + remaining), self._blocked_faults() + verified: dict[Path, str] | None = self._verify_pass( + current=current, edits=edits, pending=pending + ) + if verified is None: + self.reports.extend(pending) + return self._decline(reason=_VERIFICATION_FAILURE) + current = verified + return self._decline(reason="Rule fix pass limit exceeded") + + def _plan_pass( + self, *, files: dict[Path, str], model_paths: frozenset[Path] + ) -> tuple[dict[Path, list[LintEdit]], list[RuleFixResult], list[RuleFixResult]]: + result: LintRunResult = run_lint( + project_dir=self.project_dir, + config=replace(self.config, enabled_native_rules=("SQBRSQL",)), + value_renderer=self.adapter, + discovered_inputs=_updated_inputs(inputs=self.inputs, files=files), + source_files=files, + selected_paths=frozenset(files), + ) + edits: dict[Path, list[LintEdit]] = {} + pending: list[RuleFixResult] = [] + remaining: list[RuleFixResult] = [] + for violation in result.violations: + if not violation.code.startswith("SQBRSQL"): + continue + reason: str | None = self._refusal(violation=violation, model_paths=model_paths) + if reason is not None or violation.fix is None: + remaining.append( + _report( + violation=violation, + status="refused" if reason is not None else "unavailable", + reason=reason or "No meaning-preserving automatic fix is available", + ) + ) + continue + edit: LintEdit = violation.fix + chosen: list[LintEdit] = edits.setdefault(edit.file_path, []) + if any(edit.start < other.end and other.start < edit.end for other in chosen): + continue + chosen.append(edit) + pending.append(_report(violation=violation, status="applied", reason=_APPLIED_REASON)) + return edits, pending, remaining + + def _refusal(self, *, violation: LintViolation, model_paths: frozenset[Path]) -> str | None: + if violation.file_path in self.blocked: + return _VERIFICATION_FAILURE + if violation.file_path not in model_paths: + return "Compiler-verified Rule fixes currently require a model SQL body" + if violation.fix is not None and violation.code not in _SAFE_RULES: + return "This proposed rewrite is not proven meaning-preserving in this dialect" + if violation.code == _CONDITIONLESS_JOIN_CODE and self.config.dialect != _SNOWFLAKE_DIALECT: + return "Conditionless JOIN equivalence is only established for Snowflake" + return violation.fix_unavailable_reason + + def _verify_pass( + self, + *, + current: dict[Path, str], + edits: dict[Path, list[LintEdit]], + pending: list[RuleFixResult], + ) -> dict[Path, str] | None: + """Accept every file whose edits verify; revert failing files to their originals.""" + + candidate: dict[Path, str] = _with_edits(files=current, edits=edits) + if self._equivalent_to_baseline(files=candidate): + self.reports.extend(pending) + return candidate + failing: set[Path] = { + path + for path in edits + if not self._equivalent_to_baseline( + files=_with_edits(files=current, edits={path: edits[path]}) + ) + } + if not failing: + return None + self.blocked.update(failing) + reverted: dict[Path, str] = { + path: self.original[path] if path in self.blocked else contents + for path, contents in current.items() + } + accepted: dict[Path, list[LintEdit]] = { + path: path_edits for path, path_edits in edits.items() if path not in failing + } + candidate = _with_edits(files=reverted, edits=accepted) + if not self._equivalent_to_baseline(files=candidate): + return None + self.reports[:] = [ + _refused(report=report) if report.file_path in self.blocked else report + for report in (*self.reports, *pending) + ] + return candidate + + def _equivalent_to_baseline(self, *, files: dict[Path, str]) -> bool: + try: + if self.baseline is None: + self.baseline = compile_project(discovered_inputs=self.inputs, adapter=self.adapter) + compiled: CompiledProject = compile_project( + discovered_inputs=_updated_inputs(inputs=self.inputs, files=files), + adapter=self.adapter, + ) + valid: bool = _equivalent( + before=self.baseline, + after=compiled, + affected=_affected_models( + project=self.baseline, + changed=frozenset( + path for path, contents in files.items() if contents != self.original[path] + ), + project_dir=self.project_dir, + ), + ) + except Exception: + valid = False + return valid + + def _blocked_faults(self) -> list[LintViolation]: + return [_verification_fault(path=path) for path in sorted(self.blocked)] + + def _decline( + self, *, reason: str + ) -> tuple[dict[Path, str], tuple[RuleFixResult, ...], list[LintViolation]]: + affected: set[Path] = { + report.file_path for report in self.reports if report.status == _APPLIED + } | self.blocked + return ( + self.original, + tuple(_refused(report=report, reason=reason) for report in self.reports), + [_verification_fault(path=path, reason=reason) for path in sorted(affected)], + ) + + +def persist_changes( + *, files: dict[Path, str], updated: dict[Path, str], newlines: dict[Path, str], write: bool +) -> tuple[FormatChange, ...]: + changes: list[FormatChange] = [] + for path, contents in sorted(updated.items()): + after: str = with_newline_style(contents=contents, newline=newlines[path]) + before: str = with_newline_style(contents=files[path], newline=newlines[path]) + if after == before: + continue + changes.append(FormatChange(file_path=path, before=before, after=after)) + if write: + write_atomically(path=path, contents=after) + return tuple(changes) + + +def finalize_fix_reports( + *, reports: tuple[RuleFixResult, ...], declined_paths: set[Path] +) -> tuple[RuleFixResult, ...]: + return tuple( + replace( + report, + status="refused", + reason="Final layout formatting failed; original file retained", + ) + if report.status == _APPLIED and report.file_path in declined_paths + else report + for report in reports + ) + def write_atomically(*, path: Path, contents: str) -> None: descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) @@ -19,3 +249,137 @@ def write_atomically(*, path: Path, contents: str) -> None: os.replace(temporary_path, path) finally: temporary_path.unlink(missing_ok=True) + + +def plan_rule_fixes( + *, + project_dir: Path, + files: dict[Path, str], + config: LintConfig, + adapter: TypedSqlValueRenderer | None, + discovered_inputs: DiscoveredProjectInputs | None, + enabled: bool, +) -> tuple[dict[Path, str], tuple[RuleFixResult, ...], list[LintViolation]]: + """Plan in memory; never expose an unverified intermediate version on disk.""" + + if not enabled: + return files, (), [] + if not isinstance(adapter, BaseAdapter) or discovered_inputs is None: + raise ProjectCompileError("format --fix requires discovered compiler inputs and an adapter") + session: _FixSession = _FixSession( + project_dir=project_dir, + original=files, + config=config, + adapter=adapter, + inputs=discovered_inputs, + ) + return session.run() + + +def _with_edits(*, files: dict[Path, str], edits: dict[Path, list[LintEdit]]) -> dict[Path, str]: + candidate: dict[Path, str] = dict(files) + for path, path_edits in edits.items(): + contents: str = candidate[path] + for edit in sorted(path_edits, key=lambda item: item.start, reverse=True): + contents = contents[: edit.start] + edit.replacement + contents[edit.end :] + candidate[path] = contents + return candidate + + +def _report(*, violation: LintViolation, status: RuleFixStatus, reason: str) -> RuleFixResult: + return RuleFixResult( + file_path=violation.file_path, + code=violation.code, + line=violation.line, + status=status, + reason=reason, + ) + + +def _refused(*, report: RuleFixResult, reason: str = _VERIFICATION_FAILURE) -> RuleFixResult: + if report.status != _APPLIED: + return report + return replace(report, status="refused", reason=reason) + + +def _verification_fault(*, path: Path, reason: str = _VERIFICATION_FAILURE) -> LintViolation: + return LintViolation( + file_path=path, + line=1, + column=1, + code="format-fix-verification-failed", + message=reason, + severity=VIOLATION_SEVERITY_FAULT, + engine=LINT_ENGINE_NATIVE, + remediation="Restructure the SQL manually and recompile; the original file was retained.", + ) + + +def _updated_inputs( + *, inputs: DiscoveredProjectInputs, files: dict[Path, str] +) -> DiscoveredProjectInputs: + return replace( + inputs, + model_files=tuple( + replace( + model, + contents=files[model.file_path], + query_sql=files[model.file_path][model.contents.index(model.query_sql) :].strip(), + ) + if model.file_path in files and files[model.file_path] != model.contents + else model + for model in inputs.model_files + ), + ) + + +def _affected_models( + *, project: CompiledProject, changed: frozenset[Path], project_dir: Path +) -> frozenset[CompiledObjectKey]: + """Models in changed files and everything downstream; all models if a file is unmatched.""" + + resolved: frozenset[Path] = frozenset(path.resolve() for path in changed) + affected: set[CompiledObjectKey] = { + model.key + for model in project.models + if (project_dir / model.relative_path).resolve() in resolved + } + if len(affected) < len(resolved): + return frozenset(model.key for model in project.models) + grew: bool = True + while grew: + grew = False + for model in project.models: + if model.key not in affected and any(dep in affected for dep in model.deps): + affected.add(model.key) + grew = True + return frozenset(affected) + + +def _equivalent( + *, before: CompiledProject, after: CompiledProject, affected: frozenset[CompiledObjectKey] +) -> bool: + """Affected models must prove unchanged columns, dependencies and lineage.""" + + if any(diagnostic.is_error for diagnostic in (*before.diagnostics, *after.diagnostics)): + return False + if {model.key for model in before.models} != {model.key for model in after.models}: + return False + after_models: dict[CompiledObjectKey, CompiledModel] = { + model.key: model for model in after.models + } + for model in before.models: + if model.key not in affected: + continue + candidate: CompiledModel = after_models[model.key] + if ( + model.inferred_columns is None + or candidate.inferred_columns is None + or model.fast_lineage_columns is None + or candidate.fast_lineage_columns is None + or model.inferred_columns != candidate.inferred_columns + or model.deps != candidate.deps + or tuple(model.fast_lineage_columns) != tuple(candidate.fast_lineage_columns) + ): + return False + return True diff --git a/src/sqlbuild/lint/_helpers/native.py b/src/sqlbuild/lint/_helpers/native.py index a2b7a2684e..9855d1123e 100644 --- a/src/sqlbuild/lint/_helpers/native.py +++ b/src/sqlbuild/lint/_helpers/native.py @@ -49,6 +49,24 @@ _HEXADECIMAL_CHARACTERS: frozenset[str] = frozenset("0123456789abcdefABCDEF") +_AUDITS_KEY: re.Pattern[str] = re.compile(r"\baudits\s*\[") +_OPENERS: dict[str, str] = {"(": ")", "[": "]"} +_AUDIT_LIST_OPENER: str = "[" +_HEADER_ITEM_SEPARATOR: str = "," +_AUDITS_KEYWORD: str = "audits" +_AUDIT_WRAP_MAX_PASSES: int = 64 + + +@dataclass(frozen=True) +class _Group: + """One bracket group inside a header audit list.""" + + open_index: int + close_index: int + depth: int + commas: tuple[int, ...] + + @dataclass(frozen=True) class _RelocationOutcome: """Result of relocating a leading comment into the model description.""" @@ -188,7 +206,21 @@ def format_native_headers( headers=relocated_headers, config=config, ) - return wrapped, relocation.faults + audits_wrapped: str = _wrap_header_audits( + contents=wrapped, + headers=scan_headers(contents=wrapped), + line_width=config.line_width, + ) + return audits_wrapped, relocation.faults + + +def _header_values_or_unique(*, kind: str, header_text: str) -> object: + """Parse header values; an unparseable header compares unequal to every other result.""" + + try: + return _parse_header_values(kind=kind, header_text=header_text) + except ModelSqlParseError: + return object() def _lint_header_values( @@ -703,3 +735,107 @@ def _inner_header_text(*, header_text: str) -> str: if open_index < 0 or close_index <= open_index: return "" return header_text[open_index + 1 : close_index] + + +def _wrap_header_audits(*, contents: str, headers: tuple[HeaderSpan, ...], line_width: int) -> str: + """Split audit lists in header lines over `line_width`; keep only value-preserving rewrites.""" + + updated: str = contents + for header in reversed(headers): + header_text: str = updated[header.start : header.end] + wrapped: str = _wrap_header(header_text=header_text, line_width=line_width) + if wrapped == header_text: + continue + if _header_values_or_unique(kind=header.kind, header_text=wrapped) != ( + _header_values_or_unique(kind=header.kind, header_text=header_text) + ): + continue + updated = updated[: header.start] + wrapped + updated[header.end :] + return updated + + +def _wrap_header(*, header_text: str, line_width: int) -> str: + text: str = header_text + for _ in range(_AUDIT_WRAP_MAX_PASSES): + exploded: str | None = _explode_first_long_line(text=text, line_width=line_width) + if exploded is None: + return text + text = exploded + return text + + +def _explode_first_long_line(*, text: str, line_width: int) -> str | None: + groups: tuple[_Group, ...] = _audit_groups(text=text) + line_start: int = 0 + for line in text.split("\n"): + line_end: int = line_start + len(line) + if len(line) > line_width and _LINE_COMMENT_PREFIX not in line: + candidates: list[_Group] = [ + group + for group in groups + if line_start <= group.open_index + and group.close_index < line_end + and group.close_index - line_start >= line_width + and text[group.open_index + 1 : group.close_index].strip() + ] + if candidates: + chosen: _Group = min(candidates, key=lambda group: group.depth) + indent: str = line[: len(line) - len(line.lstrip())] + return _explode(text=text, group=chosen, indent=indent) + line_start = line_end + 1 + return None + + +def _explode(*, text: str, group: _Group, indent: str) -> str: + bounds: list[int] = [group.open_index, *group.commas, group.close_index] + items: list[str] = [ + text[start + 1 : end].strip() for start, end in zip(bounds[:-1], bounds[1:], strict=True) + ] + items = [item for item in items if item] + inner: str = indent + _HEADER_INDENT + body: str = "".join(f"\n{inner}{item}," for item in items) + return f"{text[: group.open_index + 1]}{body}\n{indent}{text[group.close_index :]}" + + +def _audit_groups(*, text: str) -> tuple[_Group, ...]: + """Bracket groups at or inside each `audits [...]` value, outside quoted text.""" + + groups: list[_Group] = [] + open_stack: list[tuple[int, str, list[int]]] = [] + audit_depths: list[int] = [] + index: int = 0 + while index < len(text): + character: str = text[index] + if character in _QUOTE_CHARACTERS: + index = quoted_value_end(text=text, start=index) + continue + if character in _OPENERS: + if character == _AUDIT_LIST_OPENER and _AUDITS_KEY.match( + text, _audits_key_start(text=text, bracket=index) + ): + audit_depths.append(len(open_stack)) + open_stack.append((index, _OPENERS[character], [])) + elif open_stack and character == open_stack[-1][1]: + start, _, commas = open_stack.pop() + if audit_depths and len(open_stack) >= audit_depths[-1]: + groups.append( + _Group( + open_index=start, + close_index=index, + depth=len(open_stack), + commas=tuple(commas), + ) + ) + if audit_depths and len(open_stack) == audit_depths[-1]: + audit_depths.pop() + elif open_stack and character == _HEADER_ITEM_SEPARATOR: + open_stack[-1][2].append(index) + index += 1 + return tuple(groups) + + +def _audits_key_start(*, text: str, bracket: int) -> int: + start: int = bracket + while start > 0 and text[start - 1].isspace(): + start -= 1 + return max(0, start - len(_AUDITS_KEYWORD)) diff --git a/src/sqlbuild/lint/_helpers/native_format.py b/src/sqlbuild/lint/_helpers/native_format.py index 991abe742f..9d22a2d0a8 100644 --- a/src/sqlbuild/lint/_helpers/native_format.py +++ b/src/sqlbuild/lint/_helpers/native_format.py @@ -31,7 +31,7 @@ class _PreparedBody: start: int end: int trailing: str - cache_key: tuple[str, str] + cache_key: tuple[str, str, str] interpolation_sites: tuple[InterpolationSite, ...] @@ -54,7 +54,7 @@ def format_native_sql_bodies( prepared_by_path: dict[Path, tuple[_PreparedBody, ...]] = {} faults: list[LintViolation] = [] - requests_by_key: dict[tuple[str, str], dict[str, object]] = {} + requests_by_key: dict[tuple[str, str, str], dict[str, object]] = {} for file_path, contents in sorted(files.items()): headers: tuple[HeaderSpan, ...] = scan_headers(contents=contents) body_ranges: tuple[tuple[int, int], ...] = lint_body_ranges( @@ -71,7 +71,12 @@ def format_native_sql_bodies( neutralized, sites = neutralize_interpolation( body=core, dialect=config.dialect, for_formatting=True ) - cache_key: tuple[str, str] = (neutralized, config.dialect) + token_widths: dict[str, int] = _sentinel_widths(sites=sites) + cache_key: tuple[str, str, str] = ( + neutralized, + config.dialect, + repr(sorted(token_widths.items())), + ) requests_by_key.setdefault( cache_key, { @@ -79,6 +84,8 @@ def format_native_sql_bodies( "sql": neutralized, "dialect": config.dialect, "max_function_call_depth": POLYGLOT_MAX_FUNCTION_CALL_DEPTH, + "line_width": config.line_width, + "token_widths": token_widths, }, ) prepared_bodies.append( @@ -91,7 +98,7 @@ def format_native_sql_bodies( ) ) prepared_by_path[file_path] = tuple(prepared_bodies) - response_cache: dict[tuple[str, str], dict[str, Any]] = _format_responses( + response_cache: dict[tuple[str, str, str], dict[str, Any]] = _format_responses( requests_by_key=requests_by_key ) formatted_files: dict[Path, str] = {} @@ -145,6 +152,21 @@ def format_native_sql_bodies( return formatted_files, faults +def _sentinel_widths(*, sites: tuple[InterpolationSite, ...]) -> dict[str, int]: + """Map each sentinel to the printed width of the SQLBuild syntax it stands in for.""" + + return { + site.sentinel.upper(): _printed_width(text=site.original_text) + for site in sites + if len(site.original_text) != len(site.sentinel) or LINE_FEED in site.original_text + } + + +def _printed_width(*, text: str) -> int: + lines: list[str] = text.splitlines() or [""] + return max(map(len, lines)) + + def _format_fault(*, file_path: Path, contents: str, start: int, reason: str) -> LintViolation: return LintViolation( file_path=file_path, @@ -162,11 +184,11 @@ def _format_fault(*, file_path: Path, contents: str, start: int, reason: str) -> def _format_responses( - *, requests_by_key: dict[tuple[str, str], dict[str, object]] -) -> dict[tuple[str, str], dict[str, Any]]: + *, requests_by_key: dict[tuple[str, str, str], dict[str, object]] +) -> dict[tuple[str, str, str], dict[str, Any]]: if not requests_by_key: return {} - ordered_requests: tuple[tuple[tuple[str, str], dict[str, object]], ...] = tuple( + ordered_requests: tuple[tuple[tuple[str, str, str], dict[str, object]], ...] = tuple( sorted(requests_by_key.items()) ) try: @@ -183,7 +205,7 @@ def _format_responses( raise NativeLintError(str(error)) from error if not isinstance(decoded, list) or len(decoded) != len(ordered_requests): raise NativeLintError("native formatter returned an invalid batch response") - responses: dict[tuple[str, str], dict[str, Any]] = {} + responses: dict[tuple[str, str, str], dict[str, Any]] = {} for (cache_key, _), raw_response in zip(ordered_requests, decoded, strict=True): if not isinstance(raw_response, dict): raise NativeLintError("native formatter returned a non-object response") diff --git a/src/sqlbuild/lint/_helpers/native_sql.py b/src/sqlbuild/lint/_helpers/native_sql.py index af631c937b..21b28cba04 100644 --- a/src/sqlbuild/lint/_helpers/native_sql.py +++ b/src/sqlbuild/lint/_helpers/native_sql.py @@ -325,6 +325,21 @@ def _authored_fix(*, raw_fix: object, code: str, body: LintBody) -> LintEdit | N or end > len(body.lint_text) ): raise NativeLintError("native lint diagnostic has an invalid fix edit") + region_start: int = start + region_end: int = end + for spans in reversed(body.passes): + if any( + region_start < span.output_end + and span.output_start < region_end + or ( + span.output_start == span.output_end + and region_start <= span.output_start < region_end + ) + for span in spans + ): + return None + region_start = map_expanded_offset(offset=region_start, passes=(spans,)).offset + region_end = map_expanded_offset(offset=region_end, passes=(spans,)).offset mapped_start: MappedOffset = map_expanded_offset(offset=start, passes=body.passes) mapped_last: MappedOffset = map_expanded_offset(offset=end - 1, passes=body.passes) if ( diff --git a/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py b/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py index 624f9ab04f..6c2f75db11 100644 --- a/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py +++ b/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py @@ -16,6 +16,7 @@ from sqlbuild.lint.constants import ( BACKTICK_CHARACTER, CLOSING_PAREN_CHARACTER, + DOLLAR_QUOTE_CHARACTER, EXPECTED_SENTINEL_OCCURRENCES, IDENTIFIER_EXTRA_CHARACTER, INTERPOLATION_NAME_EXTRA_CHARACTERS, @@ -49,13 +50,20 @@ _LINE_COMMENT_START: str = "--" _BLOCK_COMMENT_START: str = "/*" _BLOCK_COMMENT_END: str = "*/" +_DOLLAR_QUOTE_SCAN_PATTERN: str = ( + r"(?(?:[A-Za-z_][A-Za-z0-9_]*)?)\$" + r"(?:[\s\S]*?\$(?P=dollar_tag)\$|[\s\S]*\Z)" +) _NON_CODE_SCAN_PATTERN: str = ( r"--[^\n]*(?:\n|\Z)" r"|/\*[\s\S]*?(?:\*/|\Z)" r"|'(?:\\.|''|[^'\\])*(?:'|\Z)" r'|"(?:\\.|""|[^"\\])*(?:"|\Z)' + rf"|{_DOLLAR_QUOTE_SCAN_PATTERN}" ) _BACKTICK_SCAN_PATTERN: str = r"|`(?:``|[^`])*(?:`|\Z)" +_DOLLAR_QUOTE_PATTERN: re.Pattern[str] = re.compile(_DOLLAR_QUOTE_SCAN_PATTERN) @cache @@ -327,6 +335,11 @@ def _matching_paren_end(*, body: str, opening_index: int, backtick_identifiers: comment_end: int = body.find(_BLOCK_COMMENT_END, index + len(_BLOCK_COMMENT_START)) index = length if comment_end < 0 else comment_end + len(_BLOCK_COMMENT_END) continue + if quote_character is None and body.startswith(DOLLAR_QUOTE_CHARACTER, index): + dollar_quote: re.Match[str] | None = _DOLLAR_QUOTE_PATTERN.match(body, index) + if dollar_quote is not None: + index = dollar_quote.end() + continue character: str = body[index] if quote_character is not None: if ( diff --git a/src/sqlbuild/lint/constants.py b/src/sqlbuild/lint/constants.py index acd6f8ac60..36821aa5a3 100644 --- a/src/sqlbuild/lint/constants.py +++ b/src/sqlbuild/lint/constants.py @@ -80,6 +80,7 @@ SQL_QUOTE_CHARACTERS: frozenset[str] = frozenset({"'", '"'}) BACKTICK_CHARACTER: str = "`" +DOLLAR_QUOTE_CHARACTER: str = "$" SQL_ESCAPE_CHARACTER: str = "\\" SINGLE_QUOTE_CHARACTER: str = "'" OPENING_PAREN_CHARACTER: str = "(" diff --git a/src/sqlbuild/lint/main/run_format.py b/src/sqlbuild/lint/main/run_format.py index 145fbae72f..ea8a98be5e 100644 --- a/src/sqlbuild/lint/main/run_format.py +++ b/src/sqlbuild/lint/main/run_format.py @@ -7,6 +7,7 @@ from sqlbuild.compiler.compile.types import TypedSqlValueRenderer from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.classes.fixture_null_autofix import FixtureNullAutofix +from sqlbuild.lint._helpers.fixes import finalize_fix_reports, persist_changes, plan_rule_fixes from sqlbuild.lint._helpers.headers import scan_headers from sqlbuild.lint._helpers.native import ( format_native_headers, @@ -15,10 +16,7 @@ reject_unparseable_header_rewrites, safe_format_files, ) -from sqlbuild.lint._helpers.native_format import ( - format_native_sql_bodies, - with_newline_style, -) +from sqlbuild.lint._helpers.native_format import format_native_sql_bodies from sqlbuild.lint._helpers.project_files import collect_project_files, sort_violations from sqlbuild.lint._helpers.suppressions import apply_suppressions from sqlbuild.lint.constants import VIOLATION_SEVERITY_WARNING @@ -40,6 +38,7 @@ def run_format( discovered_inputs: DiscoveredProjectInputs | None = None, fixtures_only: bool = False, write: bool = True, + fix: bool = False, ) -> LintRunResult: """Format all DSL files in place and report the violations that remain.""" @@ -47,6 +46,14 @@ def run_format( project_dir=project_dir, selected_paths=selected_paths ) safe_files, newline_by_path = safe_format_files(files=files, config=config) + safe_files, rule_fixes, rule_faults = plan_rule_fixes( + project_dir=project_dir, + files=safe_files, + config=config, + adapter=value_renderer, + discovered_inputs=discovered_inputs, + enabled=fix, + ) updated_contents, format_faults = _apply_fixes( files=safe_files, config=config, @@ -54,34 +61,15 @@ def run_format( discovered_inputs=discovered_inputs, fixtures_only=fixtures_only, ) + format_faults.extend(rule_faults) + updated_contents: dict[Path, str] = {**safe_files, **updated_contents} declined_paths: set[Path] = {fault.file_path for fault in format_faults} updated_contents: dict[Path, str] = { path: contents for path, contents in updated_contents.items() if path not in declined_paths } - formatted: list[Path] = [] - changes: list[FormatChange] = [] - file_path: Path - new_contents: str - for file_path, new_contents in updated_contents.items(): - rendered_contents: str = with_newline_style( - contents=new_contents, newline=newline_by_path[file_path] - ) - original_contents: str = with_newline_style( - contents=files[file_path], newline=newline_by_path[file_path] - ) - if rendered_contents == original_contents: - continue - changes.append( - FormatChange( - file_path=file_path, - before=original_contents, - after=rendered_contents, - ) - ) - if write: - with file_path.open("w", encoding="utf-8", newline="") as handle: - _ = handle.write(rendered_contents) - formatted.append(file_path) + changes: tuple[FormatChange, ...] = persist_changes( + files=files, updated=updated_contents, newlines=newline_by_path, write=write + ) violations: list[LintViolation] = ( [] if fixtures_only @@ -105,8 +93,9 @@ def run_format( contents_by_path=final_contents, ) ), - formatted_files=tuple(sorted(formatted)), - format_changes=tuple(sorted(changes, key=lambda change: change.file_path)), + formatted_files=tuple(change.file_path for change in changes), + format_changes=changes, + rule_fixes=finalize_fix_reports(reports=rule_fixes, declined_paths=declined_paths), source_texts=final_contents, ) diff --git a/src/sqlbuild/lint/models.py b/src/sqlbuild/lint/models.py index b46c5cfd9f..b5e1292392 100644 --- a/src/sqlbuild/lint/models.py +++ b/src/sqlbuild/lint/models.py @@ -9,7 +9,7 @@ from sqlbuild.compiler.compile.models import ExpansionSpan from sqlbuild.compiler.scopes.types import DeclarationKind from sqlbuild.lint.constants import VIOLATION_SEVERITY_FAULT, VIOLATION_SEVERITY_WARNING -from sqlbuild.lint.types import LintSeverity +from sqlbuild.lint.types import LintSeverity, RuleFixStatus @dataclass(frozen=True) @@ -96,6 +96,17 @@ class LintViolation: fix_unavailable_reason: str | None = None +@dataclass(frozen=True) +class RuleFixResult: + """One applied, refused, or unavailable semantic Rule fix.""" + + file_path: Path + code: str + line: int + status: RuleFixStatus + reason: str + + @dataclass(frozen=True) class FormatChange: """One deterministic file-formatting change.""" @@ -126,6 +137,7 @@ class LintRunResult: violations: tuple[LintViolation, ...] formatted_files: tuple[Path, ...] format_changes: tuple[FormatChange, ...] = () + rule_fixes: tuple[RuleFixResult, ...] = () source_texts: Mapping[Path, str] = field(default_factory=dict, repr=False, compare=False) @property diff --git a/src/sqlbuild/lint/types.py b/src/sqlbuild/lint/types.py index 04a3317fef..298cbb8b50 100644 --- a/src/sqlbuild/lint/types.py +++ b/src/sqlbuild/lint/types.py @@ -3,6 +3,7 @@ from __future__ import annotations from enum import StrEnum +from typing import Literal class LintSeverity(StrEnum): @@ -10,3 +11,6 @@ class LintSeverity(StrEnum): FAULT = "fault" WARNING = "warning" + + +type RuleFixStatus = Literal["applied", "refused", "unavailable"] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index d7b00ebeba..259876f0bf 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1172,3 +1172,14 @@ class PythonNodeSelectionE2ETestCase: expected_missing_tables: tuple[str, ...] = () orders_sql: str = 'MODEL (materialized table);\nSELECT order_id FROM __source("raw_orders")\n' existing_sql: str = "" + + +@dataclass(frozen=True) +class QueryFingerprintEditE2ETestCase: + """One authored edit and the plan reasons the token fingerprint must produce.""" + + description: str + edits: tuple[tuple[str, str, str], ...] + format_flags: tuple[str, ...] + expected_format_summary: str + expected_reasons: dict[str, str] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_replay_backfill_lifecycle.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_replay_backfill_lifecycle.py index a85d21b530..1ce0fbeb69 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_replay_backfill_lifecycle.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_replay_backfill_lifecycle.py @@ -341,5 +341,72 @@ def test_given_all_failed_f2_and_f3_when_code_returns_to_f2_then_old_requirement assert sorted(int(str(count)) for _version, count in requirement_counts) == [1, 2] +@pytest.mark.parametrize( + "test_case", + [ + ConcurrentMicrobatchBehaviorE2ETestCase( + description="formatting-only edit keeps the pending replay requirement" + ) + ], + ids=lambda case: case.description, +) +def test_given_pending_replay_when_model_is_only_reformatted_then_replay_still_completes( + test_case: ConcurrentMicrobatchBehaviorE2ETestCase, tmp_path: Path +) -> None: + project_dir, db_path = prepare_replay_microbatch_project( + tmp_path=tmp_path, + project_name="microbatch_reformatted_replay", + database_name="reformatted_replay.duckdb", + replay_policy="bounded-4h", + ) + execute_duckdb( + db_path=db_path, + sql=( + "CREATE TABLE raw_events (id INTEGER, event_time TIMESTAMP, payload VARCHAR); " + "INSERT INTO raw_events VALUES " + "(1, '2026-01-01 00:30:00', '1'), " + "(2, '2026-01-01 01:30:00', '2'), " + "(3, '2026-01-01 02:30:00', '3')" + ), + ) + initial: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + execute_duckdb(db_path=db_path, sql="UPDATE raw_events SET payload = 'bad' WHERE id = 2") + f2_sql: str = timestamp_microbatch_model_sql( + value_expression="CAST(payload AS INTEGER) + 10", + batch_concurrency=3, + replay_policy="bounded-4h", + ) + model_path: Path = project_dir / "models" / "orders.sql" + model_path.write_text(f2_sql, encoding="utf-8") + partial: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + execute_duckdb(db_path=db_path, sql="UPDATE raw_events SET payload = '2' WHERE id = 2") + reformatted: str = f2_sql.replace( + "SELECT id, event_time,", "-- Orders with a shifted value.\nselect\n id,\n event_time," + ) + model_path.write_text(reformatted, encoding="utf-8") + retried: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + assert initial.returncode == test_case.expected_exit_code, initial.stdout + initial.stderr + assert partial.returncode != test_case.expected_exit_code + assert retried.returncode == test_case.expected_exit_code, retried.stdout + retried.stderr + assert query_duckdb( + db_path=db_path, + sql="SELECT id, value FROM main.orders ORDER BY id", + ) == [(1, 11), (2, 12), (3, 13)] + assert query_duckdb( + db_path=db_path, + sql=( + "SELECT COUNT(DISTINCT replay_requirement_id) " + "FROM main._sqlbuild_microbatches WHERE record_type = 'replay_requirement'" + ), + ) == [(1,)] + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py new file mode 100644 index 0000000000..1453ee5c17 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py @@ -0,0 +1,121 @@ +"""E2E coverage for token-stream query fingerprints in change detection.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest + +from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + QueryFingerprintEditE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + prepare_inline_project, + query_duckdb, + run_sqb, +) + +_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "fingerprint_orders"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "fingerprint.duckdb"\n\n' + '[defaults]\nmaterialized = "table"\n' + ), + "sources/raw.yml": ( + "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + ), + "functions/sql/is_large_order.sql": ( + "FUNCTION (arguments (amount_cents INTEGER), returns BOOLEAN);\n\namount_cents > 100\n" + ), + "models/stg_orders.sql": ( + "MODEL (materialized table);\n\n" + "SELECT id AS order_id, amount_cents, 'web order' AS channel\n" + 'FROM __source("raw_orders")\n' + ), + "models/large_orders.sql": ( + "MODEL (materialized table);\n\n" + "SELECT order_id, is_large_order(amount_cents) AS is_large\n" + 'FROM __ref("stg_orders")\n' + ), +} + + +@pytest.mark.parametrize( + "test_case", + [ + QueryFingerprintEditE2ETestCase( + description="sqb format edits are layout only", + edits=(), + format_flags=(), + expected_format_summary="3 files checked, 2 changed", + expected_reasons={"stg_orders": "no_change", "large_orders": "no_change"}, + ), + QueryFingerprintEditE2ETestCase( + description="comment and keyword case edits are unchanged", + edits=( + ("models/stg_orders.sql", "SELECT id", "-- Staged orders.\nselect id"), + ("functions/sql/is_large_order.sql", "amount_cents > 100", "amount_cents > 100"), + ), + format_flags=("--check",), + expected_format_summary="3 files checked, 3 changed", + expected_reasons={"stg_orders": "no_change", "large_orders": "no_change"}, + ), + QueryFingerprintEditE2ETestCase( + description="whitespace inside a string literal is a query change", + edits=(("models/stg_orders.sql", "'web order'", "'web order'"),), + format_flags=("--check",), + expected_format_summary="3 files checked, 2 changed", + expected_reasons={"stg_orders": "query_changed"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_built_project_when_sql_is_edited_then_plan_follows_token_fingerprints( + test_case: QueryFingerprintEditE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="fingerprint_orders", repo_files=_PROJECT_FILES + ) + db_path: Path = project_dir / "fingerprint.duckdb" + execute_duckdb( + db_path=db_path, + sql="CREATE TABLE raw_orders (id INTEGER, amount_cents INTEGER); " + "INSERT INTO raw_orders VALUES (1, 50), (2, 500)", + ) + built: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + stored: list[tuple[object, ...]] = query_duckdb( + db_path=db_path, + sql="SELECT node_name, from_base64(definition_b64), definition_hash FROM main._sqlbuild_fingerprints " + "WHERE node_type = 'model' ORDER BY node_name", + ) + for relative_path, before, after in test_case.edits: + path: Path = project_dir / relative_path + path.write_text(path.read_text(encoding="utf-8").replace(before, after), encoding="utf-8") + formatted: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "format", *test_case.format_flags), + project_dir=project_dir, + ) + planned: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "plan", "--json"), project_dir=project_dir + ) + + assert built.returncode == 0, built.stdout + built.stderr + assert "format-unsafe" not in formatted.stdout + formatted.stderr + assert test_case.expected_format_summary in formatted.stderr + assert [str(row[0]) for row in stored] == ["large_orders", "stg_orders"] + assert [row[2] for row in stored] == [ + compute_query_hash(query_sql=bytes(row[1]).decode(), dialect="duckdb") for row in stored + ] + assert planned.returncode == 0, planned.stdout + planned.stderr + reasons: dict[str, str] = { + str(entry["name"]): str(entry["reason"]) for entry in json.loads(planned.stdout)["models"] + } + assert {name: reasons[name] for name in test_case.expected_reasons} == ( + test_case.expected_reasons + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 7107a35c96..9f3215492c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -378,3 +378,12 @@ class FunctionHeaderKeyCompileCase: description: str repo_files: dict[str, str] expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class FunctionNameCompileTestCase: + description: str + project_toml: str + query_sql: str + expected_exit_code: int + expected_diagnostics: tuple[tuple[str, str, int, int], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_diagnostic_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_diagnostic_compile_performance.py index afed55eecd..3d495a09fa 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_diagnostic_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_diagnostic_compile_performance.py @@ -10,11 +10,13 @@ from scripts.cold_compile_performance._helpers.diagnostic_measurement import ( measure_diagnostic_compile, + measure_function_name_scan, measure_position_mapping, ) from scripts.cold_compile_performance._helpers.diagnostic_project import ( write_cascade_project, write_diagnostic_project, + write_function_name_project, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import DiagnosticPerformanceCase @@ -104,5 +106,62 @@ def test_given_deep_poisoned_dag_when_recovering_then_bounds_scaling( assert timings[1] <= timings[0] * 3.0 + 1.0 +@pytest.mark.parametrize( + "test_case", + [ + DiagnosticPerformanceCase( + "unsupported_function_spellings", + width=300, + depth=3, + expected_max_wall_seconds=20.0, + ) + ], + ids=lambda case: case.description, +) +def test_given_many_unsupported_function_spellings_when_compiling_then_bounds_scaling( + test_case: DiagnosticPerformanceCase, tmp_path: Path +) -> None: + timings: list[float] = [] + for multiplier in (1, 2): + project: Path = tmp_path / f"calls_{multiplier}" + calls: int = test_case.width * multiplier + write_function_name_project(project_dir=project, model_count=test_case.depth, calls=calls) + elapsed, payload = measure_diagnostic_compile( + project_dir=project, timeout=test_case.expected_timeout_seconds + ) + codes: Counter[str] = Counter(item["code"] for item in payload["diagnostics"]) + assert codes["B101"] == calls * test_case.depth + timings.append(elapsed) + _LOGGER.info("unsupported function spellings wall=%s", timings) + assert max(timings) < test_case.expected_max_wall_seconds + assert timings[1] <= timings[0] * 3.0 + 1.0 + + +@pytest.mark.parametrize( + "test_case", + [ + DiagnosticPerformanceCase( + "function_name_scan_scales_linearly", + diagnostic_count=4000, + expected_max_wall_seconds=1.0, + ) + ], + ids=lambda case: case.description, +) +def test_given_wide_sql_with_unsupported_spellings_when_scanning_then_scales_linearly( + test_case: DiagnosticPerformanceCase, +) -> None: + measure_function_name_scan(calls=1) + single: float = median( + measure_function_name_scan(calls=test_case.diagnostic_count) for _ in range(3) + ) + doubled: float = median( + measure_function_name_scan(calls=test_case.diagnostic_count * 2) for _ in range(3) + ) + _LOGGER.info("function name scan N=%.3fs 2N=%.3fs", single, doubled) + assert doubled < test_case.expected_max_wall_seconds + assert doubled <= single * 2.8 + 0.05 + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_name_validation.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_name_validation.py new file mode 100644 index 0000000000..38351db4a1 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_name_validation.py @@ -0,0 +1,143 @@ +"""E2E tests for compile-time rejection of built-in spellings the target dialect lacks.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import cast + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + FunctionNameCompileTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_STARTS_WITH_FUNCTION: str = ( + "FUNCTION (\n" + " arguments (value STRING, prefix STRING),\n" + " returns BOOLEAN,\n" + " database warehouse,\n" + " schema analytics,\n" + ");\n\n" + "LEFT(value, LENGTH(prefix)) = prefix\n" +) + +_SELECT: str = "SELECT\n" +_FROM: str = "\nFROM (SELECT 'anvil' AS product_name) AS catalog_items\n" +_SNOWFLAKE_PROJECT: str = 'name = "products"\nadapter = "snowflake"\n' +_DUCKDB_PROJECT: str = ( + 'name = "products"\nadapter = "duckdb"\n\n[connection]\ndatabase = ":memory:"\n' +) + + +@pytest.mark.parametrize( + "test_case", + ( + FunctionNameCompileTestCase( + description="snowflake STARTS_WITH is rejected with the Snowflake spelling", + project_toml=_SNOWFLAKE_PROJECT, + query_sql=_SELECT + " STARTS_WITH(product_name, 'a') AS is_featured" + _FROM, + expected_exit_code=1, + expected_diagnostics=( + ( + "B101", + "Unknown function 'STARTS_WITH' for dialect Snowflake; did you mean STARTSWITH?", + 4, + 5, + ), + ), + ), + FunctionNameCompileTestCase( + description="snowflake accepted spellings and an alias-named UDF compile", + project_toml=_SNOWFLAKE_PROJECT, + query_sql=( + _SELECT + " STARTSWITH(product_name, 'a') AS is_featured,\n" + " SUBSTR(product_name, 1, 2) AS short_name,\n" + " TIMESTAMPDIFF(day, '2026-01-01'::date, '2026-01-02'::date) AS age_days,\n" + " TRY_TO_DECIMAL('1.25', 10, 2) AS unit_price,\n" + " __udf(\"starts_with\")(product_name, 'a') AS has_prefix" + _FROM + ), + expected_exit_code=0, + expected_diagnostics=(), + ), + FunctionNameCompileTestCase( + description="snowflake CTE column list named like an alias compiles", + project_toml=_SNOWFLAKE_PROJECT, + query_sql="WITH day_of_week (dow) AS (SELECT 1) SELECT dow FROM day_of_week\n", + expected_exit_code=0, + expected_diagnostics=(), + ), + FunctionNameCompileTestCase( + description="snowflake derived-table alias named like an alias compiles", + project_toml=_SNOWFLAKE_PROJECT, + query_sql="SELECT x FROM (VALUES (1)) day_of_week (x)\n", + expected_exit_code=0, + expected_diagnostics=(), + ), + FunctionNameCompileTestCase( + description="duckdb STARTSWITH is rejected with the DuckDB spelling", + project_toml=_DUCKDB_PROJECT, + query_sql=_SELECT + " STARTSWITH(product_name, 'a') AS is_featured" + _FROM, + expected_exit_code=1, + expected_diagnostics=( + ( + "B101", + "Unknown function 'STARTSWITH' for dialect DuckDB; did you mean STARTS_WITH?", + 4, + 5, + ), + ), + ), + FunctionNameCompileTestCase( + description="duckdb materialized CTE column lists named like aliases compile", + project_toml=_DUCKDB_PROJECT, + query_sql=( + "WITH day_of_week (d) AS MATERIALIZED (SELECT 1), " + "day_of_month (d) AS NOT MATERIALIZED (SELECT 2) " + "SELECT day_of_week.d FROM day_of_week CROSS JOIN day_of_month\n" + ), + expected_exit_code=0, + expected_diagnostics=(), + ), + FunctionNameCompileTestCase( + description="duckdb catalogue spellings compile", + project_toml=_DUCKDB_PROJECT, + query_sql=( + _SELECT + " starts_with(product_name, 'a') AS is_featured,\n" + " prefix(product_name, 'a') AS has_prefix,\n" + " substr(product_name, 1, 2) AS short_name" + _FROM + ), + expected_exit_code=0, + expected_diagnostics=(), + ), + ), + ids=lambda case: case.description, +) +def test_given_builtin_spelling_when_compiling_then_dialect_support_is_enforced( + test_case: FunctionNameCompileTestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="products", + repo_files={ + "sqlbuild_project.toml": test_case.project_toml, + "functions/sql/starts_with.sql": _STARTS_WITH_FUNCTION, + "models/products.sql": ( + f"MODEL (database warehouse, schema analytics);\n\n{test_case.query_sql}" + ), + }, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=("compile", "--json", "--no-cache") + ) + + payload: dict[str, object] = json.loads(result.stdout) + diagnostics: list[dict[str, object]] = cast(list[dict[str, object]], payload["diagnostics"]) + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + assert ( + tuple((item["code"], item["message"], item["line"], item["column"]) for item in diagnostics) + == test_case.expected_diagnostics + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/format/FORMAT_BENCHMARKS.md b/tests/e2e/src/sqlbuild/cli/commands/main/format/FORMAT_BENCHMARKS.md new file mode 100644 index 0000000000..ee83e1fe70 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/format/FORMAT_BENCHMARKS.md @@ -0,0 +1,28 @@ +# Format performance benchmarks + +`test_format_performance.py` bounds `sqb format --check` on generated projects with neutral +orders-style names. The guards run in the `Formatter performance guards` CI job with `-n auto`, +so budgets allow for the other guards running concurrently. + +## Contract fixtures + +1,000 SQL test files whose fixtures carry typed NULL projections. Both the full format and the +`--fixtures-only` path are bounded, and fixture formatting must scale linearly across 500, 1,000 +and 2,000 files. + +## Unformatted models + +1,300 unformatted models in a fixed mix: every tenth model projects 300 computed columns, three in +ten are six-CTE chains with window functions and line comments, and the rest are short joins with +block comments and `IN` lists. Each model reads the previous one through `__ref`, so formatting +also restores intrinsic calls. A separate case formats one 6,000-line model with 3,000 chained +CTEs, which bounds per-token work on very large files. + +Budgets, about 1.5 to 2 times the typical time measured with the guards running in parallel: + +- 1,300 models: format phase under 1.5 seconds, end to end under 3.5 seconds; +- one 6,000-line model: format phase under 0.75 seconds, end to end under 2.5 seconds; +- 325, 650 and 1,300 models: each doubling at most 2.75 times the format-phase time. + +The format phase is the `Formatting SQL` timing that the command prints, so process startup and +discovery do not mask a formatter regression. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/format/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/format/_test_types.py index 9aaf0535bf..4cbe2169ba 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/format/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/format/_test_types.py @@ -5,6 +5,25 @@ from dataclasses import dataclass +@dataclass(frozen=True) +class RuleFixTestCase: + description: str + sql: str + expected_status: str + expected_code: str + expected_returncode: int + expected_unchanged: bool + expected_with: bool + + +@dataclass(frozen=True) +class RuleFixPerformanceTestCase: + description: str + width: int + depth: int + expected_with: bool = False + + @dataclass(frozen=True) class FormatPerformanceGuardTestCase: """One generated format workload and its performance budgets.""" @@ -32,3 +51,59 @@ class FormatScalingGuardTestCase: expected_max_largest_elapsed_seconds: float hard_ceiling_seconds: float expected_returncode: int + + +@dataclass(frozen=True) +class ExampleProjectFormatTestCase: + """One repository example project that formatting must leave semantically unchanged.""" + + description: str + relative_path: str + expected_format_exit_code: int = 0 + + +@dataclass(frozen=True) +class ModelFormatPerformanceTestCase: + """One generated unformatted model workload and its format budgets.""" + + description: str + model_count: int + large_model_lines: int + expected_file_count: int + expected_max_format_seconds: float + expected_max_elapsed_seconds: float + hard_ceiling_seconds: float + expected_returncode: int = 1 + + +@dataclass(frozen=True) +class ModelFormatScalingTestCase: + """One doubling profile for unformatted model formatting.""" + + description: str + model_counts: tuple[int, ...] + expected_max_doubling_ratio: float + hard_ceiling_seconds: float + expected_returncode: int = 1 + + +@dataclass(frozen=True) +class TokenPreservationFormatTestCase: + """One authored model and the exact text formatting must write for it.""" + + description: str + project_toml: str + model_name: str + authored_sql: str + expected_sql: str + + +@dataclass(frozen=True) +class TokenPreservationBuildTestCase: + """One authored model whose formatted form must build with exact column names and rows.""" + + description: str + authored_sql: str + expected_sql: str + expected_columns: list[str] + expected_rows: list[tuple[object, ...]] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/format/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/format/helpers.py index 84ade5534b..876eacafb8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/format/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/format/helpers.py @@ -2,8 +2,17 @@ from __future__ import annotations +import json +import re +import subprocess +import sys +import time from pathlib import Path +from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash + +_FORMAT_ELAPSED_PATTERN: re.Pattern[str] = re.compile(r"Formatting SQL OK \(([0-9.]+)s;") + def write_format_performance_project( *, project_dir: Path, model_count: int, test_count: int @@ -48,3 +57,76 @@ def count_typed_null_candidates(*, project_dir: Path) -> int: path.read_text(encoding="utf-8").count("CAST(NULL AS VARCHAR)") for path in (project_dir / "tests").rglob("*.sql") ) + + +def run_format_check( + *, project_dir: Path, timeout_seconds: float +) -> tuple[subprocess.CompletedProcess[str], float, float]: + """Run `sqb format --check` and return the result, wall time and format-phase time.""" + + started_at: float = time.perf_counter() + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project_dir), + "--no-color", + "format", + "--check", + ], + check=False, + capture_output=True, + text=True, + timeout=timeout_seconds, + ) + elapsed_seconds: float = time.perf_counter() - started_at + elapsed_match: re.Match[str] | None = _FORMAT_ELAPSED_PATTERN.search(result.stderr) + assert elapsed_match is not None, result.stderr + return result, elapsed_seconds, float(elapsed_match.group(1)) + + +def compiled_contract(*, project_dir: Path) -> dict[str, list[tuple[object, ...]]]: + """Compile a project; return each resource's dependencies, lineage, columns and fingerprint.""" + + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project_dir), + "compile", + "--no-cache", + "--json", + ], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + resources: dict[str, list[dict[str, object]]] = json.loads(result.stdout)["resources"] + contract: dict[str, list[tuple[object, ...]]] = {} + for kind, entries in resources.items(): + rows: list[tuple[object, ...]] = [_resource_contract(entry) for entry in entries] + contract[kind] = sorted(rows, key=repr) + return contract + + +def _resource_contract(resource: dict[str, object]) -> tuple[object, ...]: + return ( + resource.get("name"), + json.dumps(resource.get("depends_on"), sort_keys=True), + json.dumps(resource.get("lineage"), sort_keys=True), + resource.get("column_count"), + compute_query_hash(query_sql=str(resource.get("query_sql", "")), dialect="duckdb"), + ) + + +def run_sqb(*, project_dir: Path, arguments: tuple[str, ...]) -> subprocess.CompletedProcess[str]: + """Run one `sqb` command against a project and capture its output.""" + + return subprocess.run( + [str(Path(sys.executable).with_name("sqb")), "--project-dir", str(project_dir), *arguments], + check=False, + capture_output=True, + text=True, + timeout=60, + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/format/test_format_example_projects.py b/tests/e2e/src/sqlbuild/cli/commands/main/format/test_format_example_projects.py new file mode 100644 index 0000000000..ec2ba31d57 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/format/test_format_example_projects.py @@ -0,0 +1,48 @@ +"""Formatting every repository example project changes layout only.""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.format._test_types import ( + ExampleProjectFormatTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.format.helpers import compiled_contract +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import REPO_ROOT, run_sqb + + +@pytest.mark.parametrize( + "test_case", + [ + ExampleProjectFormatTestCase("website waffle shop", "website/examples/waffle-shop"), + ExampleProjectFormatTestCase("website tidy shop", "website/examples/tidy-shop"), + ExampleProjectFormatTestCase("website hero shop", "website/examples/hero-shop"), + ExampleProjectFormatTestCase("e2e waffle shop", "tests/e2e/fixtures/waffle_shop"), + ], + ids=lambda case: case.description, +) +def test_given_example_project_when_formatting_then_compiled_contract_is_unchanged( + test_case: ExampleProjectFormatTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = tmp_path / "project" + shutil.copytree(REPO_ROOT / test_case.relative_path, project_dir) + before: dict[str, list[tuple[object, ...]]] = compiled_contract(project_dir=project_dir) + + formatted: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=("--no-color", "format") + ) + checked: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=("--no-color", "format", "--check") + ) + after: dict[str, list[tuple[object, ...]]] = compiled_contract(project_dir=project_dir) + + assert formatted.returncode == test_case.expected_format_exit_code, formatted.stderr + assert "format-unsafe" not in formatted.stdout + formatted.stderr + assert checked.returncode == test_case.expected_format_exit_code, checked.stderr + assert ", 0 changed)" in checked.stderr + assert after == before diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/format/test_format_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/format/test_format_performance.py index 15871ffe92..277c94d8a9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/format/test_format_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/format/test_format_performance.py @@ -10,12 +10,16 @@ import pytest +from scripts.format_performance._helpers.model_project import write_model_format_project from tests.e2e.src.sqlbuild.cli.commands.main.format._test_types import ( FormatPerformanceGuardTestCase, FormatScalingGuardTestCase, + ModelFormatPerformanceTestCase, + ModelFormatScalingTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.format.helpers import ( count_typed_null_candidates, + run_format_check, write_format_performance_project, ) @@ -158,3 +162,84 @@ def test_given_doubling_fixture_counts_when_formatting_then_elapsed_time_scales_ for samples in (elapsed_samples, format_samples): for smaller, larger in zip(samples[:-1], samples[1:], strict=True): assert larger / smaller <= test_case.expected_max_doubling_ratio + + +@pytest.mark.performance +@pytest.mark.parametrize( + "test_case", + [ + ModelFormatPerformanceTestCase( + description="1,300 unformatted models of mixed shape format within budget", + model_count=1_300, + large_model_lines=0, + expected_file_count=1_300, + expected_max_format_seconds=1.5, + expected_max_elapsed_seconds=3.5, + hard_ceiling_seconds=30.0, + ), + ModelFormatPerformanceTestCase( + description="one 6,000-line model with 3,000 CTEs formats within budget", + model_count=0, + large_model_lines=6_000, + expected_file_count=1, + expected_max_format_seconds=0.75, + expected_max_elapsed_seconds=2.5, + hard_ceiling_seconds=30.0, + ), + ], + ids=lambda case: case.description, +) +def test_given_unformatted_models_when_checking_format_then_time_stays_bounded( + tmp_path: Path, + test_case: ModelFormatPerformanceTestCase, +) -> None: + project_dir: Path = tmp_path / "model_format_performance" + write_model_format_project( + project_dir=project_dir, + model_count=test_case.model_count, + large_model_lines=test_case.large_model_lines, + ) + + result, elapsed_seconds, format_seconds = run_format_check( + project_dir=project_dir, timeout_seconds=test_case.hard_ceiling_seconds + ) + + assert result.returncode == test_case.expected_returncode, result.stdout + result.stderr + assert "format-unsafe" not in result.stdout + result.stderr + assert ( + f"{test_case.expected_file_count} files checked, {test_case.expected_file_count} changed" + in result.stderr + ) + assert format_seconds <= test_case.expected_max_format_seconds + assert elapsed_seconds <= test_case.expected_max_elapsed_seconds + + +@pytest.mark.performance +@pytest.mark.parametrize( + "test_case", + [ + ModelFormatScalingTestCase( + description="unformatted model formatting scales linearly across doublings", + model_counts=(325, 650, 1_300), + expected_max_doubling_ratio=2.75, + hard_ceiling_seconds=30.0, + ) + ], + ids=lambda case: case.description, +) +def test_given_doubling_model_counts_when_checking_format_then_format_time_scales_linearly( + tmp_path: Path, + test_case: ModelFormatScalingTestCase, +) -> None: + format_samples: list[float] = [] + for model_count in test_case.model_counts: + project_dir: Path = tmp_path / f"model_format_scaling_{model_count}" + write_model_format_project(project_dir=project_dir, model_count=model_count) + result, _, format_seconds = run_format_check( + project_dir=project_dir, timeout_seconds=test_case.hard_ceiling_seconds + ) + assert result.returncode == test_case.expected_returncode, result.stdout + result.stderr + format_samples.append(format_seconds) + + for smaller, larger in zip(format_samples[:-1], format_samples[1:], strict=True): + assert larger <= smaller * test_case.expected_max_doubling_ratio + 0.05 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/format/test_rule_fixes.py b/tests/e2e/src/sqlbuild/cli/commands/main/format/test_rule_fixes.py new file mode 100644 index 0000000000..ea215f1c40 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/format/test_rule_fixes.py @@ -0,0 +1,127 @@ +"""Subprocess coverage for compiler-verified Rule repair and bounded fixed points.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.format._test_types import ( + RuleFixPerformanceTestCase, + RuleFixTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + RuleFixTestCase( + "verification failure", + "MODEL (sql_analysis false);\nWITH unused AS (SELECT 1 AS item) SELECT 2 AS item", + "refused", + "SQBRSQL005", + 1, + True, + True, + ), + RuleFixTestCase( + "generated dependency region", + "MODEL ();\n" + 'WITH unused AS (SELECT customer_id FROM __ref("customers")) SELECT 2 AS item', + "refused", + "SQBRSQL005", + 0, + False, + True, + ), + RuleFixTestCase( + "fixed point with overlapping unused CTE deletions", + "MODEL ();\n" + "WITH first AS (SELECT 1 AS item), second AS (SELECT item FROM first), " + "third AS (SELECT item FROM second) SELECT 2 AS item", + "applied", + "SQBRSQL005", + 0, + False, + False, + ), + ], + ids=lambda case: case.description, +) +def test_given_rule_fixes_when_running_cli_then_reports_verified_or_refused_changes( + test_case: RuleFixTestCase, tmp_path: Path +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + model.write_text(test_case.sql) + (model.parent / "customers.sql").write_text("MODEL (); SELECT 1 AS customer_id") + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(tmp_path), + "format", + "--fix", + "--json", + ], + capture_output=True, + text=True, + timeout=20, + check=False, + ) + assert result.returncode == test_case.expected_returncode, result.stderr + payload: dict[str, Any] = json.loads(result.stdout) + assert any( + fix["code"] == test_case.expected_code and fix["status"] == test_case.expected_status + for fix in payload["rule_fixes"] + ) + assert (model.read_text() == test_case.sql) == test_case.expected_unchanged + assert ("WITH" in model.read_text()) == test_case.expected_with + assert any( + v["code"] == "format-fix-verification-failed" for v in payload["violations"] + ) == bool(test_case.expected_returncode) + assert "Formatting SQL" in result.stderr + + +@pytest.mark.performance +@pytest.mark.parametrize( + "test_case", + [RuleFixPerformanceTestCase("wide", 200, 8), RuleFixPerformanceTestCase("deep", 4, 100)], + ids=lambda case: case.description, +) +def test_given_wide_or_deep_unused_ctes_when_fixing_then_work_is_bounded( + test_case: RuleFixPerformanceTestCase, tmp_path: Path +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + columns: str = ", ".join(f"{index} AS item_{index}" for index in range(test_case.width)) + ctes: list[str] = [f"items_0 AS (SELECT {columns})"] + for index in range(1, test_case.depth): + ctes.append(f"items_{index} AS (SELECT * FROM items_{index - 1})") + model.write_text("MODEL (); WITH " + ", ".join(ctes) + " SELECT 1 AS order_id") + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(tmp_path), + "format", + "--fix", + "--json", + ], + capture_output=True, + text=True, + timeout=20, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert ("WITH" in model.read_text()) == test_case.expected_with + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-vv"])) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/format/test_token_preservation.py b/tests/e2e/src/sqlbuild/cli/commands/main/format/test_token_preservation.py new file mode 100644 index 0000000000..117b786ce7 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/format/test_token_preservation.py @@ -0,0 +1,104 @@ +"""Subprocess coverage for formatting that keeps case- and line-sensitive SQL tokens.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import duckdb +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.format._test_types import ( + TokenPreservationBuildTestCase, + TokenPreservationFormatTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.format.helpers import run_sqb + +_DUCKDB_PROJECT_TOML: str = ( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "prod"\n\n' + '[connections.local]\ndatabase = "warehouse.duckdb"\n\n' + '[targets.prod]\nconnection = "local"\nschema = "prod"\n' +) + + +@pytest.mark.parametrize( + "test_case", + [ + TokenPreservationFormatTestCase( + description="BigQuery keyword-named table and hash comment survive", + project_toml='name = "inventory"\nadapter = "bigquery"\n', + model_name="stock_levels", + authored_sql=( + 'MODEL (description "Stock levels");\n\n' + "select product_id, # on-hand units\n quantity from inventory.view\n" + ), + expected_sql=( + 'MODEL (description "Stock levels");\n\n' + "SELECT\n product_id, # on-hand units\n quantity\nFROM inventory.view\n" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_case_sensitive_tokens_when_formatting_then_cli_writes_them_exactly( + test_case: TokenPreservationFormatTestCase, tmp_path: Path +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text(test_case.project_toml, encoding="utf-8") + model: Path = tmp_path / "models" / f"{test_case.model_name}.sql" + model.parent.mkdir() + model.write_text(test_case.authored_sql, encoding="utf-8") + + formatted: subprocess.CompletedProcess[str] = run_sqb( + project_dir=tmp_path, arguments=("--no-color", "format") + ) + checked: subprocess.CompletedProcess[str] = run_sqb( + project_dir=tmp_path, arguments=("--no-color", "format", "--check") + ) + + assert (formatted.returncode, checked.returncode) == (0, 0), formatted.stderr + checked.stderr + assert model.read_text(encoding="utf-8") == test_case.expected_sql + + +@pytest.mark.parametrize( + "test_case", + [ + TokenPreservationBuildTestCase( + description="split literal and keyword-named aliases build unchanged", + authored_sql=( + 'MODEL (description "Order lines", materialized table);\n\n' + "select 'north'\n'east' as region, line.pos as index, line.sku as replace " + "from (select 1 as pos, 'waffle' as sku) as line\n" + ), + expected_sql=( + 'MODEL (description "Order lines", materialized table);\n\n' + "SELECT\n 'north'\n 'east' AS region,\n line.pos AS index,\n" + " line.sku AS replace\n" + "FROM (\n SELECT\n 1 AS pos,\n 'waffle' AS sku\n) AS line\n" + ), + expected_columns=["region", "index", "replace"], + expected_rows=[("northeast", 1, "waffle")], + ), + ], + ids=lambda case: case.description, +) +def test_given_case_and_line_sensitive_tokens_when_formatting_then_build_keeps_them( + test_case: TokenPreservationBuildTestCase, tmp_path: Path +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text(_DUCKDB_PROJECT_TOML, encoding="utf-8") + model: Path = tmp_path / "models" / "order_lines.sql" + model.parent.mkdir() + model.write_text(test_case.authored_sql, encoding="utf-8") + + formatted: subprocess.CompletedProcess[str] = run_sqb( + project_dir=tmp_path, arguments=("--no-color", "format") + ) + built: subprocess.CompletedProcess[str] = run_sqb( + project_dir=tmp_path, arguments=("--no-color", "build") + ) + + assert (formatted.returncode, built.returncode) == (0, 0), formatted.stderr + built.stderr + assert model.read_text(encoding="utf-8") == test_case.expected_sql + with duckdb.connect(str(tmp_path / "warehouse.duckdb"), read_only=True) as connection: + cursor: duckdb.DuckDBPyConnection = connection.execute("SELECT * FROM prod.order_lines") + assert [column[0] for column in cursor.description] == test_case.expected_columns + assert cursor.fetchall() == test_case.expected_rows diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index e3653daf72..c5d69931d5 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -7,6 +7,15 @@ from sqlbuild.compiler.planner.models import CursorOverrides +@dataclass(frozen=True) +class SemanticFixTestCase: + description: str + sql: str + expected_code: str + dialect: str = "duckdb" + expected_status: str = "applied" + + @dataclass(frozen=True) class VariedCompileFixtureTestCase: description: str @@ -229,6 +238,34 @@ class AuthoredSpellingFormatIntegrationTestCase: expected_expression: str +@dataclass(frozen=True) +class LayoutOnlyFormatIntegrationTestCase: + """One adapter whose authored keywords, aliases and terminators formatting must keep.""" + + description: str + adapter: str + expected_body: str + + +@dataclass(frozen=True) +class LineWidthWrapIntegrationTestCase: + """One project line width and the wrapped model file formatting must produce.""" + + description: str + line_width: int + authored_sql: str + expected_sql: str + + +@dataclass(frozen=True) +class DollarQuoteFormatIntegrationTestCase: + """One dollar-quoted literal whose quote state must not hide a later intrinsic call.""" + + description: str + literal: str + expected_note: str + + @dataclass(frozen=True) class BacktickDialectFormatIntegrationTestCase: """One backtick-identifier adapter whose macro call must survive formatting.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_format.py b/tests/integration/src/sqlbuild/cli/commands/main/test_format.py index 62836e041d..531b135075 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_format.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_format.py @@ -14,6 +14,7 @@ BacktickDialectFormatIntegrationTestCase, CanonicalFixtureFormatIntegrationTestCase, DescriptionFormatIntegrationTestCase, + DollarQuoteFormatIntegrationTestCase, FormatCompileIntegrationTestCase, FormatPathArgumentsIntegrationTestCase, FormatSafetyIntegrationTestCase, @@ -21,16 +22,21 @@ FormatterDeclineIntegrationTestCase, FormatWarningIntegrationTestCase, FromValuesFormatIntegrationTestCase, + LayoutOnlyFormatIntegrationTestCase, LeadingCteCommentFormatIntegrationTestCase, + LineWidthWrapIntegrationTestCase, MixedFromValuesFormatIntegrationTestCase, TypedNullFormatIntegrationTestCase, ) from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + query_duckdb_rows, + run_build, write_from_values_format_project, write_snowflake_format_test, ) _UNFORMATTED_ORDERS_SQL: str = "MODEL (materialized table);\nselect 1 as order_id\n" +_LAYOUT_ONLY_BODY: str = 'SELECT ALL\n order_id id,\n amount,\nFROM __source("raw_orders") o;\n' @pytest.mark.parametrize( @@ -52,7 +58,7 @@ "Snowflake TIMESTAMPDIFF synonym is kept", "snowflake", "TIMESTAMPDIFF(day, ordered_at, shipped_at)", - "TIMESTAMPDIFF(DAY, ordered_at, shipped_at)", + "TIMESTAMPDIFF(day, ordered_at, shipped_at)", ), AuthoredSpellingFormatIntegrationTestCase( "Snowflake TRY_TO_DECIMAL synonym is kept", @@ -60,6 +66,12 @@ "TRY_TO_DECIMAL(amount_text, 10, 2)", "TRY_TO_DECIMAL(amount_text, 10, 2)", ), + AuthoredSpellingFormatIntegrationTestCase( + "PostgreSQL comma SUBSTRING arguments are kept", + "postgres", + "substring(order_code, 1, 2)", + "SUBSTRING(order_code, 1, 2)", + ), AuthoredSpellingFormatIntegrationTestCase( "DuckDB IFNULL and != spellings are kept", "duckdb", @@ -87,7 +99,7 @@ def test_given_authored_function_spelling_when_formatting_then_only_layout_chang assert main(["--project-dir", str(tmp_path), "format", "--json"]) == 0 capsys.readouterr() formatted: str = model.read_text() - assert f" {test_case.expected_expression} AS order_flag\n" in formatted + assert f"SELECT {test_case.expected_expression} AS order_flag\n" in formatted assert main(["--project-dir", str(tmp_path), "format", "--check", "--json"]) == 0 assert model.read_text() == formatted @@ -800,15 +812,13 @@ def test_given_format_selectors_when_scoping_then_paths_and_default_exclusions_a models.mkdir() (models / "customers.sql").write_text( "MODEL (materialized table, columns (customer_id (type INTEGER),));\n" - "SELECT\n" - " 1 AS customer_id\n", + "SELECT 1 AS customer_id\n", encoding="utf-8", ) (models / "orders.sql").write_text( 'MODEL (description "Orders.", materialized table, ' "columns (order_id (type INTEGER),));\n" - "SELECT\n" - " customer_id AS order_id\n" + "SELECT customer_id AS order_id\n" 'FROM __ref("customers")\n', encoding="utf-8", ) @@ -995,7 +1005,7 @@ def test_given_file_paths_when_formatting_then_only_named_files_change( "distinct_rows AS (\n SELECT order_id, customer_id\n FROM upstream\n" " GROUP BY ALL\n)\n\nSELECT order_id, customer_id FROM distinct_rows\n" ), - expected_fragment="),\n-- Explains the next CTE: line one,\n-- line two.\ndistinct_rows AS (", + expected_fragment="),\n\n-- Explains the next CTE: line one,\n-- line two.\ndistinct_rows AS (", ), ], ids=lambda case: case.description, @@ -1075,3 +1085,226 @@ def test_given_backtick_identifier_apostrophe_when_formatting_then_macro_call_st assert (format_exit, compile_exit) == (0, 0), output.out + output.err assert test_case.expected_literal in model_path.read_text(encoding="utf-8") + + +@pytest.mark.parametrize( + "test_case", + [ + DollarQuoteFormatIntegrationTestCase( + description="untagged dollar quote with an apostrophe", + literal="$$Customer's order$$", + expected_note="Customer's order", + ), + DollarQuoteFormatIntegrationTestCase( + description="tagged dollar quote with an apostrophe", + literal="$note$it's shipped$note$", + expected_note="it's shipped", + ), + ], + ids=lambda case: case.description, +) +def test_given_dollar_quoted_apostrophe_when_formatting_then_reference_survives_and_builds( + test_case: DollarQuoteFormatIntegrationTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "prod"\n\n' + '[connections.local]\ndatabase = "warehouse.duckdb"\n\n' + '[targets.prod]\nconnection = "local"\nschema = "prod"\n', + encoding="utf-8", + ) + models: Path = tmp_path / "models" + models.mkdir() + (models / "orders.sql").write_text( + 'MODEL (description "Orders", materialized table);\n\nSELECT 1 AS order_id\n', + encoding="utf-8", + ) + notes: Path = models / "order_notes.sql" + notes.write_text( + 'MODEL (description "Order notes", materialized table);\n\n' + f'select {test_case.literal} as note, o.order_id from __ref("orders") o\n', + encoding="utf-8", + ) + + format_exit: int = main(["--project-dir", str(tmp_path), "--no-color", "format"]) + build_exit, build_output = run_build(project_dir=tmp_path, flags=(), capsys=capsys) + + assert (format_exit, build_exit) == (0, 0), build_output + formatted: str = notes.read_text(encoding="utf-8") + assert f" {test_case.literal} AS note," in formatted + assert 'FROM __ref("orders")' in formatted + assert query_duckdb_rows( + db_path=tmp_path / "warehouse.duckdb", sql="SELECT note, order_id FROM prod.order_notes" + ) == ((test_case.expected_note, 1),) + + +@pytest.mark.parametrize( + "test_case", + [ + LayoutOnlyFormatIntegrationTestCase(f"{adapter} keeps authored tokens", adapter, body) + for adapter, body in ( + ("duckdb", _LAYOUT_ONLY_BODY), + ("snowflake", _LAYOUT_ONLY_BODY), + ("postgres", _LAYOUT_ONLY_BODY.replace(" amount,\n", " amount\n")), + ("bigquery", _LAYOUT_ONLY_BODY), + ( + "databricks", + _LAYOUT_ONLY_BODY.replace(" amount,\n", " amount\n").replace("ALL", "all"), + ), + ("sqlserver", _LAYOUT_ONLY_BODY.replace(" amount,\n", " amount\n")), + ) + ], + ids=lambda case: case.description, +) +def test_given_select_all_and_implicit_aliases_when_formatting_then_only_layout_changes( + test_case: LayoutOnlyFormatIntegrationTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + f'name = "orders"\nadapter = "{test_case.adapter}"\n', encoding="utf-8" + ) + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + authored: str = " ".join(test_case.expected_body.lower().split()) + model.write_text(f'MODEL (description "Orders");\n\n{authored}\n', encoding="utf-8") + + format_exit: int = main(["--project-dir", str(tmp_path), "--no-color", "format"]) + check_exit: int = main(["--project-dir", str(tmp_path), "--no-color", "format", "--check"]) + output: CaptureResult[str] = capsys.readouterr() + + assert (format_exit, check_exit) == (0, 0), output.out + output.err + assert model.read_text(encoding="utf-8").endswith(test_case.expected_body) + + +@pytest.mark.parametrize( + "test_case", + [ + LineWidthWrapIntegrationTestCase( + description="long audit list and long reference arguments wrap at the project width", + line_width=60, + authored_sql=( + "MODEL (\n" + ' description "Order totals",\n' + " materialized table,\n" + " columns (\n" + " order_id (audits [not_null, unique, accepted_values (values [1, 2, 3])]),\n" + " ),\n" + ");\n\n" + "select o.order_id, coalesce(o.amount, o.fallback_amount, 0) + o.tax as total " + 'from __ref("orders_with_a_long_name") o where o.amount > 0 and o.order_id > 0 ' + "and o.tax >= 0 and o.fallback_amount is null\n" + ), + expected_sql=( + "MODEL (\n" + ' description "Order totals",\n' + " materialized table,\n" + " columns (\n" + " order_id (audits [\n" + " not_null,\n" + " unique,\n" + " accepted_values (values [1, 2, 3]),\n" + " ]),\n" + " ),\n" + ");\n\n" + "SELECT\n" + " o.order_id,\n" + " COALESCE(o.amount, o.fallback_amount, 0) + o.tax AS total\n" + 'FROM __ref("orders_with_a_long_name") o\n' + "WHERE\n" + " o.amount > 0\n" + " AND o.order_id > 0\n" + " AND o.tax >= 0\n" + " AND o.fallback_amount IS NULL\n" + ), + ), + LineWidthWrapIntegrationTestCase( + description="CTE, VALUES, single-item and type-parameter layout rules build", + line_width=60, + authored_sql=( + 'MODEL (description "Order totals", materialized table);\n\n' + 'with orders as (select * from __ref("orders_with_a_long_name")),\n' + "-- paid orders only\n" + "paid as (select o.order_id, cast(coalesce(o.amount, o.fallback_amount, 0) " + "+ o.tax as decimal(18, 2)) as total from orders o where o.amount > 0),\n" + "labels as (select * from (values (1, 'paid and shipped to the customer'), " + "(2, 'cancelled by the customer')) as l(order_id, label))\n" + "select p.order_id, p.total from paid p join labels l on p.order_id = l.order_id\n" + ), + expected_sql=( + 'MODEL (description "Order totals", materialized table);\n' + "\n" + "WITH orders AS (\n" + " SELECT *\n" + ' FROM __ref("orders_with_a_long_name")\n' + "),\n" + "\n" + "-- paid orders only\n" + "paid AS (\n" + " SELECT\n" + " o.order_id,\n" + " CAST(\n" + " COALESCE(o.amount, o.fallback_amount, 0)\n" + " + o.tax AS decimal(18, 2)\n" + " ) AS total\n" + " FROM orders o\n" + " WHERE o.amount > 0\n" + "),\n" + "\n" + "labels AS (\n" + " SELECT *\n" + " FROM (\n" + " VALUES\n" + " (1, 'paid and shipped to the customer'),\n" + " (2, 'cancelled by the customer')\n" + " ) AS l(order_id, label)\n" + ")\n" + "\n" + "SELECT\n" + " p.order_id,\n" + " p.total\n" + "FROM paid p\n" + "JOIN labels l\n" + " ON p.order_id = l.order_id\n" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_project_line_width_when_formatting_then_long_lines_wrap_and_project_builds( + test_case: LineWidthWrapIntegrationTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "prod"\n\n' + f"[format]\nline_width = {test_case.line_width}\n\n" + '[connections.local]\ndatabase = "warehouse.duckdb"\n\n' + '[targets.prod]\nconnection = "local"\nschema = "prod"\n', + encoding="utf-8", + ) + models: Path = tmp_path / "models" + models.mkdir() + (models / "orders_with_a_long_name.sql").write_text( + 'MODEL (description "Orders", materialized table);\n\n' + "SELECT\n 1 AS order_id,\n 5 AS amount,\n NULL AS fallback_amount,\n 1 AS tax\n", + encoding="utf-8", + ) + model: Path = models / "order_totals.sql" + model.write_text(test_case.authored_sql, encoding="utf-8") + + unformatted_check: int = main( + ["--project-dir", str(tmp_path), "--no-color", "format", "--check"] + ) + format_exit: int = main(["--project-dir", str(tmp_path), "--no-color", "format"]) + formatted_check: int = main(["--project-dir", str(tmp_path), "--no-color", "format", "--check"]) + build_exit, build_output = run_build(project_dir=tmp_path, flags=(), capsys=capsys) + + assert (unformatted_check, format_exit, formatted_check, build_exit) == (1, 0, 0, 0), ( + build_output + ) + assert model.read_text(encoding="utf-8") == test_case.expected_sql + assert query_duckdb_rows( + db_path=tmp_path / "warehouse.duckdb", sql="SELECT order_id, total FROM prod.order_totals" + ) == ((1, 6),) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py b/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py new file mode 100644 index 0000000000..43f3c23360 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py @@ -0,0 +1,243 @@ +"""Real compiler and CLI coverage for transactional Rule fixes.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import duckdb +import pytest + +from sqlbuild.cli.commands.main.entrypoint.entry import main +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + FormatCompileIntegrationTestCase, + SemanticFixTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [FormatCompileIntegrationTestCase("bare union", "UNION DISTINCT")], + ids=lambda case: case.description, +) +def test_given_bare_union_when_fixing_then_preserves_output_and_is_idempotent( + test_case: FormatCompileIntegrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + original = 'MODEL (description "Orders");\nSELECT 1 AS order_id UNION SELECT 2 AS order_id\n' + model.write_text(original) + assert main(["--project-dir", str(tmp_path), "format", "--fix", "--check", "--json"]) == 1 + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + assert model.read_text() == original + assert any( + fix["code"] == "SQBRSQL008" and fix["status"] == "applied" for fix in payload["rule_fixes"] + ) + assert main(["--project-dir", str(tmp_path), "format", "--fix", "--json"]) == 0 + capsys.readouterr() + assert test_case.expected_literal in model.read_text() + assert main(["--project-dir", str(tmp_path), "format", "--fix", "--check", "--json"]) == 0 + + +@pytest.mark.parametrize( + "test_case", + [FormatCompileIntegrationTestCase("null comparison", "= NULL")], + ids=lambda case: case.description, +) +def test_given_null_comparison_when_fixing_then_refuses_semantics_change( + test_case: FormatCompileIntegrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + model.write_text('MODEL (description "Orders");\nSELECT 1 AS order_id WHERE 1 = NULL\n') + main(["--project-dir", str(tmp_path), "format", "--fix", "--json"]) + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + assert any( + fix["code"] == "SQBRSQL001" and fix["status"] == "refused" for fix in payload["rule_fixes"] + ) + assert test_case.expected_literal in model.read_text() + + +@pytest.mark.parametrize( + "test_case", + [FormatCompileIntegrationTestCase("unanalysed model", "format-fix-verification-failed")], + ids=lambda case: case.description, +) +def test_given_unanalysed_model_when_fixing_then_verification_preserves_original_file( + test_case: FormatCompileIntegrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + models: Path = tmp_path / "models" + models.mkdir() + (models / "customers.sql").write_text( + 'MODEL (description "Customers");\nSELECT 1 AS customer_id\n' + ) + model: Path = models / "orders.sql" + original = ( + 'MODEL (description "Orders", sql_analysis false);\n' + "WITH unused AS (SELECT 2 AS customer_id) SELECT 1 AS order_id\n" + ) + model.write_text(original) + assert main(["--project-dir", str(tmp_path), "format", "--fix", "--json"]) == 1 + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + assert model.read_text() == original + assert any(v["code"] == test_case.expected_literal for v in payload["violations"]) + + +@pytest.mark.parametrize( + "test_case", + [ + SemanticFixTestCase( + "comma product", + "SELECT a.order_id, b.customer_id FROM " + "(SELECT 1 AS order_id) AS a, (SELECT 2 AS customer_id) AS b", + "SQBRSQL002", + ), + SemanticFixTestCase( + "unused CTE", + "WITH unused AS (SELECT 2 AS customer_id) SELECT 1 AS order_id", + "SQBRSQL005", + ), + SemanticFixTestCase( + "constant row count", + "SELECT COUNT(1) AS order_count FROM (SELECT 1 AS order_id UNION ALL SELECT 2) AS o", + "SQBRSQL017", + ), + SemanticFixTestCase( + "implicit inner join", + "SELECT a.order_id, b.customer_id FROM (SELECT 1 AS order_id) AS a " + "JOIN (SELECT 1 AS order_id, 2 AS customer_id) AS b ON a.order_id = b.order_id", + "SQBRSQL025", + ), + SemanticFixTestCase( + "redundant distinct", + "SELECT DISTINCT order_id FROM (SELECT 1 AS order_id) AS orders GROUP BY order_id", + "SQBRSQL006", + ), + ], + ids=lambda case: case.description, +) +def test_given_safe_native_fix_when_formatting_then_compiler_and_query_results_are_preserved( + test_case: SemanticFixTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + model.write_text('MODEL (description "Orders", schema analytics);\n' + test_case.sql) + assert main(["--project-dir", str(tmp_path), "format", "--fix", "--json"]) == 0 + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + assert any( + fix["code"] == test_case.expected_code and fix["status"] == "applied" + for fix in payload["rule_fixes"] + ) + with duckdb.connect() as connection: + expected: list[tuple[Any, ...]] = connection.execute(test_case.sql).fetchall() + actual: list[tuple[Any, ...]] = connection.execute( + model.read_text().split(";", 1)[1] + ).fetchall() + assert actual == expected + + +@pytest.mark.parametrize( + "test_case", + [ + SemanticFixTestCase( + "Snowflake cartesian JOIN", + "SELECT a.order_id, b.customer_id FROM " + "(SELECT 1 AS order_id) AS a JOIN (SELECT 2 AS customer_id) AS b", + "SQBRSQL003", + dialect="snowflake", + ), + SemanticFixTestCase( + "DuckDB invalid JOIN", + "SELECT a.order_id, b.customer_id FROM " + "(SELECT 1 AS order_id) AS a JOIN (SELECT 2 AS customer_id) AS b", + "SQBRSQL003", + expected_status="refused", + ), + SemanticFixTestCase( + "parentheses precedence", + "SELECT DISTINCT (1 + 2) * 3 AS order_id", + "SQBRSQL013", + expected_status="refused", + ), + SemanticFixTestCase( + "unused table alias", + "SELECT order_id FROM (SELECT 1 AS order_id) AS o", + "SQBRSQL023", + expected_status="refused", + ), + SemanticFixTestCase( + "boolean CASE", + "SELECT CASE WHEN 1 = 2 THEN TRUE ELSE FALSE END AS active", + "SQBRSQL030", + expected_status="refused", + ), + ], + ids=lambda case: case.description, +) +def test_given_dialect_sensitive_rewrite_when_fixing_then_requires_semantic_proof( + test_case: SemanticFixTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + f'name = "orders"\nadapter = "{test_case.dialect}"\n' + ) + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + model.write_text( + 'MODEL (description "Orders", database example, schema analytics);\n' + test_case.sql + ) + assert main(["--project-dir", str(tmp_path), "format", "--fix", "--json"]) == 0 + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + assert any( + fix["code"] == test_case.expected_code and fix["status"] == test_case.expected_status + for fix in payload["rule_fixes"] + ) + + +@pytest.mark.parametrize( + "test_case", + [FormatCompileIntegrationTestCase("one unverifiable file", "format-fix-verification-failed")], + ids=lambda case: case.description, +) +def test_given_one_unverifiable_file_when_fixing_then_only_that_file_keeps_its_original( + test_case: FormatCompileIntegrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + models: Path = tmp_path / "models" + models.mkdir() + verified: Path = models / "orders.sql" + verified.write_text( + 'MODEL (description "Orders");\nSELECT 1 AS order_id UNION SELECT 2 AS order_id\n' + ) + unverifiable: Path = models / "customers.sql" + unverifiable_sql: str = ( + 'MODEL (description "Customers", sql_analysis false);\n' + "WITH unused AS (SELECT 2 AS customer_id) SELECT 1 AS customer_id\n" + ) + unverifiable.write_text(unverifiable_sql) + + exit_code: int = main(["--project-dir", str(tmp_path), "format", "--fix", "--json"]) + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + + assert exit_code == 1 + assert "UNION DISTINCT" in verified.read_text() + assert unverifiable.read_text() == unverifiable_sql + assert { + (Path(fix["file"]).name, fix["code"], fix["status"]) for fix in payload["rule_fixes"] + } >= { + ("orders.sql", "SQBRSQL008", "applied"), + ("customers.sql", "SQBRSQL005", "refused"), + } + faults: set[tuple[str, str]] = { + (Path(violation["file"]).name, violation["code"]) for violation in payload["violations"] + } + assert ("customers.sql", test_case.expected_literal) in faults + assert ("orders.sql", test_case.expected_literal) not in faults + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-vv"])) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 580feb9d4a..0ff2aae644 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -333,8 +333,8 @@ def write_previous_function_fingerprints( target_schema=function.destination.schema, target_name=function.destination.name, run_id="previous_run", - definition_hash=compute_query_hash(fingerprint_sql), - schema_fingerprint=compute_query_hash(""), + definition_hash=compute_query_hash(query_sql=fingerprint_sql, dialect=None), + schema_fingerprint=compute_query_hash(query_sql="", dialect=None), definition=fingerprint_sql, ts=datetime.now(tz=UTC), ), diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py index d96afdaa5e..f85ca4e113 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py @@ -530,9 +530,9 @@ def test_given_declaration_change_when_compiling_then_updates_dependent_identity ) changed_model: CompiledModel = compile_first_model(project_dir=tmp_path) - query_hash_changed: bool = compute_query_hash(initial_model.query_sql) != compute_query_hash( - changed_model.query_sql - ) + query_hash_changed: bool = compute_query_hash( + query_sql=initial_model.query_sql, dialect=None + ) != compute_query_hash(query_sql=changed_model.query_sql, dialect=None) metadata_changed: bool = build_model_version_identity_metadata_json( model=initial_model ) != build_model_version_identity_metadata_json(model=changed_model) diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/_test_types.py index 58979b3228..03bb59ce74 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/_test_types.py @@ -84,6 +84,7 @@ class NormalizeQuerySqlTestCase: @dataclass(frozen=True) class ComputeQueryHashTestCase: description: str + dialect: str | None query_sql: str expected_hash: str @@ -91,6 +92,15 @@ class ComputeQueryHashTestCase: @dataclass(frozen=True) class ComputeQueryHashStabilityTestCase: description: str + dialect: str | None query_a: str query_b: str expected_same_hash: bool + + +@dataclass(frozen=True) +class QueryFingerprintFailureTestCase: + description: str + dialect: str + query_sql: str + expected_message: str diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py index db57ef4d73..41cb79882a 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py @@ -2,12 +2,14 @@ import pytest +from sqlbuild.compiler.fingerprints.exceptions import QueryFingerprintError from sqlbuild.compiler.fingerprints.main._normalize_query_sql import normalize_query_sql from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash from tests.unit.src.sqlbuild.compiler.fingerprints.main._test_types import ( ComputeQueryHashStabilityTestCase, ComputeQueryHashTestCase, NormalizeQuerySqlTestCase, + QueryFingerprintFailureTestCase, ) @@ -54,17 +56,56 @@ def test_given_raw_query_when_normalizing_then_returns_expected_text( "test_case", [ ComputeQueryHashTestCase( - description="produces sha256 hex digest for simple query", - query_sql="SELECT id, name FROM orders", - expected_hash="c87cdf54b562650a0cc936e8c4243e32513acc37fe42527c19c1a3421c9c105c", + "generic token stream", + None, + "SELECT id, name FROM orders", + "264bcb1f81ce5ba25394a052016edc265aefba85c0698f616d3037f9d872d2bd", + ), + ComputeQueryHashTestCase( + "DuckDB dollar quote and cast", + "duckdb", + "SELECT o.id, $$Customer's note$$ AS note, amount::DECIMAL(10, 2) AS amount " + "FROM orders AS o", + "07786643d9593498ac2b2394e6801cb578b23ba0f9a0e70dbcdaf0d3a3cbf9db", + ), + ComputeQueryHashTestCase( + "PostgreSQL tagged dollar quote", + "postgres", + "SELECT o.id, $tag$it's$tag$ AS note, amount::NUMERIC AS amount FROM orders AS o", + "8db865c6f239b990c917754df176bd3c9b2c0c322b9af1437298ec98b67241f9", + ), + ComputeQueryHashTestCase( + "Snowflake path access and QUALIFY", + "snowflake", + "SELECT o.id, payload:customer.name::STRING AS customer FROM orders AS o " + "QUALIFY ROW_NUMBER() OVER (PARTITION BY o.id ORDER BY o.id) = 1", + "c75f7b5931a18e1a074ffa7ab113c256fc838285af75a3f9feb62d61cc76d94d", + ), + ComputeQueryHashTestCase( + "BigQuery raw string and backtick path", + "bigquery", + "SELECT o.id, r'raw\\d' AS pattern FROM `project.dataset.orders` AS o", + "ce893f2fd9ab11ba453fc3c02e77b6690df6c3d1d8bc141fc19cf3891d9a61b4", + ), + ComputeQueryHashTestCase( + "Databricks optimizer hint", + "databricks", + "SELECT /*+ BROADCAST(o) */ o.id FROM `catalog`.`orders` AS o", + "7cd96f42bcb764136a7f423181302ba13cead02bb6cfb7382502a3a2aa4107c3", + ), + ComputeQueryHashTestCase( + "SQL Server bracket identifiers", + "sqlserver", + "SELECT TOP 10 [o].[id], N'order' AS kind FROM [dbo].[orders] AS o", + "f4fac3edb7682bbeddf4f73cc199493d45f1419bc8e2cd1d972fead2d67e6b71", ), ], ids=lambda case: case.description, ) -def test_given_query_sql_when_computing_hash_then_returns_expected_digest( +def test_given_dialect_query_when_computing_hash_then_golden_fingerprint_is_stable( test_case: ComputeQueryHashTestCase, ) -> None: - result: str = compute_query_hash(test_case.query_sql) + result: str = compute_query_hash(query_sql=test_case.query_sql, dialect=test_case.dialect) assert result == test_case.expected_hash @@ -73,22 +114,95 @@ def test_given_query_sql_when_computing_hash_then_returns_expected_digest( "test_case", [ ComputeQueryHashStabilityTestCase( - description="whitespace-only differences produce the same hash", - query_a="SELECT id FROM orders", - query_b="SELECT id FROM orders", - expected_same_hash=True, + "whitespace and line breaks are ignored", + "duckdb", + "SELECT id FROM orders", + "select\n id\nfrom orders", + True, + ), + ComputeQueryHashStabilityTestCase( + "comments are ignored", + "snowflake", + "SELECT id -- order key\nFROM orders /* all */", + "SELECT id FROM orders", + True, + ), + ComputeQueryHashStabilityTestCase( + "keyword and built-in function case is ignored", + "duckdb", + "SELECT COUNT(id) FROM orders WHERE id > 0 LIMIT 1", + "select count(id) from orders where id > 0 limit 1", + True, ), ComputeQueryHashStabilityTestCase( - description="casing differences produce different hashes", - query_a="SELECT id FROM orders", - query_b="select id from orders", - expected_same_hash=False, + "unquoted identifier case is significant", + "duckdb", + "SELECT Id FROM Orders", + "SELECT id FROM orders", + False, ), ComputeQueryHashStabilityTestCase( - description="different queries produce different hashes", - query_a="SELECT id FROM orders", - query_b="SELECT id FROM customers", - expected_same_hash=False, + "alias case is significant", + "duckdb", + "SELECT 1 AS Foo", + "SELECT 1 AS foo", + False, + ), + ComputeQueryHashStabilityTestCase( + "Snowflake path key case is significant", + "snowflake", + "SELECT payload:customerId FROM orders", + "SELECT payload:customerid FROM orders", + False, + ), + ComputeQueryHashStabilityTestCase( + "BigQuery qualified table case is significant", + "bigquery", + "SELECT * FROM ds.Orders", + "SELECT * FROM ds.orders", + False, + ), + ComputeQueryHashStabilityTestCase( + "user-defined function case is significant", + "duckdb", + "SELECT MyFunc(id) FROM orders", + "SELECT myfunc(id) FROM orders", + False, + ), + ComputeQueryHashStabilityTestCase( + "string literal whitespace is significant", + "duckdb", + "SELECT 'a b' AS code", + "SELECT 'a b' AS code", + False, + ), + ComputeQueryHashStabilityTestCase( + "quoted identifier case is significant", + "snowflake", + 'SELECT "Id" FROM orders', + 'SELECT "ID" FROM orders', + False, + ), + ComputeQueryHashStabilityTestCase( + "number spelling is significant", + "duckdb", + "SELECT 1.0 AS amount", + "SELECT 1.00 AS amount", + False, + ), + ComputeQueryHashStabilityTestCase( + "optimizer hints are significant", + "databricks", + "SELECT /*+ BROADCAST(o) */ o.id FROM orders o", + "SELECT /*+ REPARTITION(3) */ o.id FROM orders o", + False, + ), + ComputeQueryHashStabilityTestCase( + "different queries produce different hashes", + None, + "SELECT id FROM orders", + "SELECT id FROM customers", + False, ), ], ids=lambda case: case.description, @@ -96,7 +210,26 @@ def test_given_query_sql_when_computing_hash_then_returns_expected_digest( def test_given_two_queries_when_computing_hashes_then_stability_matches_expected( test_case: ComputeQueryHashStabilityTestCase, ) -> None: - hash_a: str = compute_query_hash(test_case.query_a) - hash_b: str = compute_query_hash(test_case.query_b) + hash_a: str = compute_query_hash(query_sql=test_case.query_a, dialect=test_case.dialect) + hash_b: str = compute_query_hash(query_sql=test_case.query_b, dialect=test_case.dialect) assert (hash_a == hash_b) == test_case.expected_same_hash + + +@pytest.mark.parametrize( + "test_case", + [ + QueryFingerprintFailureTestCase( + "unterminated string literal", + "duckdb", + "SELECT 'open FROM orders", + "Unterminated string", + ) + ], + ids=lambda case: case.description, +) +def test_given_untokenizable_sql_when_computing_hash_then_raises_fingerprint_error( + test_case: QueryFingerprintFailureTestCase, +) -> None: + with pytest.raises(QueryFingerprintError, match=test_case.expected_message): + _ = compute_query_hash(query_sql=test_case.query_sql, dialect=test_case.dialect) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py index c9af14752d..0352763fc1 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py @@ -105,7 +105,9 @@ def build_snapshot_for_metadata_test_case( target_schema=None, target_name="orders", run_id="run_001", - definition_hash=compute_query_hash("SELECT 1 AS order_id"), + definition_hash=compute_query_hash( + query_sql="SELECT 1 AS order_id", dialect=None + ), schema_fingerprint="schema_a", definition="SELECT 1 AS order_id", metadata_json=test_case.previous_metadata_json, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index 6399426ed4..2b18caf1fa 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -54,7 +54,7 @@ ) _QUERY_SQL: str = "SELECT id, name FROM orders" -_MATCHING_HASH: str = compute_query_hash(_QUERY_SQL) +_MATCHING_HASH: str = compute_query_hash(query_sql=_QUERY_SQL, dialect=None) _DIFFERENT_HASH: str = "completely_different_hash" @@ -697,7 +697,8 @@ def test_given_direct_project_function_hash_change_when_detecting_changes_then_m target_name="orders", run_id="run_001", definition_hash=compute_query_hash( - "SELECT is_large_order(amount) AS large_order FROM orders" + query_sql="SELECT is_large_order(amount) AS large_order FROM orders", + dialect=None, ), schema_fingerprint="schema_a", definition="SELECT is_large_order(amount) AS large_order FROM orders", @@ -765,7 +766,7 @@ def test_given_renamed_model_when_detecting_changes_then_reports_rename_and_own_ handover: Fingerprint = replace( snapshot.fingerprints.models["orders"], definition=test_case.previous_definition, - definition_hash=compute_query_hash(test_case.previous_definition), + definition_hash=compute_query_hash(query_sql=test_case.previous_definition, dialect=None), ) result: ChangeDetectionResult = detect_model_changes( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py index 0325b7da41..1e9d71ef49 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py @@ -90,7 +90,9 @@ def test_given_renamed_references_when_matching_recorded_query_then_returns_mapp target_schema="main", target_name="order_summary", run_id="run", - definition_hash=compute_query_hash(test_case.recorded_query_sql), + definition_hash=compute_query_hash( + query_sql=test_case.recorded_query_sql, dialect="duckdb" + ), schema_fingerprint="", definition=test_case.recorded_query_sql, ts=_BUILT_AT, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index de3896c33e..9bbd8e628f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -1309,7 +1309,7 @@ def build_fingerprint(*, query_sql: str) -> Fingerprint: target_schema="main", target_name="is_completed_order", run_id="run-1", - definition_hash=compute_query_hash(query_sql), + definition_hash=compute_query_hash(query_sql=query_sql, dialect=None), schema_fingerprint="", definition=query_sql, ts=datetime(2026, 1, 1, tzinfo=UTC), diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/_test_types.py index dc428fe649..9d47f1817a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/_test_types.py @@ -15,3 +15,12 @@ class MergeRecomputedModelChangesTestCase: expected_change_kind: ChangeKind expected_backfill_action: BackfillAction expected_version_hash: str | None + + +@dataclass(frozen=True) +class DefinitionHashFailureTestCase: + """One untokenizable definition whose planning error must name its resource.""" + + description: str + expected_message: str + expected_code: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_definition_hashes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_definition_hashes.py new file mode 100644 index 0000000000..5264136d02 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_definition_hashes.py @@ -0,0 +1,55 @@ +"""Unit tests for planner definition fingerprints that name failing resources.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.planner._helpers.identity.hashing import ( + function_definition_hash, + model_definition_hash, +) +from sqlbuild.compiler.planner.exceptions import PlannerInputError +from tests.unit.src.sqlbuild.compiler.planner._helpers.identity._test_types import ( + DefinitionHashFailureTestCase, +) + +_UNTOKENIZABLE_SQL: str = "SELECT 'open FROM orders" + + +@pytest.mark.parametrize( + "test_case", + [DefinitionHashFailureTestCase("model", "Model 'orders' SQL cannot be fingerprinted", "S025")], + ids=lambda case: case.description, +) +def test_given_untokenizable_model_when_hashing_then_planning_error_names_model( + test_case: DefinitionHashFailureTestCase, +) -> None: + with pytest.raises(PlannerInputError, match=test_case.expected_message) as raised: + _ = model_definition_hash( + model_name="orders", query_sql=_UNTOKENIZABLE_SQL, dialect="duckdb" + ) + + assert raised.value.code == test_case.expected_code + + +@pytest.mark.parametrize( + "test_case", + [ + DefinitionHashFailureTestCase( + "SQL function", "Function 'is_open' cannot be fingerprinted", "S025" + ) + ], + ids=lambda case: case.description, +) +def test_given_untokenizable_sql_function_when_hashing_then_planning_error_names_function( + test_case: DefinitionHashFailureTestCase, +) -> None: + with pytest.raises(PlannerInputError, match=test_case.expected_message) as raised: + _ = function_definition_hash( + function_name="is_open", + fingerprint_sql=_UNTOKENIZABLE_SQL, + language="sql", + dialect="duckdb", + ) + + assert raised.value.code == test_case.expected_code diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py index d5ab78845a..b343ca4b77 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py @@ -78,3 +78,11 @@ class OldNameRowsTestCase: grants_copied: int | None reason: str | None expected_rows: tuple[str, ...] + + +@dataclass(frozen=True) +class UnfingerprintableCandidateTestCase: + description: str + query_sql: str + expected_message: str + expected_code: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py new file mode 100644 index 0000000000..b9d4a799f8 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py @@ -0,0 +1,122 @@ +"""Unit coverage for fingerprinting column-migration candidates.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from pathlib import Path +from unittest.mock import Mock + +import pytest + +from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo +from sqlbuild.compiler.compile.models import ( + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, + CompileModelConfig, +) +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.fingerprints.models import Fingerprint +from sqlbuild.compiler.planner._helpers.migrations.columns import plan_column_migrations +from sqlbuild.compiler.planner.exceptions import PlannerInputError +from sqlbuild.compiler.planner.models import ( + DeferralInputs, + PlannerOverrides, + PlannerRuntime, + PlannerScope, + WarehouseFingerprints, + WarehouseSnapshot, +) +from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations._test_types import ( + UnfingerprintableCandidateTestCase, +) +from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations.helpers import BASE_CONFIG + + +@pytest.mark.parametrize( + "test_case", + [ + UnfingerprintableCandidateTestCase( + description="unterminated string literal", + query_sql="SELECT 'open AS order_status FROM raw_orders", + expected_message="Model 'orders' SQL cannot be fingerprinted", + expected_code="S025", + ) + ], + ids=lambda case: case.description, +) +def test_given_untokenizable_tracked_model_when_planning_column_migrations_then_error_names_model( + test_case: UnfingerprintableCandidateTestCase, +) -> None: + model: CompiledModel = CompiledModel( + key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="orders"), + deps=(), + name="orders", + relative_path=Path("models/orders.sql"), + query_sql=test_case.query_sql, + config=CompileModelConfig(values=dict(BASE_CONFIG)), + destination=CompiledRelationLocation( + database=None, schema="analytics", name="orders", qualified_name=None + ), + ) + adapter: Mock = Mock() + adapter.sql_analysis_dialect.return_value = "duckdb" + runtime: PlannerRuntime = PlannerRuntime( + project=CompiledProject( + run_id="run-1", + effective_target_name="test", + effective_connection={}, + effective_vars={}, + models=(model,), + ), + adapter=adapter, + connection=object(), + ) + scope: PlannerScope = PlannerScope( + upstream_deps={model.key: ()}, + downstream_deps={model.key: ()}, + all_keys={"orders": model.key}, + models_by_name={"orders": model}, + selected_keys=frozenset({model.key}), + execution_order=(model.key,), + ) + snapshot: WarehouseSnapshot = WarehouseSnapshot( + existing_relations={ + "orders": RelationInfo( + database=None, schema="analytics", name="orders", relation_type="BASE TABLE" + ) + }, + existing_columns={"orders": (ColumnInfo(name="order_status", type="VARCHAR"),)}, + fingerprints=WarehouseFingerprints( + models={ + "orders": Fingerprint( + node_type="model", + node_name="orders", + target_database=None, + target_schema="analytics", + target_name="orders", + run_id="run-0", + definition_hash="previous", + schema_fingerprint="", + definition="SELECT 'open' AS order_status FROM raw_orders", + ts=datetime(2026, 1, 15, tzinfo=UTC), + ) + } + ), + column_dialect="duckdb", + ) + + with pytest.raises(PlannerInputError, match=test_case.expected_message) as raised: + _ = plan_column_migrations( + runtime=runtime, + scope=scope, + snapshot=snapshot, + full_refresh_model_names=frozenset(), + physical_relations={}, + overrides=PlannerOverrides(), + deferral=DeferralInputs(), + source_columns={}, + ) + + assert raised.value.code == test_case.expected_code diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py index ab5d2d167b..1cd0ea0875 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py @@ -70,10 +70,12 @@ def test_given_function_fingerprint_when_detecting_change_then_returns_reason( target_schema=test_case.target_schema, ) reason: PlanReason = detect_function_change( + function=function, fingerprint_sql=build_compiled_function_fingerprint_sql(function), fingerprint=test_case.existing_function_fingerprints.get(function.name), query_change_tracking=True, full_refresh=False, + dialect=None, ) assert reason == test_case.expected_reason diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py index f2eb067281..ae828ac29e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py @@ -34,7 +34,7 @@ def model_definition_hash(project: CompiledProject, name: str) -> str: models_by_name: dict[str, CompiledModel] = {model.name: model for model in project.models} model: CompiledModel = models_by_name[name] - return compute_query_hash(model.query_sql) + return compute_query_hash(query_sql=model.query_sql, dialect=project.sql_analysis_dialect) def build_sqlbuild_model_selector_project() -> CompiledProject: @@ -156,7 +156,7 @@ def create_protected_model_state(*, adapter: DuckDbAdapter, connection: Any) -> target_schema="main", target_name="order_history", run_id="previous-run", - definition_hash=compute_query_hash(query_sql), + definition_hash=compute_query_hash(query_sql=query_sql, dialect=None), schema_fingerprint="", definition=query_sql, metadata_json=build_version_identity_metadata_json( diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py index 65fecd6cca..e4905e0d1d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py @@ -68,7 +68,7 @@ def build_table_function_graph_identities(*, base_query: str) -> DirectModelVers selected_keys=frozenset(upstream), execution_order=(base_key, function_key, consumer_key), ) - return build_direct_model_version_identities(functions=(function,), scope=scope) + return build_direct_model_version_identities(functions=(function,), scope=scope, dialect=None) def _build_model( diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py index e8133cde74..f578be893f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py @@ -54,8 +54,10 @@ def test_given_table_function_return_contract_change_when_hashing_then_identity_ return_columns=(FunctionReturnColumn(name="order_id", type=test_case.changed_type),), ) - original_hash: str = build_function_local_hashes(functions=(function,))[function.name] - changed_hash: str = build_function_local_hashes(functions=(changed_function,))[ + original_hash: str = build_function_local_hashes(functions=(function,), dialect=None)[ + function.name + ] + changed_hash: str = build_function_local_hashes(functions=(changed_function,), dialect=None)[ changed_function.name ] diff --git a/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py b/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py index 3d70633f8c..584ab15bb9 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py @@ -184,3 +184,23 @@ class UnicodePreparationTestCase: description: str prefix: str expected_lint_calls: tuple[str, ...] + + +@dataclass(frozen=True) +class DollarQuotePreparationTestCase: + """Dollar-quoted text that the Python and native lint scanners must treat as opaque.""" + + description: str + dialect: str + literal: str + expected_sites: tuple[str, ...] + + +@dataclass(frozen=True) +class HeaderAuditWrapTestCase: + """A MODEL header and its expected audit-list layout at one line width.""" + + description: str + line_width: int + header: str + expected_header: str diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_header_wrapping.py b/tests/unit/src/sqlbuild/lint/_helpers/test_header_wrapping.py new file mode 100644 index 0000000000..00131498a8 --- /dev/null +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_header_wrapping.py @@ -0,0 +1,62 @@ +"""Unit tests for wrapping long MODEL header audit lists.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.lint._helpers.native import format_native_headers +from sqlbuild.lint.models import LintConfig +from tests.unit.src.sqlbuild.lint._helpers._test_types import HeaderAuditWrapTestCase + +_BODY: str = "\nSELECT 1 AS status\n" + + +@pytest.mark.parametrize( + "test_case", + [ + HeaderAuditWrapTestCase( + "audits then their arguments go one per line", + 50, + 'MODEL (\n audits [expression_is_true (name "positive", expression "total > 0")],\n);\n', + "MODEL (\n audits [\n expression_is_true (\n" + ' name "positive",\n expression "total > 0",\n ),\n ],\n);\n', + ), + HeaderAuditWrapTestCase( + "quoted values are never split", + 30, + 'MODEL (\n audits [expression_is_true (expression "a, b, c, d > 0")],\n);\n', + "MODEL (\n audits [\n expression_is_true (\n" + ' expression "a, b, c, d > 0",\n ),\n ],\n);\n', + ), + HeaderAuditWrapTestCase( + "a line with a comment is left alone", + 30, + "MODEL (\n audits [not_null, unique, accepted_values (values [1, 2])], -- keep\n);\n", + "MODEL (\n audits [not_null, unique, accepted_values (values [1, 2])], -- keep\n);\n", + ), + HeaderAuditWrapTestCase( + "lists outside audits are left alone", + 30, + "MODEL (\n tags [orders, customers, products, inventory],\n);\n", + "MODEL (\n tags [orders, customers, products, inventory],\n);\n", + ), + ], + ids=lambda case: case.description, +) +def test_given_long_header_audits_when_formatting_then_lists_wrap_idempotently( + test_case: HeaderAuditWrapTestCase, +) -> None: + config: LintConfig = LintConfig(line_width=test_case.line_width) + + formatted, faults = format_native_headers( + contents=test_case.header + _BODY, file_path=Path("models/orders.sql"), config=config + ) + again, _ = format_native_headers( + contents=formatted, file_path=Path("models/orders.sql"), config=config + ) + + assert faults == () + assert formatted == test_case.expected_header + _BODY + assert again == formatted diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_native_sql.py b/tests/unit/src/sqlbuild/lint/_helpers/test_native_sql.py index 5e3827ed1d..431f98746f 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_native_sql.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_native_sql.py @@ -326,6 +326,52 @@ def test_given_multi_branch_boolean_case_when_linting_then_partial_fix_is_withhe assert violation.fix is None +@pytest.mark.parametrize( + "test_case", + [ + NativeSqlFixTestCase( + description="equal-length expansion wholly inside an unused CTE", + sql="WITH unused AS (SELECT 42 AS order_id) SELECT 1 AS order_id", + expected_code="SQBRSQL005", + ) + ], + ids=lambda case: case.description, +) +def test_given_generated_interior_with_authored_boundaries_when_linting_then_fix_is_refused( + test_case: NativeSqlFixTestCase, tmp_path: Path +) -> None: + target: Path = tmp_path / "orders.sql" + start: int = test_case.sql.index("42") + body: LintBody = LintBody( + file_path=target, + body_start=0, + body_end=len(test_case.sql), + lint_text=test_case.sql, + passes=( + ( + ExpansionSpan( + source_start=start, + source_end=start + 2, + output_start=start, + output_end=start + 2, + ), + ), + ), + ) + result: dict[Path, tuple[LintViolation, ...]] = native_sql.run_native_sql_lint( + bodies=(body,), + contents_by_path={target: test_case.sql.replace("42", "__")}, + config=LintConfig(dialect="duckdb", enabled_native_rules=(test_case.expected_code,)), + ) + violation: LintViolation = result[target][0] + assert violation.code == test_case.expected_code + assert violation.fix is None + assert ( + violation.fix_unavailable_reason + == "fix overlaps generated SQL or a non-contiguous authored region" + ) + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py b/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py index 28eef1a300..e4e7cb90fe 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py @@ -13,6 +13,7 @@ from sqlbuild.lint.models import InterpolationSite from tests.unit.src.sqlbuild.lint._helpers._test_types import ( DialectNeutralizeInterpolationTestCase, + DollarQuotePreparationTestCase, NeutralizeInterpolationTestCase, RestoreFailureTestCase, RestoreInterpolationTestCase, @@ -395,3 +396,63 @@ def test_given_mangled_sentinels_when_restoring_then_error_is_raised( with pytest.raises(InterpolationRestorationError) as error: _ = restore_interpolation(fixed=test_case.fixed_neutralized, sites=sites) assert test_case.expected_message_fragment in str(error.value) + + +@pytest.mark.parametrize( + "test_case", + [ + DollarQuotePreparationTestCase( + "untagged apostrophe", + "duckdb", + "$$Customer's order$$", + ("@label($$Customer's order$$)", '__ref("orders")'), + ), + DollarQuotePreparationTestCase( + "tagged apostrophe", + "postgres", + "$note$it's$note$", + ("@label($note$it's$note$)", '__ref("orders")'), + ), + DollarQuotePreparationTestCase( + "hidden intrinsic", + "snowflake", + "$$__ref('items')$$", + ("@label($$__ref('items')$$)", '__ref("orders")'), + ), + DollarQuotePreparationTestCase( + "hidden macro paren", + "duckdb", + "$$@m( ')$$", + ("@label($$@m( ')$$)", '__ref("orders")'), + ), + ], + ids=lambda case: case.description, +) +def test_given_dollar_quoted_text_when_neutralizing_then_python_and_native_sites_agree( + test_case: DollarQuotePreparationTestCase, +) -> None: + sql: str = ( + f"SELECT {test_case.literal} AS note, @label({test_case.literal}) AS label " + 'FROM __ref("orders") o' + ) + lint_sql, lint_sites = neutralize_interpolation(body=sql, dialect=test_case.dialect) + format_sql, format_sites = neutralize_interpolation( + body=sql, dialect=test_case.dialect, for_formatting=True + ) + native: tuple[str, list[tuple[str, int, int, int, int, str]], list[str]] | None = ( + _native.prepare_lint_sql( + { + "expanded": sql, + "before_expansion": sql, + "prior_sites": [], + "dialect": test_case.dialect, + } + ) + ) + + assert tuple(site.original_text for site in lint_sites) == test_case.expected_sites + assert tuple(site.original_text for site in format_sites) == test_case.expected_sites + assert native is not None + assert native[0] == lint_sql + assert tuple(InterpolationSite(*site) for site in native[1]) == lint_sites + assert restore_interpolation(fixed=format_sql, sites=format_sites) == sql diff --git a/tests/unit/src/sqlbuild/lint/_test_types.py b/tests/unit/src/sqlbuild/lint/_test_types.py index 1743e69a0d..478a55d411 100644 --- a/tests/unit/src/sqlbuild/lint/_test_types.py +++ b/tests/unit/src/sqlbuild/lint/_test_types.py @@ -95,3 +95,24 @@ class ExpandedTypedConstantTestCase: model_path: str authored_sql: str expected_sql: str + + +@dataclass(frozen=True) +class KeywordNamedIdentifierFormatTestCase: + """A keyword-named identifier whose authored case formatting and fingerprints must keep.""" + + description: str + dialect: str + sql: str + name: str + expected_name_occurrences: int + + +@dataclass(frozen=True) +class ReservedKeywordFoldTestCase: + """A reserved keyword whose letter case the fingerprint must ignore.""" + + description: str + dialect: str + keyword: str + expected_same_fingerprint: bool diff --git a/tests/unit/src/sqlbuild/lint/helpers.py b/tests/unit/src/sqlbuild/lint/helpers.py index a47ffe97a6..7816e885b5 100644 --- a/tests/unit/src/sqlbuild/lint/helpers.py +++ b/tests/unit/src/sqlbuild/lint/helpers.py @@ -1,5 +1,7 @@ """Project writers shared by lint behavior tests.""" +import itertools +import re from pathlib import Path _PROJECT_TOML: str = 'name = "demo"\nadapter = "duckdb"\n' @@ -40,3 +42,89 @@ def write_fixture_format_project( encoding="utf-8", ) return test_file + + +_RESERVED_KEYWORDS_DIR: Path = ( + Path(__file__).resolve().parents[5] + / "crates" + / "sqlbuild-rules-native" + / "src" + / "sql_tokens" + / "reserved_keywords" +) +_DIALECTS: tuple[str, ...] = ("duckdb", "postgres", "bigquery", "snowflake", "tsql", "databricks") +_KEYWORD_NAMED_IDENTIFIERS: tuple[str, ...] = ( + "Left", + "Filter", + "Rows", + "Index", + "View", + "Replace", + "Any", + "Some", + "Only", + "Semi", + "Anti", + "Pivot", + "Tablesample", + "Range", + "Row", + "Nulls", + "Ignore", + "Within", + "Recursive", + "Exclude", + "Preceding", + "Key", + "Type", + "Date", + "First", + "Value", + "Order", +) +_NAME_SHAPES: tuple[tuple[str, str], ...] = ( + ("alias", "select order_id as {name} from orders"), + ("qualified column", "select o.{name} from orders o"), + ("qualified table", "select * from inventory.{name}"), +) +_NON_RESERVED_SHAPES: tuple[tuple[str, str], ...] = ( + ("projection", "select {name} from orders"), + ("table", "select * from {name}"), + ("operand", "select order_id, {name} ^ 2 from orders"), +) +_DUCKDB_SHAPES: tuple[tuple[str, str], ...] = (("struct key", "select {{{name}: 1}} as s"),) + + +def reserved_keywords(dialect: str) -> frozenset[str]: + """Return the lower-case reserved keywords of one dialect from the native data files.""" + + text: str = (_RESERVED_KEYWORDS_DIR / f"{dialect}.txt").read_text(encoding="utf-8") + return frozenset(re.findall(r"^[a-z_]+$", text, flags=re.MULTILINE)) + + +def keyword_name_positions() -> tuple[tuple[str, str, str, str], ...]: + """Return (dialect, name, position, template) for names that must keep their case.""" + + positions: list[tuple[str, str, str, str]] = [] + for dialect, name in itertools.product(_DIALECTS, _KEYWORD_NAMED_IDENTIFIERS): + unreserved_shapes: tuple[tuple[str, str], ...] = _NON_RESERVED_SHAPES + { + "duckdb": _DUCKDB_SHAPES + }.get(dialect, ()) + shapes: tuple[tuple[str, str], ...] = ( + _NAME_SHAPES + + { + False: unreserved_shapes, + True: (), + }[name.lower() in reserved_keywords(dialect)] + ) + positions.extend((dialect, name, shape, template) for shape, template in shapes) + return tuple(positions) + + +def reserved_keyword_pairs() -> tuple[tuple[str, str], ...]: + """Return every (dialect, reserved keyword) pair from the native data files.""" + + pairs: list[tuple[str, str]] = [] + for dialect in _DIALECTS: + pairs.extend((dialect, keyword) for keyword in sorted(reserved_keywords(dialect))) + return tuple(pairs) diff --git a/tests/unit/src/sqlbuild/lint/test_cli_format.py b/tests/unit/src/sqlbuild/lint/test_cli_format.py index 782d03465f..b81b9a9119 100644 --- a/tests/unit/src/sqlbuild/lint/test_cli_format.py +++ b/tests/unit/src/sqlbuild/lint/test_cli_format.py @@ -219,7 +219,7 @@ def test_given_all_null_empty_fixture_when_formatting_then_uses_empty_fixture_ma assert "Formatted files:" in capsys.readouterr().out formatted: str = test_file.read_text(encoding="utf-8") assert test_case.expected_fragment in formatted - assert "__ref__orders AS (\n SELECT\n *\n FROM __empty_fixture()\n)" in formatted + assert "__ref__orders AS (\n SELECT *\n FROM __empty_fixture()\n)" in formatted assert "CAST(NULL" not in formatted.split("__expected__orders", maxsplit=1)[0] diff --git a/tests/unit/src/sqlbuild/lint/test_native_format_case.py b/tests/unit/src/sqlbuild/lint/test_native_format_case.py new file mode 100644 index 0000000000..fa0eee29a6 --- /dev/null +++ b/tests/unit/src/sqlbuild/lint/test_native_format_case.py @@ -0,0 +1,72 @@ +"""Native formatting and fingerprints fold only reserved keywords and built-in calls.""" + +from __future__ import annotations + +import json + +import pytest + +import sqlbuild._native as native_module +from tests.unit.src.sqlbuild.lint._test_types import ( + KeywordNamedIdentifierFormatTestCase, + ReservedKeywordFoldTestCase, +) +from tests.unit.src.sqlbuild.lint.helpers import keyword_name_positions, reserved_keyword_pairs + + +@pytest.mark.parametrize( + "test_case", + [ + KeywordNamedIdentifierFormatTestCase( + description=f"{dialect} {name} as {position}", + dialect=dialect, + sql=template.format(name=name), + name=name, + expected_name_occurrences=1, + ) + for dialect, name, position, template in keyword_name_positions() + ], + ids=lambda case: case.description, +) +def test_given_keyword_named_identifier_when_formatting_then_case_is_kept_and_fingerprinted( + test_case: KeywordNamedIdentifierFormatTestCase, +) -> None: + response: dict[str, object] = json.loads( + native_module.format_sql_json( + json.dumps({"version": 1, "sql": test_case.sql, "dialect": test_case.dialect}) + ) + ) + printed: str = str(response.get("sql") or "") + swapped: str = test_case.sql.replace(test_case.name, test_case.name.swapcase()) + + assert response["formatted"] is False or ( + printed.count(test_case.name) == test_case.expected_name_occurrences + ), test_case.description + assert native_module.query_fingerprint( + test_case.sql, test_case.dialect + ) != native_module.query_fingerprint(swapped, test_case.dialect), test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + ReservedKeywordFoldTestCase( + description=f"{dialect} {keyword}", + dialect=dialect, + keyword=keyword, + expected_same_fingerprint=True, + ) + for dialect, keyword in reserved_keyword_pairs() + ], + ids=lambda case: case.description, +) +def test_given_reserved_keyword_when_fingerprinting_then_its_case_is_folded( + test_case: ReservedKeywordFoldTestCase, +) -> None: + lower: str = f"select order_id {test_case.keyword.lower()} x from orders" + upper: str = f"select order_id {test_case.keyword.upper()} x from orders" + + assert ( + native_module.query_fingerprint(lower, test_case.dialect) + == native_module.query_fingerprint(upper, test_case.dialect) + ) == test_case.expected_same_fingerprint, test_case.description diff --git a/tests/unit/src/sqlbuild/lint/test_run_lint.py b/tests/unit/src/sqlbuild/lint/test_run_lint.py index 40838593e9..b58d65fe4e 100644 --- a/tests/unit/src/sqlbuild/lint/test_run_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_run_lint.py @@ -237,7 +237,7 @@ def test_given_comment_free_body_when_formatting_twice_then_output_is_canonical_ assert len(first.formatted_files) == test_case.expected_value assert second.formatted_files == () - assert "SELECT\n a,\n b\nFROM items\nWHERE\n a = 1\n" in target.read_text(encoding="utf-8") + assert "SELECT\n a,\n b\nFROM items\nWHERE a = 1\n" in target.read_text(encoding="utf-8") @pytest.mark.parametrize( diff --git a/website/src/content/docs/docs/cli/format.mdx b/website/src/content/docs/docs/cli/format.mdx index f60577016f..4d1f5d9534 100644 --- a/website/src/content/docs/docs/cli/format.mdx +++ b/website/src/content/docs/docs/cli/format.mdx @@ -15,6 +15,7 @@ sqb format [flags] |------|-------------| | `--check` | Fail when formatting changes are needed without writing | | `--diff` | Print the proposed diff without writing | +| `--fix` | Also apply compiler-verified, meaning-preserving Rule fixes before layout formatting | | `--json` | Print structured results | | `--select`, `-s` | Format selected models | | `--select-file` | Read selectors from a file | @@ -31,23 +32,101 @@ configured [Rules](/docs/concepts/rules), which never mutate source. ## Line width and descriptions -Long model and scenario descriptions are reflowed deterministically. Ordinary authored line breaks -are normalized as spaces, while blank lines preserve paragraph boundaries. Configure the maximum -physical line width in `sqlbuild_project.toml` (the default is `100`): +Configure the maximum physical line width in `sqlbuild_project.toml` (the default is `100`): ```toml [format] line_width = 100 ``` +Long model and scenario descriptions are reflowed deterministically. Ordinary authored line breaks +are normalized as spaces, while blank lines preserve paragraph boundaries. + +Clause layout follows a few fixed rules: + +- A `SELECT` list with exactly one item stays on the `SELECT` line when the item fits (`SELECT *`, + `SELECT DISTINCT order_id`). Two or more items go one per line. +- `WHERE`, `QUALIFY` and `HAVING` with one top-level condition stay on the clause line when it + fits (`WHERE amount > 0`). Several `AND`/`OR` conditions go one per line. +- Each CTE closes with `),` followed by one blank line, and one blank line separates the last CTE + from the statement that uses it. Comments above a CTE stay directly above its name. +- A `VALUES` list that does not fit on its line puts each row on its own line under `VALUES`, and a + row is never split, even when it is longer than the width. +- When an operand of `+`, `-`, `*`, `/`, `%`, `||`, `AND` or `OR` spans several lines, the + operator starts the next line. +- A broken `DECODE` puts its first argument on its own line, then one search/result pair per line, + then any default. + +SQL lines longer than the width, counting any trailing alias and comma, are broken at the outermost +structure first: before top-level `AND` and `OR` (never inside `BETWEEN ... AND`), then before +arithmetic operators, then by putting the arguments of the widest outermost function call, `IN` +list, or array one per line, or the branches of a `CASE` on their own lines. Inside `OVER (...)`, +`PARTITION BY` and `ORDER BY` go on separate lines. Type parameters such as `NUMBER(38, 2)` or +`VARCHAR(MAX)` are never split; the enclosing call breaks instead. Header `audits [...]` lists put +each audit, and then each audit argument, on its own line. String literals, comments, and lines that +carry a comment are never broken, so a line can stay longer than the width when nothing on it can be +split. `sqb format --check` fails while a long line can still be shortened. + ## What formatting preserves +Formatting changes layout only: it prints the authored SQL tokens in their original order and +changes only the whitespace between them and the letter case of keywords and built-in function +names. A word is upper-cased only when the parse confirms it is a keyword at that position, such as +`ORDER BY`, `CASE WHEN`, `NULLS LAST`, `IS NULL` or `JOIN`, or when it names a built-in function at +a call site, such as `coalesce(`. Identifiers keep their authored case even when they are spelled +like keywords, such as `t.view`, `AS index` or a table named `view`, and so do aliases, qualified +names, JSON path keys and user-defined function names. + +Query fingerprints ignore the case of reserved keywords and built-in function names only, so +formatting lowercase SQL may change a model's fingerprint once, when a non-reserved keyword such as +`by` or `nulls` is upper-cased. Formatting already formatted SQL never changes its fingerprint. + +Comments in every dialect form are kept, including `//` in Snowflake and `#` in BigQuery, and +adjacent string literals split across lines stay on separate lines. It does not add `AS` to aliases +or remove statement terminators or trailing commas. Every result is checked against the authored +token stream and comment positions, and a body whose tokens would change is reported as +`format-unsafe` instead of being written. + Formatting preserves authored cast types, postfix casts, quoted literals, variant paths, typed lambda parameters, and supported SQL function spellings while applying canonical layout. SQLBuild calls keep their authored spelling, including zero-argument cursor and empty-fixture intrinsics. CTE-producing macros remain authored calls rather than expanded project SQL, with each call on its own CTE-list line and leading comments attached to the node they describe. +## Rule fixes + +`sqb format` on its own never changes what SQL means. `sqb format --fix` also applies Rule edits +that are known to preserve meaning, then formats the layout. `--fix --check` reports the proposed +changes without writing files, and `--fix --diff` prints their diff. + +Edits are planned in memory and repeated until no more apply; overlapping edits wait for the next +pass. Every pass recompiles the project and compares output columns (including types and +nullability), dependencies, and column lineage of each edited model and everything downstream of it +against the original compilation. When a pass cannot be proved, the edits of each file are checked +on their own: files whose edits cannot be proved keep their original contents and report +`format-fix-verification-failed`, while the other files are still fixed. A cycle or the 128-pass +limit fails without writing any edit. + +| Rule | Decision | Reason | +| --- | --- | --- | +| SQBRSQL001 NULL comparison | Refused | Replacing `= NULL` with `IS NULL` changes three-valued logic. | +| SQBRSQL002 implicit cartesian join | Applied | A comma-separated product becomes an explicit `CROSS JOIN`; mixed join precedence is excluded. | +| SQBRSQL003 conditionless join | Applied in Snowflake | Snowflake defines an unqualified `JOIN` without `ON` or `USING` as a cartesian product. Other dialects are refused because the original syntax may be invalid. | +| SQBRSQL005 unused CTE | Applied | Unreferenced `SELECT` CTEs contribute no rows. Mutation statements and commented CTEs are excluded. | +| SQBRSQL006 redundant DISTINCT | Applied | The projected and grouping expressions are proved to coincide. | +| SQBRSQL008 bare UNION | Applied | `UNION DISTINCT` states the existing duplicate elimination, in dialects that support it. | +| SQBRSQL013 parenthesized DISTINCT | Refused | Removing parentheses can change operator precedence. | +| SQBRSQL017 constant row count | Applied | `COUNT(1)` and `COUNT(*)` both count every row. | +| SQBRSQL023 unused table alias | Refused | Not yet reviewed as meaning-preserving. | +| SQBRSQL025 implicit inner join | Applied | A bare `JOIN ... ON` is an inner join. | +| SQBRSQL030 boolean CASE simplification | Refused | The `COALESCE` rewrite is not proved portable to every supported dialect. | + +Edits that touch macro or intrinsic expansions are refused, and so are edits outside model SQL +bodies, because recompiling cannot prove their output contract. Findings without a proposed edit +are reported as unavailable. Human output lists each applied, refused, and unavailable fix with its +reason; JSON output includes a `rule_fixes` array with `file`, `line`, `code`, `status`, and +`reason`. Refused and unavailable fixes still need a manual change. + ## Unsafe bodies `sqb format` reports a file-specific `format-unsafe` fault whenever a SQL body cannot be safely diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 1a97666da6..bbed80c769 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -388,7 +388,7 @@ default_audit_run_scope = "final" | Field | Default | Description | |-------|---------|-------------| | `sql_analysis` | `true` | Enable SQL syntax analysis, column binding, type inference, output inference, and semantic validation at compile time | -| `query_change_tracking` | `true` | Track query fingerprints for change detection | +| `query_change_tracking` | `true` | Track query fingerprints for change detection. A fingerprint covers the SQL tokens of the compiled query and ignores whitespace, comments, and the letter case of reserved keywords and built-in function names, so formatting-only edits do not rebuild. Every other word, including non-reserved keywords, identifiers, aliases, JSON path keys and user-defined function names, counts exactly, as do string literals, quoted identifiers, numbers, and optimizer hints. | | `column_contract_mode` | `implicit` | Controls whether column declarations on models without a `contract` declaration activate static shape/nullability validation. `implicit` preserves that validation; `explicit` treats columns as metadata and audit attachment unless the model declares `contract enforced`. Model-level `contract enforced` and `contract none` override this setting. Explicit type enforcement remains independent. See [Contracts](/docs/concepts/models/contracts). | | `concurrency` | `1` | Maximum parallel model execution (currently serial only) | | `microbatch_concurrency` | `false` | Explicitly permit models with `batch_concurrency > 1`; concurrent batches use immutable coordination facts | diff --git a/website/src/content/docs/docs/concepts/semantic-compilation.mdx b/website/src/content/docs/docs/concepts/semantic-compilation.mdx index b5ed1f3068..ae2b4359bd 100644 --- a/website/src/content/docs/docs/concepts/semantic-compilation.mdx +++ b/website/src/content/docs/docs/concepts/semantic-compilation.mdx @@ -56,6 +56,19 @@ Snowflake's function catalogue is partial and deliberately does not reject unkno Function signatures and overload coverage remain partial; compilation is not a replacement for executing SQL unit tests. +Built-in function spellings are checked for Snowflake and DuckDB. The SQL parser treats alias +spellings such as `STARTS_WITH` and `STARTSWITH` as one built-in, so a spelling the warehouse +rejects would otherwise compile. SQLBuild compares each call's authored name, case-insensitively, +with the dialect's accepted spellings and reports a blocking B101 error naming the dialect's +spelling, for example `Unknown function 'STARTS_WITH' for dialect Snowflake; did you mean +STARTSWITH?`. Snowflake's accepted spellings come from its function reference and include +documented synonyms such as `SUBSTR` and `SUBSTRING`, `DATEDIFF` and `TIMESTAMPDIFF`, and +`TRY_TO_DECIMAL` and `TRY_TO_NUMBER`. DuckDB's come from its function catalogue. A name is +rejected only when the parser maps it to the same built-in as an accepted spelling, so project +UDFs called through `__udf`, schema-qualified and method calls, quoted names, and unknown +functions are not affected. There is no per-model suppression: use the suggested spelling, or +schema-qualify a call to an external UDF that shares an alias name. Other dialects are not checked. + Derived-table column aliases, UNPIVOT, and Snowflake GROUPING SETS are checked. DuckDB and Snowflake reject excess relation aliases; fewer aliases than output columns remain valid. Missing UNPIVOT input columns are errors. @@ -158,7 +171,7 @@ the notice never reduces them to a count alone. | B002 | E201 | Unknown column | | B003 | E221 | Ambiguous column | | B004–B005 | E222–E223 | Reference/scope errors | -| B101 | E202 | Unknown function | +| B101 | E202 or SQLBuild | Unknown function, or a built-in spelling the dialect does not accept | | B102 | E203 or project declaration | Invalid function arity | | B210–B219 | E210–E219 | Proven native type/shape errors on enabled dialects | | B211–B218 | W210–W216 | Implicit conversions; blocking, fixed by an explicit `CAST` or comparison |