feat: support quoted mixed-case columns and read SQL with adapter lexical rules (#1270)

- Every compile-time scanner (declaration references, macro calls, SQL macros, `@@` interpolation) reads strings and comments with the adapter's lexical rules, including E-strings; `@py` arguments use Python string rules.
- Quoted mixed-case column names such as `"listingId"` work in MODEL headers, fixtures, contracts, Snowflake planning and builds, and runtime contract, type-enforcement, incremental and diff checks match them as the compiler does.
- DuckDB existence checks follow `SET search_path`, case-insensitively.
- A build reuses the planner's Snowflake schema listings for up to a minute.
- **Behaviour change:** a `rule_ignores` path without glob characters that does not exist now fails compilation, as `rule_exceptions` already did.
- `sqb query` shows transient progress only for slow steps; the Snowflake adapter turns off connector platform detection unless the user set `SNOWFLAKE_DISABLE_PLATFORM_DETECTION`; the import hook skips installed packages.
- Pins `polyglot-sql-chio==0.13.10`, makes three DuckShop core models deterministic, and moves repository links to the sqlstack-io organization.

Author: kvlonge

Cargo.lock

@@ -72,14 +72,14 @@ junction = { version = "2.1.0", default-features = false }
 mimalloc = { version = "0.1", default-features = false, features = ["local_dynamic_tls"] }
 proc-macro2 = { version = "1", features = ["span-locations"] }
 pyo3 = { version = "0.26", features = ["abi3-py312"] }
-polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "8767d688c5b504ad609645e27ab77a260d91c5a1", default-features = false, features = [
+polyglot-sql = { git = "https://github.com/sqlstack-io/polyglot.git", rev = "8767d688c5b504ad609645e27ab77a260d91c5a1", default-features = false, features = [
     "ast-tools",
     "builder",
     "all-dialects",
     "semantic",
     "stacker",
 ] }
-polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "8767d688c5b504ad609645e27ab77a260d91c5a1", default-features = false, features = [
+polyglot-sql-function-catalogs = { git = "https://github.com/sqlstack-io/polyglot.git", rev = "8767d688c5b504ad609645e27ab77a260d91c5a1", default-features = false, features = [
     "dialect-duckdb",
 ] }
 blake3 = { version = "1", features = ["rayon"] }

Cargo.toml

@@ -116,7 +116,8 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_check_outcomes.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_attach.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_unbuilt_dependencies.py \
-	tests/e2e/src/sqlbuild/cli/commands/main/build/test_validation_failures.py
+	tests/e2e/src/sqlbuild/cli/commands/main/build/test_validation_failures.py \
+	tests/e2e/src/sqlbuild/cli/commands/main/build/test_quoted_mixed_case_columns.py
 
 E2E_DUCKDB_BUILD_INCREMENTAL_PATHS := \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay \
@@ -156,6 +157,7 @@ E2E_DUCKDB_CLI_DATA_PATHS := \
 	tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_fixture_columns.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_lambda.py \
+	tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_escaped_quotes.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_snapshot_staleness.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_unmocked_inputs.py
 

Makefile

@@ -1,5 +1,5 @@
 <p align="center">
-  <img src="https://raw.githubusercontent.com/chio-labs/sqlbuild/main/.github/sqlbuild-logo-clean.png" alt="SQLBuild" width="100%">
+  <img src="https://raw.githubusercontent.com/sqlstack-io/sqlbuild/main/.github/sqlbuild-logo-clean.png" alt="SQLBuild" width="100%">
 </p>
 
 <p align="center">
@@ -21,7 +21,7 @@ no external state database, no manifest files and no paid tier.
 Its compiler and planner run in Rust. Warehouse execution and extensions use Python.
 
 <p align="center">
-  <img src="https://raw.githubusercontent.com/chio-labs/sqlbuild/main/.github/demos/rename.gif" alt="Moving and renaming an incremental model with sqb mv: sqb plan migrates the existing table instead of rebuilding it" width="100%">
+  <img src="https://raw.githubusercontent.com/sqlstack-io/sqlbuild/main/.github/demos/rename.gif" alt="Moving and renaming an incremental model with sqb mv: sqb plan migrates the existing table instead of rebuilding it" width="100%">
 </p>
 
 Move an incremental model to a new folder and name with `sqb mv`, and `sqb plan` migrates its table
@@ -100,7 +100,7 @@ running on local DuckDB.
 `SELECT` fails at compile time, before anything reaches the warehouse.
 
 <p align="center">
-  <img src="https://raw.githubusercontent.com/chio-labs/sqlbuild/main/.github/demos/contract.gif" alt="sqb compile fails because units_sold is not in the enforced contract and the declared waffles_sold column is missing" width="100%">
+  <img src="https://raw.githubusercontent.com/sqlstack-io/sqlbuild/main/.github/demos/contract.gif" alt="sqb compile fails because units_sold is not in the enforced contract and the declared waffles_sold column is missing" width="100%">
 </p>
 
 Both sides are reported: the new column isn't in the contract, and the declared one is gone.
@@ -133,7 +133,7 @@ A mart that reads `__source("raw__payments")` directly now fails, and so does th
 has no mock for it:
 
 <p align="center">
-  <img src="https://raw.githubusercontent.com/chio-labs/sqlbuild/main/.github/demos/rules.gif" alt="sqb compile reports the custom rule XSQBRARCH001 on the line that reads a raw source, and a unit test with no mock for that source" width="100%">
+  <img src="https://raw.githubusercontent.com/sqlstack-io/sqlbuild/main/.github/demos/rules.gif" alt="sqb compile reports the custom rule XSQBRARCH001 on the line that reads a raw source, and a unit test with no mock for that source" width="100%">
 </p>
 
 See [rules](https://sqlbuild.com/docs/concepts/rules/).
@@ -143,7 +143,7 @@ See [rules](https://sqlbuild.com/docs/concepts/rules/).
 `sqb scope --as-path` previews moving a model before you move it.
 
 <p align="center">
-  <img src="https://raw.githubusercontent.com/chio-labs/sqlbuild/main/.github/demos/scope.gif" alt="sqb scope previews moving daily_revenue: the enum and macro it uses would be lost, so both usages are invalidated" width="100%">
+  <img src="https://raw.githubusercontent.com/sqlstack-io/sqlbuild/main/.github/demos/scope.gif" alt="sqb scope previews moving daily_revenue: the enum and macro it uses would be lost, so both usages are invalidated" width="100%">
 </p>
 
 Look at `Lost` and `Invalidated usages`: the model uses an enum and a macro that are private to

README.md

@@ -119,7 +119,7 @@ project and does not change it:
 - raw `edge__*` sources in `sources/edge_cases.yml`, loaded by `python/loaders/edge_cases.py` from
   fixed rows that `data_scale` does not change, so core data and core expectations stay the same.
 
-It has 38 models (7 staging, 22 intermediate, 9 marts), 61 unit tests and 4 scenarios, in four
+It has 38 models (7 staging, 22 intermediate, 9 marts), 62 unit tests and 4 scenarios, in four
 groups:
 
 | Group | Covers |
@@ -145,6 +145,14 @@ Benchmarks and core measurements leave the area out:
   with `--exclude edge_cases`;
 - the construct census counts the core project unless given `--area edge` or `--area all`.
 
+Changes to the core project's compiled SQL, after which benchmark results are not comparable with
+earlier ones:
+
+- `int_cart_funnel`, `int_delivery_slot_utilisation` and `rpt_delivery_exceptions_monthly` give the
+  same results on every run: added SKUs are sorted, the weekday average utilisation is an exact
+  decimal sum over a count, the busiest slot of equally busy ones is the earliest, and the top
+  exception code of equally frequent ones is the first in alphabetical order.
+
 ## Status
 
 Last verified on Snowflake with SQLBuild 0.175.0 at `data_scale = "1"`:
@@ -156,7 +164,7 @@ Last verified on Snowflake with SQLBuild 0.175.0 at `data_scale = "1"`:
 | `sqb test` | 266 of 266 pass |
 | `sqb scenario test` | 8 of 8 pass |
 | `sqb scenario capture` | Captures all 8; snapshots are in `project/tests/_scenario_snapshots/` |
-| `sqb scenario test --local --strict` | In the core copy, 0 of 8 pass: 5 fail in translation, 3 report a stale snapshot |
+| `sqb scenario test --local --strict --exclude edge_cases` | 0 of 8 pass: 5 fail in translation, 3 report a stale snapshot |
 
 Local replay translates each scenario's Snowflake SQL to DuckDB on a best-effort basis. These five
 failures are the baseline that better translation should remove:
@@ -181,12 +189,12 @@ the same rows, apart from the snapshots, which record build times, and three mod
 vary from one run to the next (`int_cart_funnel`, `int_delivery_slot_utilisation` and
 `rpt_delivery_exceptions_monthly`).
 
-The edge-case area, verified on Snowflake with SQLBuild 0.184.0 on the `edge` target:
+The edge-case area, verified on Snowflake with SQLBuild built after 0.185.0 on the `edge` target:
 
 | Step | Result |
 |------|--------|
-| `sqb build --select tag:edge_case --exclude stg_edge_partner_listings+` | Passes; the disabled model is skipped |
-| `sqb test --select tag:edge_case` | 61 of 61 pass |
+| `sqb build --select tag:edge_case` | Passes; the disabled model is skipped |
+| `sqb test --select tag:edge_case` | 62 of 62 pass |
 | `sqb scenario test --select edge_cases` | 4 of 4 pass |
 | Two-load replay of the delivery change log | The merged status equals the full rebuild after each load |
 
@@ -213,15 +221,10 @@ noted here:
 - Compilation accepts SQL that Snowflake rejects: `CONNECT_BY_ISLEAF`, which Snowflake does not
   have, and a CTE whose body is a bare `VALUES` list. `int_category_paths` finds leaf categories
   with a join instead, and test case tables use `SELECT ... FROM VALUES ... AS v(...)`.
-- A source column whose name is quoted mixed case, such as `"listingId"`, cannot be read once the
-  table exists: planning folds the quoted reference to lower case and fails with `B002`. Fixtures
-  cannot name such columns either (`B302` in unit tests, `S510` in scenarios), and a `MODEL`
-  header cannot declare a quoted column name. `stg_edge_partner_listings` has a reviewed
-  `SQBRTEST202` ignore, `rpt_edge_partner_listing_export` a reviewed `SQBRTEST201` ignore, and the
-  edge build excludes `stg_edge_partner_listings+`.
-- Local scenario replay stops at compile (`P001`, unclosed quoted string) on a model with a
-  backslash-escaped quote in a string literal, `'O\'Brien'` in `stg_edge_ticket_notes`, so local
-  replay runs in the core copy.
+- On Snowflake, a model that names a column in quoted mixed case, such as `"unitPrice"`, loses the
+  static types of the columns it reads from a CTE through a table alias (`FROM listings AS l`), so
+  `rpt_edge_partner_listing_export` declares only the name and audits of `"listingId"`, without
+  types.
 - `SQBRSQL018` can only take proof from single-column keys. Append-only feeds redeliver rows, so
   their id is unique only together with `_loaded_at` (the `unique_per_delivery` audit), and the
   staging models that keep the latest delivery carry reviewed exceptions.

benchmarks/duckshop/README.md

@@ -133,16 +133,13 @@ on Snowflake. Report any failure rather than editing the expectations to match.
 
 ```bash
 sqb scenario capture --exclude edge_cases
-python ../scripts/core_project.py /tmp/duckshop-core
-(cd /tmp/duckshop-core && sqb scenario test --local --strict)
+sqb scenario test --local --strict --exclude edge_cases
 ```
 
 Capture materialises each scenario's fixtures in Snowflake, downloads them as JSONL into
 `tests/_scenario_snapshots/`, and drops what it created. The local run then replays all 8
 scenarios on your machine from those files, with no warehouse connection; `--strict` fails on a
-missing or stale snapshot instead of skipping it. It runs in a copy of the project without the
-edge-case area, because local replay of the full project stops at compile on an edge-case model
-(see Known limitations in the README). Delete the copy afterwards.
+missing or stale snapshot instead of skipping it.
 
 With SQLBuild 0.175.0 the local run is expected to end with `ERROR=8`: five scenarios fail while
 translating Snowflake SQL to DuckDB, and three report `X602` (stale snapshot) because of a
@@ -175,18 +172,14 @@ collation). Check with `SHOW PARAMETERS LIKE 'DEFAULT_DDL_COLLATION' IN SCHEMA <
 ### Load, build and test
 
 ```bash
-sqb build --target edge --select tag:edge_case --exclude stg_edge_partner_listings+ --concurrency 8
+sqb build --target edge --select tag:edge_case --concurrency 8
 sqb test --target edge --select tag:edge_case --concurrency 8
 sqb scenario test --target edge --select edge_cases
 ```
 
 The build loads the `edge__*` sources from fixed rows that `data_scale` does not change, then
 builds the edge models and runs their tests and audits. `fct_edge_delivery_events` is disabled and
 reports `skip: disabled` (see [Pending acceptance cases](#pending-acceptance-cases)).
-`stg_edge_partner_listings` and its downstream models are excluded because planning fails once the
-`edge__partner_listings` table exists (see Known limitations in the README); the
-`edge_partner_listings__star_contract_and_quoted_export` scenario covers the models downstream of
-it.
 
 ### Replay the delivery change log as two loads
 

benchmarks/duckshop/RUNBOOK.md

@@ -1,7 +1,8 @@
 MODEL (
   description """
     One row per web session with its cart activity worked out from event item arrays using FILTER
-    and TRANSFORM lambdas: items added, removed, net units, and the checkout steps reached.
+    and TRANSFORM lambdas: items added (distinct SKUs in sorted order), removed, net units, and the
+    checkout steps reached.
   """,
   columns (
     session_id (type VARCHAR, nullable false, audits [not_null, unique]),
@@ -91,10 +92,13 @@ final AS (
     s.session_id,
     s.anonymous_id,
     s.session_start_utc,
-    ARRAY_DISTINCT(TRANSFORM(
-      ARRAY_FLATTEN(COALESCE(sc.added_item_lists, ARRAY_CONSTRUCT())),
-      i -> UPPER(i:sku::VARCHAR)
-    ))::ARRAY AS added_skus,
+    -- ARRAY_AGG and ARRAY_DISTINCT leave element order unspecified; sort for a stable result.
+    ARRAY_SORT(
+      ARRAY_DISTINCT(TRANSFORM(
+        ARRAY_FLATTEN(COALESCE(sc.added_item_lists, ARRAY_CONSTRUCT())),
+        i -> UPPER(i:sku::VARCHAR)
+      ))
+    )::ARRAY AS added_skus,
     COALESCE(sc.removed_sku_count, 0)::INT AS removed_sku_count,
     COALESCE(sc.net_units, 0)::INT AS net_units,
     sc.max_added_unit_minor::INT AS max_added_unit_minor,

benchmarks/duckshop/project/models/intermediate/clickstream/int_cart_funnel.sql

@@ -49,15 +49,29 @@ warehouse_regions AS (
     ON w.country_code = c.country_code
 ),
 
+busiest_slots AS (
+  SELECT
+    s.warehouse_code,
+    s.slot_start_local::DATE AS slot_date_local,
+    s.slot_start_local AS busiest_slot_start_local
+  FROM slots AS s
+  WHERE s.capacity <> 0
+  -- The earliest of equally busy slots, so ties give one answer.
+  QUALIFY
+    ROW_NUMBER() OVER (
+      PARTITION BY s.warehouse_code, s.slot_start_local::DATE
+      ORDER BY s.booked / s.capacity DESC, s.slot_start_local, s.slot_id
+    ) = 1
+),
+
 by_day AS (
   SELECT
     s.warehouse_code,
     s.slot_start_local::DATE AS slot_date_local,
     COUNT(*) AS slot_count,
     SUM(s.capacity) AS total_capacity,
     SUM(s.booked) AS total_booked,
-    COUNT_IF(s.remaining_capacity = 0) AS full_slot_count,
-    MAX_BY(s.slot_start_local, s.booked / NULLIF(s.capacity, 0)) AS busiest_slot_start_local
+    COUNT_IF(s.remaining_capacity = 0) AS full_slot_count
   FROM slots AS s
   GROUP BY
     s.warehouse_code,
@@ -72,13 +86,16 @@ with_calendar AS (
     d.total_capacity,
     d.total_booked,
     d.full_slot_count,
-    d.busiest_slot_start_local,
+    b.busiest_slot_start_local,
     r.region_code,
     (
       h.holiday_date IS NOT NULL
     ) AS is_holiday,
     d.total_booked / NULLIF(d.total_capacity, 0) AS utilisation_raw
   FROM by_day AS d
+  LEFT JOIN busiest_slots AS b
+    ON d.warehouse_code = b.warehouse_code
+    AND d.slot_date_local = b.slot_date_local
   LEFT JOIN warehouse_regions AS r
     ON d.warehouse_code = r.warehouse_code
   LEFT JOIN holidays AS h
@@ -108,9 +125,13 @@ final AS (
       @const("unknown_region_code")
     )::VARCHAR AS region_code,
     d.is_holiday::BOOLEAN AS is_holiday,
+    -- The average as an exact decimal sum over a count, so rounding gives one answer.
     ROUND(
       d.utilisation_raw
-        - AVG(d.utilisation_raw) OVER (
+        - SUM(d.utilisation_raw) OVER (
+          PARTITION BY d.warehouse_code, DAYOFWEEKISO(d.slot_date_local)
+        )
+        / COUNT(d.utilisation_raw) OVER (
           PARTITION BY d.warehouse_code, DAYOFWEEKISO(d.slot_date_local)
         ),
       4

benchmarks/duckshop/project/models/intermediate/fulfilment/int_delivery_slot_utilisation.sql

@@ -4,6 +4,9 @@ MODEL (
     and prices in major units
   """,
   tags [edge_case],
+  columns (
+    "listingId" (nullable false, audits [not_null, unique]),
+  ),
 );
 
 WITH listings AS (