Skip to content

Commit 62fe1b8

Browse files
committed
fix(scenario): run scenario fixture and check SQL as authored
Scenario expected, assertion, helper and fixture SQL was parsed and regenerated with the generic dialect to swap relation markers, so dialect syntax was rewritten or rejected (S506) and the executed SQL differed from what was authored. Markers are now replaced in the authored text, skipping comments and quoted text under the project adapter's lexical rules, also for --local replay: backslash escapes on Snowflake, BigQuery and Databricks, E'...' strings on DuckDB and Postgres, nested block comments where the dialect nests them, and BigQuery '#' and Snowflake '//' line comments. A marker the scenario cannot resolve, or SQL with an unclosed string or comment, fails planning with S511 naming the scenario file instead of being left in the executed SQL. Local replay no longer regenerates SQL captured from DuckDB.
1 parent 8a42599 commit 62fe1b8

38 files changed

Lines changed: 1310 additions & 478 deletions

File tree

‎Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ E2E_DUCKDB_BUILD_INCREMENTAL_PATHS := \
116116
E2E_DUCKDB_CLI_DATA_PATHS := \
117117
tests/e2e/src/sqlbuild/cli/commands/main/load \
118118
tests/e2e/src/sqlbuild/cli/commands/main/providers \
119+
tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_authored_check_sql.py \
119120
tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py \
120121
tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_capture.py \
121122
tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_namespace.py

‎src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,7 @@ This:
269269
1. Checks snapshot freshness via the input fingerprint (missing/stale snapshots are skipped by default)
270270
2. Creates a temporary DuckDB database at `target/run/scenarios/<scenario_name>/local.duckdb`
271271
3. Loads JSONL snapshots into typed DuckDB tables using column metadata from `scenario.json`
272-
4. Transpiles model and check SQL from the project adapter dialect to DuckDB
272+
4. Transpiles model and check SQL from the project adapter dialect to DuckDB; SQL captured from a DuckDB project runs as authored
273273
5. Builds functions, models, and runs expected/assertion checks in DuckDB
274274
6. Keeps the local DuckDB file for inspection (it lives under `target/`, so it's always retained)
275275

@@ -329,3 +329,4 @@ See the [CLI reference](../cli/scenario.md) for full command documentation.
329329

330330
- Custom materializations are not supported in scenarios yet. Scenario models using custom materializations will fail with a clear error.
331331
- Local replay transpiles SQL from the project adapter dialect to DuckDB. Adapter-specific SQL that cannot be translated will produce a clear error with the failing resource name and reason.
332+
- Outside local replay, scenario fixture, expected and assertion SQL runs as authored: SQLBuild replaces `__ref`, `__source`, `__seed` and `__dbt_ref` calls with relation names and may wrap a partial fixture to add its missing columns, but never regenerates the query. Calls inside comments and quoted text are left unchanged. Quoted text and comments follow the project adapter's rules, including with `--local`: backslash escapes on Snowflake, BigQuery and Databricks, `E'...'` strings on DuckDB and Postgres, nested block comments on DuckDB, Postgres, Databricks and SQL Server, `#` line comments on BigQuery, and `//` line comments on Snowflake. A call naming a relation the scenario does not provide, or SQL with an unclosed string or comment, fails planning with `S511` naming the scenario file.

‎src/sqlbuild/adapter/contract/classes/base_adapter.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@
7575
)
7676
from sqlbuild.compiler.compile.types import FunctionLanguage
7777
from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord
78+
from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax
7879
from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS
7980
from sqlbuild.spec.contracts.models import SeedCsvSettings
8081
from sqlbuild.sql_values.exceptions import SqlValueRenderingError
@@ -101,6 +102,7 @@ class BaseAdapter(RetentionAdapterMixin, StrictAdapter):
101102

102103
adapter_name: ClassVar[str]
103104
sql_analysis_dialect_name: ClassVar[str | None] = None
105+
sql_lexical_syntax: ClassVar[SqlLexicalSyntax] = SqlLexicalSyntax()
104106
max_identifier_length: ClassVar[int] = 63
105107
state_tables_transient: ClassVar[bool] = False
106108
relation_grants_supported: ClassVar[bool] = True

‎src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@
8080
from sqlbuild.adapter.type_system.main.types_equal import types_equal
8181
from sqlbuild.compiler.compile.types import FunctionLanguage
8282
from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord
83+
from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax
8384
from sqlbuild.diagnostics.main.log_sql import log_sql
8485
from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS
8586
from sqlbuild.spec.contracts.models import SeedCsvSettings
@@ -100,6 +101,9 @@ def get_columns_for_relations(
100101
)
101102

102103
sql_analysis_dialect_name: ClassVar[str | None] = "duckdb"
104+
sql_lexical_syntax: ClassVar[SqlLexicalSyntax] = SqlLexicalSyntax(
105+
escape_string_prefix=True, nested_block_comments=True
106+
)
103107
relation_grants_supported: ClassVar[bool] = False
104108

105109
def supports_zero_copy_clone(self) -> bool:

‎src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,7 @@
127127
)
128128
from sqlbuild.compiler.compile.types import FunctionLanguage
129129
from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord
130+
from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax
130131
from sqlbuild.diagnostics.main.log_sql import log_sql
131132
from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle
132133
from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS
@@ -150,6 +151,11 @@ def get_columns_for_relations(
150151

151152
adapter_name: ClassVar[str] = BuiltinAdapter.BIGQUERY.value
152153
sql_analysis_dialect_name: ClassVar[str | None] = "bigquery"
154+
sql_lexical_syntax: ClassVar[SqlLexicalSyntax] = SqlLexicalSyntax(
155+
backslash_escape_quotes=frozenset({"'", '"', "`"}),
156+
triple_quoted_strings=True,
157+
line_comment_prefixes=frozenset({"--", "#"}),
158+
)
153159
views_read_with_reader_access: ClassVar[bool] = True
154160
max_identifier_length: ClassVar[int] = 1024
155161
_snapshot_sql_dialect: ClassVar[SnapshotSqlDialect] = SnapshotSqlDialect(

‎src/sqlbuild/adapters/databricks/classes/databricks_adapter.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@
115115
)
116116
from sqlbuild.compiler.compile.types import FunctionLanguage
117117
from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord
118+
from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax
118119
from sqlbuild.diagnostics.main.log_sql import log_sql
119120
from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS
120121
from sqlbuild.spec.contracts.models import SeedCsvSettings
@@ -136,6 +137,11 @@ def get_columns_for_relations(
136137

137138
adapter_name: ClassVar[str] = BuiltinAdapter.DATABRICKS.value
138139
sql_analysis_dialect_name: ClassVar[str | None] = "databricks"
140+
sql_lexical_syntax: ClassVar[SqlLexicalSyntax] = SqlLexicalSyntax(
141+
backslash_escape_quotes=frozenset({"'", '"'}),
142+
raw_string_prefix=True,
143+
nested_block_comments=True,
144+
)
139145
max_identifier_length: ClassVar[int] = 255
140146
_snapshot_sql_dialect: ClassVar[SnapshotSqlDialect] = SnapshotSqlDialect(
141147
timestamp_type="TIMESTAMP",

‎src/sqlbuild/adapters/postgres/classes/postgres_adapter.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@
9898
from sqlbuild.adapters.postgres.constants import TABLE_FUNCTION_RETURN_TYPE
9999
from sqlbuild.compiler.compile.types import FunctionLanguage
100100
from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord
101+
from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax
101102
from sqlbuild.diagnostics.main.log_sql import log_sql
102103
from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS
103104
from sqlbuild.spec.contracts.models import SeedCsvSettings
@@ -119,6 +120,9 @@ def get_columns_for_relations(
119120

120121
adapter_name: ClassVar[str] = BuiltinAdapter.POSTGRES.value
121122
sql_analysis_dialect_name: ClassVar[str | None] = "postgres"
123+
sql_lexical_syntax: ClassVar[SqlLexicalSyntax] = SqlLexicalSyntax(
124+
escape_string_prefix=True, nested_block_comments=True
125+
)
122126
max_identifier_length: ClassVar[int] = 63
123127
_snapshot_sql_dialect: ClassVar[SnapshotSqlDialect] = SnapshotSqlDialect(
124128
timestamp_type="TIMESTAMP",

‎src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@
138138
)
139139
from sqlbuild.compiler.compile.types import FunctionLanguage
140140
from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord
141+
from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax
141142
from sqlbuild.cost.main.collection import collect_snowflake_cost
142143
from sqlbuild.cost.models import RunCostSummary
143144
from sqlbuild.cost.types import CostCapability, CostStatus
@@ -164,6 +165,9 @@ class SnowflakeAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter):
164165

165166
adapter_name: ClassVar[str] = BuiltinAdapter.SNOWFLAKE.value
166167
sql_analysis_dialect_name: ClassVar[str | None] = "snowflake"
168+
sql_lexical_syntax: ClassVar[SqlLexicalSyntax] = SqlLexicalSyntax(
169+
backslash_escape_quotes=frozenset({"'"}), line_comment_prefixes=frozenset({"--", "//"})
170+
)
167171
execution_duration_limit_seconds: ClassVar[int | None] = MAX_STATEMENT_TIMEOUT_SECONDS
168172
metadata_inspection_concurrency: ClassVar[int] = _METADATA_INSPECTION_CONCURRENCY
169173
exact_column_inspection_limit: ClassVar[int] = _EXACT_COLUMN_INSPECTION_LIMIT

‎src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@
105105
)
106106
from sqlbuild.compiler.compile.types import FunctionLanguage
107107
from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord
108+
from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax
108109
from sqlbuild.diagnostics.main.log_sql import log_sql
109110
from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS
110111
from sqlbuild.spec.contracts.models import SeedCsvSettings
@@ -135,6 +136,7 @@ def get_columns_for_relations(
135136

136137
adapter_name: ClassVar[str] = BuiltinAdapter.SQLSERVER.value
137138
sql_analysis_dialect_name: ClassVar[str | None] = "tsql"
139+
sql_lexical_syntax: ClassVar[SqlLexicalSyntax] = SqlLexicalSyntax(nested_block_comments=True)
138140
max_identifier_length: ClassVar[int] = 128
139141
_snapshot_sql_dialect: ClassVar[SnapshotSqlDialect] = SnapshotSqlDialect(
140142
timestamp_type="DATETIME2",

‎src/sqlbuild/cli/commands/_helpers/scenario_execution/local_run.py‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
from sqlbuild.compiler.compile.models import CompiledSqlScenario
2020
from sqlbuild.compiler.pipeline.models import CompilePipelineResult
2121
from sqlbuild.executor.pipeline.main.run import run_scenario_local_test_pipeline
22-
from sqlbuild.executor.scenario.models import ScenarioRunResult
22+
from sqlbuild.executor.scenario.models import ScenarioLocalReplaySource, ScenarioRunResult
2323
from sqlbuild.executor.scenario.types import ScenarioLocalRunStatus
2424
from sqlbuild.presentation.classes.cli_style import CliStyle
2525
from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter
@@ -34,8 +34,7 @@ def run_local_scenarios(
3434
adapter: BaseAdapter,
3535
project_name: str,
3636
strict: bool,
37-
capture_adapter: str,
38-
capture_dialect: str,
37+
replay_source: ScenarioLocalReplaySource,
3938
target_dir: Path,
4039
output_context: ScenarioRunOutputContext,
4140
) -> int:
@@ -66,8 +65,7 @@ def run_local_scenarios(
6665
adapter=adapter,
6766
project_name=project_name,
6867
strict=strict,
69-
capture_adapter=capture_adapter,
70-
capture_dialect=capture_dialect,
68+
replay_source=replay_source,
7169
on_scenario_start=lambda _scenario: (
7270
scenario_status.start(activity) if status_is_tty else None
7371
),

0 commit comments

Comments
 (0)