Skip to content

Commit db5cd56

Browse files
authored
fix(compiler)!: reject unknown schema-change values and follow dialect quoting in compile scanners (#912)
* fix(compiler)!: reject unknown on_schema_change, replay_on_change and dlt_sources keys Unknown on_schema_change and replay_on_change values fell back to the default policy, and dlt_sources groups and resources accepted any key. They now fail compilation: MODEL headers and [defaults]/[path_defaults] name the value, list the valid ones and show a valid header or TOML line; dlt keys get the nearest supported key as help. bounded-<duration> must carry a positive duration. The planner no longer maps unknown values to a default. BREAKING CHANGE: unknown on_schema_change or replay_on_change values and unknown dlt_sources keys now fail compilation. Refs: CHI-465 * fix(compiler): follow adapter lexical rules in compile-step SQL scanners Reference extraction, SQL test and scenario CTE scanning, assertion target detection, scope relationships and native SQL test planning now honour the active adapter's lexical syntax (backslash escapes, escape and raw string prefixes, triple quotes, dialect line comments and nested block comments) instead of generic SQL rules. Cache keys include the lexical syntax. Refs: CHI-466 * fix(compiler): stop double-encoding JOIN USING columns in exact-name binding When a relation carries exact (quoted) column names, such as a source table read from the warehouse, binding encodes every identifier. JOIN USING columns were encoded twice, once through the SELECT and once through the JOIN node, so valid joins failed with B002 "JOIN USING column must exist on both sides". The capped microbatch dependency fixtures read a real source table again. Refs: CHI-467 * fix(cli): print a JSON diagnostic report when compile --json stops on an error When compile raised an error, such as a header parse failure, --json wrote nothing to stdout. It now prints one JSON document with has_errors, stopped, and the error's code, message and help, matching the stderr text. The exit code is unchanged and progress stays on stderr. Refs: CHI-470 * test(executor): make the microbatch overlap test wait for overlap instead of sleeping The concurrent microbatch test relied on a 50 ms sleep for three delta stagings to overlap and saw two under load. After each model's serial first batch, the tracking adapter now holds stagings until three are active, with a 30 s timeout that fails the test with a clear message. Refs: CHI-468 * fix(rules): key the Rules cache on the native build, not only the version Local development builds shared one SQLBuild version, so the Rules cache under target/rules-cache reused results from older builds until it was cleared. A build script now hashes the native crate sources and lockfile, and the ruleset fingerprint, model and custom-rule cache identities, and the SQL rule cache use the version plus that hash. Released builds keep a stable identity. Refs: CHI-471 * test(e2e): use the valid replay_on_change forward in microbatch lifecycle fixtures
1 parent c93a99c commit db5cd56

129 files changed

Lines changed: 2979 additions & 409 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎crates/sqlbuild-rules-native/Cargo.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ stacker.workspace = true
3030
sqlparser.workspace = true
3131
toml.workspace = true
3232

33+
[build-dependencies]
34+
sha2.workspace = true
35+
3336
[dev-dependencies]
3437
tempfile.workspace = true
3538

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
//! Derive a content identity for this native build so local caches notice source changes.
2+
//!
3+
//! The identity hashes every file under the listed crate entries plus the workspace lockfile,
4+
//! keyed by `/`-separated paths relative to the crate root so it matches across platforms.
5+
//! Entries that do not exist (for example the lockfile outside a workspace checkout) are left
6+
//! out of both the hash and the `rerun-if-changed` list, because Cargo reruns a build script on
7+
//! every build when a watched path is missing.
8+
9+
use sha2::{Digest, Sha256};
10+
use std::path::{Component, Path, PathBuf};
11+
12+
const HASHED_ENTRIES: [&str; 3] = ["Cargo.toml", "src", "function_names"];
13+
const WORKSPACE_LOCKFILE: &str = "../../Cargo.lock";
14+
15+
fn main() -> Result<(), String> {
16+
let crate_dir =
17+
PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").map_err(|error| error.to_string())?);
18+
let mut files: Vec<(String, PathBuf)> = Vec::new();
19+
for entry in HASHED_ENTRIES {
20+
let path = crate_dir.join(entry);
21+
if path.exists() {
22+
println!("cargo:rerun-if-changed={}", path.display());
23+
collect_files(&crate_dir, &path, &mut files)?;
24+
}
25+
}
26+
files.sort();
27+
let lockfile = crate_dir.join(WORKSPACE_LOCKFILE);
28+
if lockfile.is_file() {
29+
println!("cargo:rerun-if-changed={}", lockfile.display());
30+
files.push((WORKSPACE_LOCKFILE.to_owned(), lockfile));
31+
}
32+
let mut digest = Sha256::new();
33+
for (label, file) in &files {
34+
digest.update(label.as_bytes());
35+
digest.update([0]);
36+
digest.update(std::fs::read(file).map_err(|error| error.to_string())?);
37+
digest.update([0]);
38+
}
39+
let identity = format!("{:x}", digest.finalize());
40+
println!(
41+
"cargo:rustc-env=SQLBUILD_NATIVE_SOURCE_HASH={}",
42+
&identity[..16]
43+
);
44+
Ok(())
45+
}
46+
47+
fn portable_label(crate_dir: &Path, path: &Path) -> Result<String, String> {
48+
let relative = path
49+
.strip_prefix(crate_dir)
50+
.map_err(|error| error.to_string())?;
51+
let parts: Vec<String> = relative
52+
.components()
53+
.map(|component| match component {
54+
Component::Normal(part) => Ok(part.to_string_lossy().into_owned()),
55+
other => Err(format!("unexpected path component {other:?}")),
56+
})
57+
.collect::<Result<_, _>>()?;
58+
Ok(parts.join("/"))
59+
}
60+
61+
fn collect_files(
62+
crate_dir: &Path,
63+
path: &Path,
64+
files: &mut Vec<(String, PathBuf)>,
65+
) -> Result<(), String> {
66+
if path.is_file() {
67+
files.push((portable_label(crate_dir, path)?, path.to_path_buf()));
68+
return Ok(());
69+
}
70+
if !path.is_dir() {
71+
return Ok(());
72+
}
73+
for entry in std::fs::read_dir(path).map_err(|error| error.to_string())? {
74+
collect_files(
75+
crate_dir,
76+
&entry.map_err(|error| error.to_string())?.path(),
77+
files,
78+
)?;
79+
}
80+
Ok(())
81+
}

‎crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple};
77
use pyo3::{FromPyObject, pyfunction, wrap_pyfunction};
88

99
use crate::configuration::main::load;
10-
use crate::constants::API_VERSION;
10+
use crate::constants::{API_VERSION, NATIVE_BUILD_IDENTITY};
1111
use crate::engine::main::evaluate;
1212
use crate::models::CatalogueResponse;
1313
use crate::rules::main::{catalogue, selected_codes};
@@ -469,5 +469,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
469469
module.add_function(wrap_pyfunction!(render_owned_skill, module)?)?;
470470
module.add_function(wrap_pyfunction!(skill_freshness, module)?)?;
471471
module.add("API_VERSION", API_VERSION)?;
472+
module.add("BUILD_IDENTITY", NATIVE_BUILD_IDENTITY)?;
472473
Ok(())
473474
}

‎crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/helper_scope.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -242,7 +242,7 @@ fn is_generated_cte_name(name: &str) -> bool {
242242

243243
/// Distinct lowercase identifiers outside strings and comments, in first-appearance order.
244244
fn identifier_tokens(sql: &str, patterns: &SqlTestPatterns) -> Vec<String> {
245-
let protected = protected_ranges(&patterns.protected, sql);
245+
let protected = protected_ranges(&patterns.lexical, sql);
246246
let mut seen: HashSet<String> = HashSet::new();
247247
let mut tokens: Vec<String> = Vec::new();
248248
let mut push = |token: &str| {

‎crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/markers.rs‎

Lines changed: 18 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -5,21 +5,21 @@ use std::collections::HashSet;
55
use regex::{Captures, Regex};
66

77
use crate::compiler::_helpers::sql_tests::planning::compile_error;
8-
use crate::sql_scan::main::matching_paren::matching_paren;
8+
use crate::sql_scan::main::dialect_matching_paren::dialect_matching_paren;
9+
use crate::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges;
910
use crate::sql_scan::main::skip_whitespace::skip_whitespace;
10-
use crate::sql_scan::models::QuotePolicy;
11-
use crate::sql_scan::models::Unclosed;
11+
use crate::sql_scan::models::{LexicalSyntax, Unclosed};
1212

1313
pub(crate) fn replace_callable_markers<F>(
1414
sql: &str,
1515
pattern: &Regex,
16-
protected_pattern: &Regex,
16+
syntax: &LexicalSyntax,
1717
mut replacement: F,
1818
) -> Result<(String, HashSet<String>), String>
1919
where
2020
F: FnMut(&str, &str) -> Option<(String, bool)>,
2121
{
22-
let protected = protected_ranges(protected_pattern, sql);
22+
let protected = protected_ranges(syntax, sql);
2323
let mut output = String::with_capacity(sql.len());
2424
let mut reached: HashSet<String> = HashSet::new();
2525
let mut cursor = 0;
@@ -34,7 +34,7 @@ where
3434
if !sql[suffix_start..].starts_with('(') {
3535
continue;
3636
}
37-
let suffix_end = matching_paren_end(sql, suffix_start)?;
37+
let suffix_end = matching_paren_end(sql, suffix_start, syntax)?;
3838
let Some(name) = captures.get(1).map(|value| value.as_str()) else {
3939
continue;
4040
};
@@ -53,8 +53,8 @@ where
5353
Ok((output, reached))
5454
}
5555

56-
fn matching_paren_end(sql: &str, open: usize) -> Result<usize, String> {
57-
matching_paren(sql.as_bytes(), open, QuotePolicy::COMPILER)
56+
fn matching_paren_end(sql: &str, open: usize, syntax: &LexicalSyntax) -> Result<usize, String> {
57+
dialect_matching_paren(sql.as_bytes(), open, syntax)
5858
.map(|close| close + 1)
5959
.map_err(|error| {
6060
compile_error(match error {
@@ -68,27 +68,27 @@ fn matching_paren_end(sql: &str, open: usize) -> Result<usize, String> {
6868
pub(crate) fn replace_named_markers<F>(
6969
sql: &str,
7070
pattern: &Regex,
71-
protected_pattern: &Regex,
71+
syntax: &LexicalSyntax,
7272
mut replacement: F,
7373
) -> String
7474
where
7575
F: FnMut(&str) -> Option<String>,
7676
{
77-
replace_marker_captures(sql, pattern, protected_pattern, |captures| {
77+
replace_marker_captures(sql, pattern, syntax, |captures| {
7878
replacement(captures.get(1)?.as_str())
7979
})
8080
}
8181

8282
pub(crate) fn replace_dbt_ref_markers<F>(
8383
sql: &str,
8484
pattern: &Regex,
85-
protected_pattern: &Regex,
85+
syntax: &LexicalSyntax,
8686
mut replacement: F,
8787
) -> String
8888
where
8989
F: FnMut(&str) -> Option<String>,
9090
{
91-
replace_marker_captures(sql, pattern, protected_pattern, |captures| {
91+
replace_marker_captures(sql, pattern, syntax, |captures| {
9292
let first = captures.get(1)?.as_str();
9393
let name = captures.get(2).map_or_else(
9494
|| first.to_string(),
@@ -101,13 +101,13 @@ where
101101
fn replace_marker_captures<F>(
102102
sql: &str,
103103
pattern: &Regex,
104-
protected_pattern: &Regex,
104+
syntax: &LexicalSyntax,
105105
mut replacement: F,
106106
) -> String
107107
where
108108
F: FnMut(&Captures<'_>) -> Option<String>,
109109
{
110-
let protected = protected_ranges(protected_pattern, sql);
110+
let protected = protected_ranges(syntax, sql);
111111
let mut output = String::with_capacity(sql.len());
112112
let mut cursor = 0;
113113
for captures in pattern.captures_iter(sql) {
@@ -128,8 +128,8 @@ where
128128
output
129129
}
130130

131-
pub(crate) fn marker_names(pattern: &Regex, protected_pattern: &Regex, sql: &str) -> Vec<String> {
132-
let protected = protected_ranges(protected_pattern, sql);
131+
pub(crate) fn marker_names(pattern: &Regex, syntax: &LexicalSyntax, sql: &str) -> Vec<String> {
132+
let protected = protected_ranges(syntax, sql);
133133
let mut names: Vec<String> = Vec::new();
134134
for captures in pattern.captures_iter(sql) {
135135
let Some(full) = captures.get(0) else {
@@ -145,11 +145,8 @@ pub(crate) fn marker_names(pattern: &Regex, protected_pattern: &Regex, sql: &str
145145
names
146146
}
147147

148-
pub(crate) fn protected_ranges(pattern: &Regex, sql: &str) -> Vec<(usize, usize)> {
149-
pattern
150-
.find_iter(sql)
151-
.map(|value| (value.start(), value.end()))
152-
.collect()
148+
pub(crate) fn protected_ranges(syntax: &LexicalSyntax, sql: &str) -> Vec<(usize, usize)> {
149+
dialect_non_code_ranges(sql, syntax)
153150
}
154151

155152
pub(crate) fn in_protected_range(index: usize, ranges: &[(usize, usize)]) -> bool {

0 commit comments

Comments
 (0)