Skip to content

Commit 30905fd

Browse files
authored
refactor: host lint fixes, migration proofs and dbt pipelines in Rust (#1107)
* refactor: move column rename candidate helpers to Rust * refactor: move lint compile-fact verification into native rules * chore: update Python boundary after migration and lint helpers * refactor: host dbt invocation and result backfill in Rust * chore: update Python boundary after dbt pipeline port * refactor: port migration query proofs and fingerprints to Rust * chore: update Python boundary after migration proof port * refactor: host compiler-verified lint fix sessions natively * refactor: prepare lint expansions and header ranges natively * chore: update Python boundary after lint orchestration port * refactor: host dbt SQLBuild planning and execution gates * refactor: host remaining dbt project pipelines in Rust * chore: update Python boundary after dbt orchestration port * fix: allow native setup time for formatter CI guards * test: allow temporary full-build memory headroom * ci: allow cache guard dependency setup time
1 parent 96fa105 commit 30905fd

113 files changed

Lines changed: 6004 additions & 3228 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/verify.yml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -412,7 +412,7 @@ jobs:
412412
needs: [changes, lockfile]
413413
if: needs.changes.outputs.format_performance == 'true'
414414
runs-on: ubuntu-24.04
415-
timeout-minutes: 10
415+
timeout-minutes: 30
416416
steps:
417417
- uses: actions/checkout@v4
418418
- uses: astral-sh/setup-uv@v5
@@ -430,6 +430,7 @@ jobs:
430430
- name: Install dependencies
431431
run: uv sync --locked --all-groups --all-extras --python 3.12
432432
- name: Run formatter performance guards
433+
timeout-minutes: 10
433434
run: >-
434435
uv run pytest
435436
tests/e2e/src/sqlbuild/cli/commands/main/format/test_format_performance.py
@@ -659,7 +660,7 @@ jobs:
659660
needs: [changes, lockfile]
660661
if: needs.changes.outputs.performance == 'true'
661662
runs-on: ubuntu-24.04
662-
timeout-minutes: 15
663+
timeout-minutes: 30
663664
strategy:
664665
fail-fast: false
665666
matrix:
@@ -681,6 +682,7 @@ jobs:
681682
- name: Install dependencies
682683
run: uv sync --locked --all-groups --all-extras
683684
- name: Run fresh-process compile cache guard under 4 GiB
685+
timeout-minutes: 15
684686
env:
685687
BENCHMARK_MODELS: ${{ matrix.models }}
686688
MEMORY_LIMIT_BYTES: "4294967296"
@@ -703,6 +705,7 @@ jobs:
703705
704706
- name: Run varied 1k warm-cache guard under 4 GiB
705707
if: matrix.models == 3000
708+
timeout-minutes: 15
706709
env:
707710
MEMORY_LIMIT_BYTES: "4294967296"
708711
run: |

‎crates/sqlbuild-analysis/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ pub mod lineage_graph;
2121
pub mod manifest;
2222
pub mod planner_changes;
2323
pub mod planner_identity;
24+
pub mod planner_migrations;
2425
pub mod planner_scope;
2526
pub mod planner_selection;
2627
pub mod planner_strategy;
Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
//! Canonical migration AST values.
2+
3+
use crate::planner_migrations::constants::{HAVING_MAX_KEY, OPAQUE_TUPLE_FIELDS};
4+
use serde_json::Value;
5+
use std::collections::BTreeSet;
6+
7+
/// Tuple-valued AST fields are opaque to the historical Python dict/list walkers.
8+
pub(crate) fn opaque_field(key: &str, value: &Value) -> bool {
9+
key == HAVING_MAX_KEY
10+
|| (OPAQUE_TUPLE_FIELDS.contains(&key)
11+
&& value
12+
.as_array()
13+
.is_some_and(|items| !items.is_empty() && items.iter().all(Value::is_array)))
14+
}
15+
16+
pub(crate) fn strip_formatting(node: &mut Value) {
17+
match node {
18+
Value::Object(object) => {
19+
object.shift_remove("span");
20+
for (key, value) in object {
21+
if opaque_field(key, value) {
22+
continue;
23+
}
24+
if key.ends_with("comments") {
25+
*value = Value::Array(Vec::new());
26+
} else {
27+
strip_formatting(value);
28+
}
29+
}
30+
}
31+
Value::Array(items) => items.iter_mut().for_each(strip_formatting),
32+
_ => {}
33+
}
34+
}
35+
36+
pub(crate) fn canonical(node: &mut Value) {
37+
match node {
38+
Value::Array(items) => items.iter_mut().for_each(canonical),
39+
Value::Object(object) => {
40+
for (key, value) in object.iter_mut() {
41+
if !opaque_field(key, value) {
42+
canonical(value);
43+
}
44+
}
45+
if object.get("quoted") == Some(&Value::Bool(false))
46+
&& let Some(Value::String(name)) = object.get_mut("name")
47+
{
48+
*name = name.to_lowercase();
49+
}
50+
}
51+
_ => {}
52+
}
53+
}
54+
55+
pub(crate) fn identifier(node: &Value) -> Option<&str> {
56+
node.get("name")?.as_str()
57+
}
58+
59+
pub(crate) fn identifiers(node: &Value) -> Vec<String> {
60+
items(node)
61+
.iter()
62+
.filter_map(identifier)
63+
.map(str::to_lowercase)
64+
.collect()
65+
}
66+
67+
pub(crate) fn items(node: &Value) -> &[Value] {
68+
node.as_array().map_or(&[], Vec::as_slice)
69+
}
70+
71+
pub(crate) fn clause_items(node: &Value) -> &[Value] {
72+
if node.is_array() {
73+
items(node)
74+
} else {
75+
items(&node["expressions"])
76+
}
77+
}
78+
79+
pub(crate) fn references(node: &Value, unqualified: bool) -> BTreeSet<String> {
80+
matching_fields(node, "column")
81+
.into_iter()
82+
.filter(|column| !unqualified || column["table"].is_null())
83+
.filter_map(|column| identifier(&column["name"]))
84+
.map(str::to_lowercase)
85+
.collect()
86+
}
87+
88+
pub(crate) fn matching_fields<'a>(node: &'a Value, key: &str) -> Vec<&'a Value> {
89+
let mut found: Vec<&Value> = Vec::new();
90+
let mut pending: Vec<&Value> = vec![node];
91+
while let Some(current) = pending.pop() {
92+
match current {
93+
Value::Array(items) => pending.extend(items),
94+
Value::Object(object) => {
95+
if let Some(value) = object.get(key) {
96+
found.push(value);
97+
}
98+
pending.extend(
99+
object
100+
.iter()
101+
.filter(|(key, value)| !opaque_field(key, value))
102+
.map(|(_, value)| value),
103+
);
104+
}
105+
_ => {}
106+
}
107+
}
108+
found
109+
}
Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
//! Conservative alias resolution in unchanged and renamed projections.
2+
3+
use crate::planner_migrations::_helpers::ast::identifier;
4+
use crate::planner_migrations::models::QueryShape;
5+
use serde_json::Value;
6+
use std::collections::BTreeSet;
7+
8+
pub(crate) fn other_references(shape: &QueryShape, excluded: &str) -> BTreeSet<String> {
9+
shape
10+
.projections
11+
.iter()
12+
.filter(|item| item.name.to_lowercase() != excluded)
13+
.flat_map(|item| item.references.iter().cloned())
14+
.collect()
15+
}
16+
17+
pub(crate) fn binding_proven<E>(
18+
shapes: (&QueryShape, &QueryShape),
19+
names: (&str, &str),
20+
input_columns: &mut impl FnMut(&str, &str) -> Result<Option<BTreeSet<String>>, E>,
21+
) -> Result<bool, E> {
22+
let (previous, current) = shapes;
23+
let (origin, destination) = names;
24+
let mut referenced = previous.clause_references.clone();
25+
referenced.extend(current.clause_references.iter().cloned());
26+
referenced.extend(other_references(previous, origin));
27+
referenced.extend(other_references(current, destination));
28+
if !referenced.contains(origin) && !referenced.contains(destination) {
29+
return Ok(true);
30+
}
31+
let Some(relations) = &current.input_relations else {
32+
return Ok(false);
33+
};
34+
let mut inputs = current.local_input_columns.clone();
35+
for (kind, name) in relations {
36+
let Some(columns) = input_columns(kind, name)? else {
37+
return Ok(false);
38+
};
39+
inputs.extend(columns);
40+
}
41+
if inputs.contains(destination) {
42+
return Ok(false);
43+
}
44+
if inputs.contains(origin) {
45+
return Ok(previous
46+
.projections
47+
.iter()
48+
.any(|item| item.name.to_lowercase() == origin && item.passthrough));
49+
}
50+
Ok(!current.clause_references.contains(origin)
51+
&& !other_references(current, "").contains(origin))
52+
}
53+
54+
pub(crate) fn substitute(node: &mut Value, names: &[(String, String)]) {
55+
match node {
56+
Value::Array(items) => items.iter_mut().for_each(|item| substitute(item, names)),
57+
Value::Object(object) => {
58+
if object.contains_key("select") {
59+
return;
60+
}
61+
if let Some(column) = object.get_mut("column").filter(|column| column.is_object())
62+
&& column["table"].is_null()
63+
&& column["name"]["quoted"] == false
64+
&& let Some(name) = identifier(&column["name"])
65+
&& let Some((_, replacement)) = names.iter().rev().find(|(key, _)| key == name)
66+
{
67+
column["name"]["name"] = Value::String(replacement.clone());
68+
return;
69+
}
70+
object
71+
.values_mut()
72+
.for_each(|value| substitute(value, names));
73+
}
74+
_ => {}
75+
}
76+
}
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
//! Non-projection clauses and positional references of a SELECT.
2+
3+
use crate::planner_migrations::_helpers::ast::{clause_items, items, references};
4+
use crate::planner_migrations::constants::NUMBER_LITERAL;
5+
use serde_json::Value;
6+
use std::collections::BTreeSet;
7+
8+
pub(crate) fn clause_references(select: &Value) -> BTreeSet<String> {
9+
let clauses: Vec<Value> = select
10+
.as_object()
11+
.into_iter()
12+
.flat_map(|object| object.iter())
13+
.filter(|(key, _)| !["expressions", "with", "from", "joins"].contains(&key.as_str()))
14+
.map(|(_, value)| value.clone())
15+
.chain(
16+
items(&select["joins"])
17+
.iter()
18+
.filter(|join| join.is_object())
19+
.map(|join| join["on"].clone()),
20+
)
21+
.collect();
22+
references(&Value::Array(clauses), true)
23+
}
24+
25+
pub(crate) fn positional(select: &Value) -> bool {
26+
clause_items(&select["group_by"])
27+
.iter()
28+
.chain(clause_items(&select["order_by"]).iter().map(|item| {
29+
if item.is_object() {
30+
&item["this"]
31+
} else {
32+
item
33+
}
34+
}))
35+
.chain(clause_items(&select["distinct_on"]))
36+
.any(|item| item["literal"]["literal_type"] == NUMBER_LITERAL)
37+
}
Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
//! Python-compatible compact sorted JSON for migration proofs.
2+
3+
use serde_json::Value;
4+
use sqlbuild_core::json::main::dumps::dumps;
5+
use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions};
6+
7+
pub(crate) fn encode(value: &Value) -> Result<String, String> {
8+
dumps(
9+
&converted(value)?,
10+
&JsonDialect::Stdlib(StdlibJsonOptions {
11+
sort_keys: true,
12+
..StdlibJsonOptions::new(None).with_separators(",", ":")
13+
}),
14+
)
15+
.map_err(|error| format!("{error:?}"))
16+
}
17+
18+
fn converted(value: &Value) -> Result<JsonValue, String> {
19+
Ok(match value {
20+
Value::Null => JsonValue::Null,
21+
Value::Bool(flag) => JsonValue::Bool(*flag),
22+
Value::String(text) => JsonValue::String(text.clone()),
23+
Value::Number(number) => {
24+
if !number.is_f64() {
25+
JsonValue::Integer(
26+
JsonInteger::parse(&number.to_string()).ok_or("invalid integer")?,
27+
)
28+
} else {
29+
JsonValue::Float(number.as_f64().ok_or("invalid float")?)
30+
}
31+
}
32+
Value::Array(items) => {
33+
JsonValue::Array(items.iter().map(converted).collect::<Result<_, _>>()?)
34+
}
35+
Value::Object(object) => JsonValue::Object(
36+
object
37+
.iter()
38+
.map(|(key, value)| Ok((key.clone(), converted(value)?)))
39+
.collect::<Result<_, String>>()?,
40+
),
41+
})
42+
}

0 commit comments

Comments
 (0)