Skip to content

Commit 6a81536

Browse files
committed
refactor: use native SQLite history connections
Keep local history connections and query rows native while preserving the public storage API, connection overrides, transaction recovery and interruptible locking. Preserve literal filename semantics even when bundled SQLite enables URI interpretation globally. Cover persistence without Python connect and exact relative filenames through the public API. Amp-Thread-ID: T-01a18715-4e00-708e-a87d-6bd109748d66
1 parent 97a274b commit 6a81536

11 files changed

Lines changed: 414 additions & 51 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/sqlbuild-python/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ fensu-policy.workspace = true
2020
mimalloc.workspace = true
2121
pyo3.workspace = true
2222
rayon.workspace = true
23+
rusqlite.workspace = true
2324
rustc-hash.workspace = true
2425
serde.workspace = true
2526
serde_json.workspace = true

‎crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs‎

Lines changed: 27 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@ use sqlbuild_state::execution_history::models::{
4141
use sqlbuild_state::execution_history::types::UtcTime;
4242

4343
use crate::bindings::_helpers::boundary::python_calls::imported;
44-
use crate::bindings::_helpers::boundary::python_calls::keywords;
4544
use crate::bindings::_helpers::execution_history::appends::prepared_batch;
4645
use crate::bindings::_helpers::execution_history::conversion::{
4746
checked, event_filter_spec, exact_int, history_error_of_kind, new_namespace, now,
@@ -53,12 +52,13 @@ use crate::bindings::_helpers::execution_history::statements::{
5352
column, execute, execute_plain, execute_query, fetch_one, tuple_of,
5453
};
5554
use crate::bindings::_helpers::native_host::host;
55+
use crate::bindings::_helpers::native_host::sqlite_connection::{SqliteConnection, sqlite_params};
56+
use crate::bindings::_helpers::native_host::sqlite_cursor::SqliteCursor;
5657
use crate::bindings::_helpers::native_threads::reentrant_lock::ReentrantLock;
5758
use crate::bindings::_helpers::records::fields::field as native_field;
5859

5960
const SQLITE_MODULE: &str = "sqlite3";
6061
const PATHLIB_MODULE: &str = "pathlib";
61-
const MILLIS_PER_SECOND: f64 = 1000.0;
6262
const CLOSED: &str = "SQLite execution history storage is closed";
6363
const NOT_AWARE: &str = "SQLite history timestamp is not timezone-aware";
6464
const INTEGRITY_CONFLICT: ExecutorHost = ExecutorHost::IntegrityConflictError;
@@ -233,7 +233,10 @@ impl SqliteExecutionHistory {
233233
};
234234
if let Err(error) = execute_plain(&connection, SQLITE_COMMIT) {
235235
if sqlite_failure(py, &error)?
236-
&& native_field(&connection, "in_transaction")?.is_truthy()?
236+
&& match connection.cast::<SqliteConnection>() {
237+
Ok(connection) => connection.get().in_transaction(py)?,
238+
Err(_) => native_field(&connection, "in_transaction")?.is_truthy()?,
239+
}
237240
{
238241
let _ = execute_plain(&connection, SQLITE_ROLLBACK);
239242
}
@@ -310,6 +313,16 @@ impl SqliteExecutionHistory {
310313
fn replace_projection(connection: &Bound<'_, PyAny>, projected: &[RunRow]) -> PyResult<()> {
311314
let py: Python<'_> = connection.py();
312315
execute_plain(connection, SQLITE_DELETE_RUNS)?;
316+
if let Ok(connection) = connection.cast::<SqliteConnection>() {
317+
for run in projected {
318+
connection.get().execute_native(
319+
py,
320+
SQLITE_RUN_INSERT,
321+
&sqlite_params(&run_insert_query(Dialect::Sqlite, run).params)?,
322+
)?;
323+
}
324+
return Ok(());
325+
}
313326
let rows: Vec<Bound<'_, PyAny>> = projected
314327
.iter()
315328
.map(|run| {
@@ -347,10 +360,16 @@ impl SqliteExecutionHistory {
347360
None => {
348361
let query = event_insert_query(Dialect::Sqlite, event, &now(py)?);
349362
let cursor: Bound<'py, PyAny> = execute_query(connection, &query)?;
350-
let storage_id: Bound<'py, PyAny> = native_field(&cursor, "lastrowid")?;
351-
connection
352-
.call_method1("execute", (SQLITE_EVENT_BY_STORAGE_ID, (storage_id,)))?
353-
.call_method0("fetchone")?
363+
let storage_id: i64 = match cursor.cast::<SqliteCursor>() {
364+
Ok(cursor) => cursor.get().last_row_id,
365+
Err(_) => native_field(&cursor, "lastrowid")?.extract()?,
366+
};
367+
fetch_one(&execute(
368+
connection,
369+
SQLITE_EVENT_BY_STORAGE_ID,
370+
&[SqlParam::Integer(storage_id)],
371+
)?)?
372+
.unwrap_or_else(|| py.None().into_bound(py))
354373
}
355374
};
356375
if namespace.is_none() {
@@ -491,25 +510,10 @@ impl SqliteExecutionHistory {
491510
let mut connection: Option<Bound<'_, PyAny>> = None;
492511
let opened: PyResult<()> = (|| {
493512
prepare_history_directory(&database)?;
494-
let sqlite = py.import(SQLITE_MODULE)?;
495-
let opened: Bound<'_, PyAny> = native_field(&sqlite, "connect")?.call(
496-
(database.as_str(),),
497-
Some(&keywords(
498-
py,
499-
&[
500-
(
501-
"timeout",
502-
(busy_timeout_ms as f64 / MILLIS_PER_SECOND).into_bound_py_any(py)?,
503-
),
504-
("isolation_level", py.None().into_bound(py)),
505-
("check_same_thread", false.into_bound_py_any(py)?),
506-
],
507-
)?),
508-
)?;
513+
let opened = Bound::new(py, SqliteConnection::open(py, &database)?)?.into_any();
509514
connection = Some(opened.clone());
510515
slf.setattr("_connection", &opened)?;
511516
this.with_lock(py, || {
512-
opened.setattr("row_factory", native_field(&sqlite, "Row")?)?;
513517
execute_plain(&opened, &format!("PRAGMA busy_timeout = {busy_timeout_ms}"))?;
514518
execute_plain(&opened, SQLITE_FOREIGN_KEYS)?;
515519
if database != MEMORY_PATH {

‎crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ use pyo3::types::PyTuple;
55
use sqlbuild_state::execution_history::models::{Query, SqlParam};
66

77
use crate::bindings::_helpers::execution_history::conversion::python_params;
8+
use crate::bindings::_helpers::native_host::sqlite_connection::{SqliteConnection, sqlite_params};
9+
use crate::bindings::_helpers::native_host::sqlite_cursor::{SqliteCursor, SqliteRow};
810

911
/// `target.execute(sql, params)`.
1012
pub(crate) fn execute<'py>(
@@ -13,6 +15,15 @@ pub(crate) fn execute<'py>(
1315
params: &[SqlParam],
1416
) -> PyResult<Bound<'py, PyAny>> {
1517
let py: Python<'py> = target.py();
18+
if let Ok(connection) = target.cast::<SqliteConnection>() {
19+
return Bound::new(
20+
py,
21+
connection
22+
.get()
23+
.execute_native(py, sql, &sqlite_params(params)?)?,
24+
)
25+
.map(Bound::into_any);
26+
}
1627
target.call_method1("execute", (sql, python_params(py, params)?))
1728
}
1829

@@ -21,6 +32,9 @@ pub(crate) fn execute_plain<'py>(
2132
target: &Bound<'py, PyAny>,
2233
sql: &str,
2334
) -> PyResult<Bound<'py, PyAny>> {
35+
if target.is_instance_of::<SqliteConnection>() {
36+
return execute(target, sql, &[]);
37+
}
2438
target.call_method1("execute", (sql,))
2539
}
2640

@@ -34,6 +48,13 @@ pub(crate) fn execute_query<'py>(
3448

3549
/// The first row `cursor.fetchone()` returns, or `None`.
3650
pub(crate) fn fetch_one<'py>(cursor: &Bound<'py, PyAny>) -> PyResult<Option<Bound<'py, PyAny>>> {
51+
if let Ok(cursor) = cursor.cast::<SqliteCursor>() {
52+
return cursor
53+
.get()
54+
.next_row()
55+
.map(|row| Bound::new(cursor.py(), row).map(Bound::into_any))
56+
.transpose();
57+
}
3758
let row: Bound<'py, PyAny> = cursor.call_method0("fetchone")?;
3859
Ok((!row.is_none()).then_some(row))
3960
}
@@ -45,6 +66,9 @@ pub(crate) fn fetch_all<'py>(cursor: &Bound<'py, PyAny>) -> PyResult<Vec<Bound<'
4566

4667
/// `row[name]`.
4768
pub(crate) fn column<'py>(row: &Bound<'py, PyAny>, name: &str) -> PyResult<Bound<'py, PyAny>> {
69+
if let Ok(row) = row.cast::<SqliteRow>() {
70+
return row.get().column(row.py(), name);
71+
}
4872
row.get_item(name)
4973
}
5074

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
1-
//! The one Python module native code imports by path, and the staying-Python entries it names.
1+
//! Permanent Python host calls and compatibility views of native-owned execution data.
22
33
pub(crate) mod connections;
44
pub(crate) mod host;
55
pub(crate) mod registration;
66
pub(crate) mod results;
7+
pub(crate) mod sqlite_connection;
8+
pub(crate) mod sqlite_cursor;
Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
//! Rust-owned SQLite connection; Python views exist only for connection overrides.
2+
3+
use std::collections::VecDeque;
4+
use std::path::PathBuf;
5+
use std::sync::{Arc, Mutex, PoisonError};
6+
7+
use pyo3::exceptions::PyValueError;
8+
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyclass, pymethods};
9+
use rusqlite::types::Value;
10+
use rusqlite::{Connection, Error, ErrorCode, OpenFlags, params_from_iter};
11+
use sqlbuild_state::execution_history::constants::MEMORY_PATH;
12+
use sqlbuild_state::execution_history::models::SqlParam;
13+
14+
use crate::bindings::_helpers::boundary::python_calls::imported;
15+
use crate::bindings::_helpers::native_host::sqlite_cursor::{SqliteCursor, SqliteRow};
16+
17+
/// Preserve DB-API exception classes at the public storage-error boundary.
18+
fn database_error(py: Python<'_>, error: Error) -> PyErr {
19+
let class: &str = match &error {
20+
Error::SqliteFailure(error, _) if error.code == ErrorCode::ConstraintViolation => {
21+
"IntegrityError"
22+
}
23+
Error::SqliteFailure(error, _)
24+
if matches!(
25+
error.code,
26+
ErrorCode::DatabaseCorrupt | ErrorCode::NotADatabase
27+
) =>
28+
{
29+
"DatabaseError"
30+
}
31+
_ => "OperationalError",
32+
};
33+
match imported(py, "sqlite3", class).and_then(|class| class.call1((error.to_string(),))) {
34+
Ok(error) => PyErr::from_value(error),
35+
Err(error) => error,
36+
}
37+
}
38+
39+
/// Parameters of history SQL contain no Python-owned values.
40+
pub(crate) fn sqlite_params(params: &[SqlParam]) -> PyResult<Vec<Value>> {
41+
params
42+
.iter()
43+
.map(|value| {
44+
Ok(match value {
45+
SqlParam::Null => Value::Null,
46+
SqlParam::Text(value) => Value::Text(value.clone()),
47+
SqlParam::Integer(value) => Value::Integer(*value),
48+
SqlParam::Flag(value) => Value::Integer(i64::from(*value)),
49+
SqlParam::Timestamp(_) | SqlParam::TextList(_) => {
50+
return Err(PyValueError::new_err("Postgres-only history parameter"));
51+
}
52+
})
53+
})
54+
.collect()
55+
}
56+
57+
#[pyclass(frozen, module = "sqlbuild._native", name = "_HistorySqliteConnection")]
58+
pub(crate) struct SqliteConnection {
59+
connection: Mutex<Option<Connection>>,
60+
}
61+
62+
impl SqliteConnection {
63+
/// Open literal filenames, using absolute paths because bundled SQLite enables URI handling globally.
64+
pub(crate) fn open(py: Python<'_>, path: &str) -> PyResult<Self> {
65+
let flags = OpenFlags::default() & !OpenFlags::SQLITE_OPEN_URI;
66+
let database: PathBuf = if path == MEMORY_PATH {
67+
PathBuf::from(path)
68+
} else {
69+
std::env::current_dir()?.join(path)
70+
};
71+
let connection = py
72+
.detach(|| Connection::open_with_flags(database, flags))
73+
.map_err(|error| database_error(py, error))?;
74+
Ok(Self {
75+
connection: Mutex::new(Some(connection)),
76+
})
77+
}
78+
79+
fn with_connection<T: Send>(
80+
&self,
81+
py: Python<'_>,
82+
body: impl FnOnce(&Connection) -> rusqlite::Result<T> + Send,
83+
) -> PyResult<T> {
84+
let result = py.detach(|| {
85+
let guard = self
86+
.connection
87+
.lock()
88+
.unwrap_or_else(PoisonError::into_inner);
89+
guard.as_ref().map(body)
90+
});
91+
match result {
92+
Some(result) => result.map_err(|error| database_error(py, error)),
93+
None => Err(PyErr::from_value(
94+
imported(py, "sqlite3", "ProgrammingError")?
95+
.call1(("Cannot operate on a closed database.",))?,
96+
)),
97+
}
98+
}
99+
100+
pub(crate) fn execute_native(
101+
&self,
102+
py: Python<'_>,
103+
sql: &str,
104+
params: &[Value],
105+
) -> PyResult<SqliteCursor> {
106+
self.with_connection(py, |connection| {
107+
let mut statement = connection.prepare_cached(sql)?;
108+
let columns = Arc::new(
109+
statement
110+
.column_names()
111+
.into_iter()
112+
.map(str::to_owned)
113+
.collect(),
114+
);
115+
let count: usize = statement.column_count();
116+
let mut cursor = statement.query(params_from_iter(params))?;
117+
let mut rows: VecDeque<SqliteRow> = VecDeque::new();
118+
while let Some(row) = cursor.next()? {
119+
rows.push_back(SqliteRow {
120+
columns: Arc::clone(&columns),
121+
values: (0..count)
122+
.map(|index| row.get(index))
123+
.collect::<rusqlite::Result<_>>()?,
124+
});
125+
}
126+
Ok(SqliteCursor {
127+
rows: Mutex::new(rows),
128+
last_row_id: connection.last_insert_rowid(),
129+
})
130+
})
131+
}
132+
}
133+
134+
#[pymethods]
135+
impl SqliteConnection {
136+
#[pyo3(signature = (sql, parameters = None))]
137+
fn execute(
138+
&self,
139+
py: Python<'_>,
140+
sql: &str,
141+
parameters: Option<&Bound<'_, PyAny>>,
142+
) -> PyResult<SqliteCursor> {
143+
let mut params: Vec<Value> = Vec::new();
144+
if let Some(parameters) = parameters {
145+
for value in parameters.try_iter()? {
146+
let value = value?;
147+
params.push(if value.is_none() {
148+
Value::Null
149+
} else if let Ok(value) = value.extract::<i64>() {
150+
Value::Integer(value)
151+
} else {
152+
Value::Text(value.extract()?)
153+
});
154+
}
155+
}
156+
self.execute_native(py, sql, &params)
157+
}
158+
159+
fn executemany(
160+
&self,
161+
py: Python<'_>,
162+
sql: &str,
163+
parameters: &Bound<'_, PyAny>,
164+
) -> PyResult<SqliteCursor> {
165+
let mut result = SqliteCursor {
166+
rows: Mutex::default(),
167+
last_row_id: 0,
168+
};
169+
for row in parameters.try_iter()? {
170+
result = self.execute(py, sql, Some(&row?))?;
171+
}
172+
Ok(result)
173+
}
174+
175+
#[getter]
176+
pub(crate) fn in_transaction(&self, py: Python<'_>) -> PyResult<bool> {
177+
self.with_connection(py, |connection| Ok(!connection.is_autocommit()))
178+
}
179+
180+
pub(crate) fn close(&self, py: Python<'_>) -> PyResult<()> {
181+
py.detach(|| {
182+
self.connection
183+
.lock()
184+
.unwrap_or_else(PoisonError::into_inner)
185+
.take();
186+
});
187+
Ok(())
188+
}
189+
}

0 commit comments

Comments
 (0)