diff --git a/crates/sqltide-python/src/sensors/mod.rs b/crates/sqltide-python/src/sensors/mod.rs index 696abd2c..ca626589 100644 --- a/crates/sqltide-python/src/sensors/mod.rs +++ b/crates/sqltide-python/src/sensors/mod.rs @@ -2,6 +2,7 @@ //! User handlers still run through the Python evaluation shim and receive the Python contexts. mod dispatch; +mod providers; mod queries; mod registry; mod repository; @@ -17,5 +18,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_class::()?; module.add_function(wrap_pyfunction!(dispatch::effective_sensor_status, module)?)?; + module.add_function(wrap_pyfunction!(providers::provider_name, module)?)?; registry::register(module) } diff --git a/crates/sqltide-python/src/sensors/providers.rs b/crates/sqltide-python/src/sensors/providers.rs new file mode 100644 index 00000000..9417dc9f --- /dev/null +++ b/crates/sqltide-python/src/sensors/providers.rs @@ -0,0 +1,401 @@ +//! Discover the project's provider classes under `providers/` and validate their settings. + +use pyo3::exceptions::{PyException, PyValueError}; +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyList, PyString, PyTuple, PyType}; + +use super::support::{chained_to_handled, text}; +use crate::dev_server::support::attr; +use crate::py_kwargs; + +const PROVIDERS_DIRECTORY_NAME: &str = "providers"; +const PYTHON_INIT_MODULE_STEM: &str = "__init__"; +const PROVIDER_EXCEPTIONS: &str = "sqltide.provider.exceptions"; +const PROVIDER_NAME_PATTERN: &str = "[a-z_][a-z0-9_]*"; + +type Any<'py> = Bound<'py, PyAny>; + +/// `ProviderDiscoveryError(message)`, raised `from cause` when one is given. +fn discovery_error(py: Python<'_>, message: String, cause: Option) -> PyErr { + let error: PyErr = match attr(py, PROVIDER_EXCEPTIONS, "ProviderDiscoveryError") + .and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => return error, + }; + let Some(cause) = cause else { + return chained_to_handled(py, error); + }; + let context: Any<'_> = cause.value(py).clone().into_any(); + error.set_cause(py, Some(cause)); + if let Err(failed) = error.value(py).setattr("__context__", context) { + return failed; + } + error +} + +/// `path.relative_to(project_dir)`. +fn relative<'py>(path: &Any<'py>, project_dir: &Any<'py>) -> PyResult> { + path.call_method1("relative_to", (project_dir,)) +} + +/// `Path(value).resolve()`. +fn resolved<'py>(value: &Any<'py>) -> PyResult> { + attr(value.py(), "pathlib", "Path")? + .call1((value,))? + .call_method0("resolve") +} + +/// Removes the modules of the project's root package that another project directory left in +/// `sys.modules`, so a second project imports its own package. +fn evict_stale_project_package_modules(root_module: &str, project_dir: &Any<'_>) -> PyResult<()> { + let py: Python<'_> = project_dir.py(); + let modules: Any<'_> = py.import("sys")?.getattr("modules")?; + let root_path: Any<'_> = project_dir.div(root_module)?.call_method0("resolve")?; + let prefix: String = format!("{root_module}."); + let items: Bound<'_, PyTuple> = PyTuple::new( + py, + modules + .call_method0("items")? + .try_iter()? + .collect::>>>()?, + )?; + for item in items.iter() { + let module_name: Any<'_> = item.get_item(0)?; + let module: Any<'_> = item.get_item(1)?; + if module_name.ne(root_module)? + && !module_name + .call_method1("startswith", (prefix.as_str(),))? + .is_truthy()? + { + continue; + } + let evict = || modules.call_method1("pop", (&module_name, py.None())); + if let Some(module_file) = module.getattr_opt("__file__")? + && module_file.is_instance_of::() + { + let inside = || -> PyResult> { + resolved(&module_file)? + .call_method1("relative_to", (project_dir.call_method0("resolve")?,)) + }; + match inside() { + Ok(_) => {} + Err(error) if error.is_instance_of::(py) => { + evict()?; + } + Err(error) => return Err(error), + } + continue; + } + let Some(module_paths) = module + .getattr_opt("__path__")? + .filter(|paths| !paths.is_none()) + else { + evict()?; + continue; + }; + let mut in_root: bool = false; + for path in module_paths.try_iter()? { + if resolved(&path?)?.eq(&root_path)? { + in_root = true; + break; + } + } + if !in_root { + evict()?; + } + } + Ok(()) +} + +/// Imports one provider file as `.` with the project directory on `sys.path`, +/// reusing the module already loaded from the same file. +fn load_provider_module<'py>(file_path: &Any<'py>, project_dir: &Any<'py>) -> PyResult> { + let py: Python<'py> = file_path.py(); + let parts: Any<'py> = relative(file_path, project_dir)? + .call_method1("with_suffix", ("",))? + .getattr("parts")?; + let module_name: String = "." + .into_pyobject(py)? + .call_method1("join", (parts,))? + .extract()?; + let root_module: &str = module_name.split('.').next().unwrap_or_default(); + evict_stale_project_package_modules(root_module, project_dir)?; + let sys: Bound<'py, PyModule> = py.import("sys")?; + let modules: Any<'py> = sys.getattr("modules")?; + let existing: Any<'py> = modules.call_method1("get", (&module_name,))?; + if !existing.is_none() { + if let Some(existing_file) = existing.getattr_opt("__file__")? + && existing_file.is_instance_of::() + && resolved(&existing_file)?.eq(file_path.call_method0("resolve")?)? + { + return Ok(existing); + } + modules.call_method1("pop", (&module_name, py.None()))?; + } + let util: Bound<'py, PyModule> = py.import("importlib.util")?; + let spec: Any<'py> = util.call_method1("spec_from_file_location", (&module_name, file_path))?; + if spec.is_none() || spec.getattr("loader")?.is_none() { + return Err(discovery_error( + py, + format!("Could not load provider file {}", text(file_path)?), + None, + )); + } + let module: Any<'py> = util.call_method1("module_from_spec", (&spec,))?; + let old_path: Bound<'py, PyList> = PyList::new( + py, + sys.getattr("path")? + .try_iter()? + .collect::>>>()?, + )?; + sys.getattr("path")? + .call_method1("insert", (0, project_dir.str()?))?; + let loaded: PyResult<()> = (|| { + modules.set_item(&module_name, &module)?; + spec.getattr("loader")? + .call_method1("exec_module", (&module,))?; + Ok(()) + })(); + let restored: PyResult<()> = sys.setattr("path", old_path); + let loaded: PyResult<()> = match loaded { + Err(error) if error.is_instance_of::(py) => { + let message: PyResult = (|| { + Ok(format!( + "Failed to import provider file {}: {}", + text(&relative(file_path, project_dir)?)?, + text(error.value(py).as_any())? + )) + })(); + match message { + Ok(message) => Err(discovery_error(py, message, Some(error))), + Err(failed) => { + let failure: &Bound<'_, pyo3::exceptions::PyBaseException> = failed.value(py); + if failure.getattr("__context__")?.is_none() { + failure.setattr("__context__", error.value(py))?; + } + Err(failed) + } + } + } + outcome => outcome, + }; + loaded?; + restored?; + Ok(module) +} + +/// `provider_class.name()`, reporting an invalid name with the class and its file. +fn discovered_name<'py>( + provider_class: &Any<'py>, + file_path: &Any<'py>, + project_dir: &Any<'py>, +) -> PyResult> { + let py: Python<'py> = provider_class.py(); + match provider_class.call_method0("name") { + Err(error) + if error.is_instance(py, &attr(py, PROVIDER_EXCEPTIONS, "ProviderInputError")?) => + { + let message: String = format!( + "Provider class {} in {} has an invalid provider name: {}", + text(&provider_class.getattr("__name__")?)?, + text(&relative(file_path, project_dir)?)?, + text(error.value(py).as_any())? + ); + Err(discovery_error(py, message, Some(error))) + } + outcome => outcome, + } +} + +/// Each pydantic validation error as `location: message [type]`, one per line. +fn format_provider_validation_error(error: &Any<'_>) -> PyResult { + let py: Python<'_> = error.py(); + let mut details: Vec = Vec::new(); + for item in error + .call_method("errors", (), Some(&py_kwargs!(py; include_input = false)?))? + .try_iter()? + { + let item: Any<'_> = item?; + let location: Any<'_> = item.call_method1("get", ("loc", PyTuple::empty(py)))?; + let location_text: String = if location.is_truthy()? { + location + .try_iter()? + .map(|part| text(&part?)) + .collect::>>()? + .join(".") + } else { + "".to_owned() + }; + let message: Any<'_> = item.call_method1("get", ("msg", "invalid value"))?; + let error_type: Any<'_> = item.call_method1("get", ("type", "validation_error"))?; + details.push(format!( + "{location_text}: {} [{}]", + text(&message)?, + text(&error_type)? + )); + } + if details.is_empty() { + return Ok("invalid provider settings".to_owned()); + } + Ok(details.join("\n")) +} + +/// `provider_class()`, reporting invalid settings with the provider name and file. +fn provider_instance<'py>( + provider_class: &Any<'py>, + provider_name: &Any<'py>, + file_path: &Any<'py>, + project_dir: &Any<'py>, +) -> PyResult> { + let py: Python<'py> = provider_class.py(); + match provider_class.call0() { + Err(error) if error.is_instance(py, &attr(py, "pydantic", "ValidationError")?) => { + let message: String = format!( + "Provider '{}' in {} has invalid settings:\n{}", + text(provider_name)?, + text(&relative(file_path, project_dir)?)?, + format_provider_validation_error(error.value(py).as_any())? + ); + Err(discovery_error(py, message, Some(error))) + } + outcome => outcome, + } +} + +/// The concrete `Provider` subclasses each public `providers/` module defines, in file then class +/// name order, as `DiscoveredProvider` records with validated settings. +pub(super) fn discover_provider_classes<'py>(project_dir: &Any<'py>) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let providers_root: Any<'py> = project_dir.div(PROVIDERS_DIRECTORY_NAME)?; + if !providers_root.call_method0("is_dir")?.is_truthy()? { + return Ok(PyTuple::empty(py).into_any()); + } + let provider: Any<'py> = attr(py, "sqltide.providers", "Provider")?; + let provider_type: &Bound<'py, PyType> = provider.cast()?; + let discovered_class: Any<'py> = attr(py, "sqltide.provider.models", "DiscoveredProvider")?; + let is_abstract: Any<'py> = attr(py, "inspect", "isabstract")?; + let seen_names: Bound<'py, PyDict> = PyDict::new(py); + let mut discovered: Vec> = Vec::new(); + let sorted: Any<'py> = py + .import("builtins")? + .getattr("sorted")? + .call1((providers_root.call_method1("rglob", ("*.py",))?,))?; + for file_path in sorted.try_iter()? { + let file_path: Any<'py> = file_path?; + if file_path.getattr("stem")?.eq(PYTHON_INIT_MODULE_STEM)? + || file_path + .getattr("name")? + .call_method1("startswith", ("_",))? + .is_truthy()? + { + continue; + } + let module: Any<'py> = load_provider_module(&file_path, project_dir)?; + let module_name: Any<'py> = module.getattr("__name__")?; + let members: Any<'py> = + attr(py, "inspect", "getmembers")?.call1((&module, attr(py, "inspect", "isclass")?))?; + for member in members.try_iter()? { + let value: Any<'py> = member?.get_item(1)?; + if value.getattr("__module__")?.ne(&module_name)? { + continue; + } + if value.is(&provider) + || !value.cast::()?.is_subclass(provider_type)? + || is_abstract.call1((&value,))?.is_truthy()? + { + continue; + } + let name: Any<'py> = discovered_name(&value, &file_path, project_dir)?; + if let Some(existing_path) = seen_names.get_item(&name)? { + return Err(discovery_error( + py, + format!( + "Duplicate provider name '{}' found in {} and {}", + text(&name)?, + text(&relative(&existing_path, project_dir)?)?, + text(&relative(&file_path, project_dir)?)? + ), + None, + )); + } + seen_names.set_item(&name, &file_path)?; + let relative_path: Any<'py> = relative(&file_path, project_dir)?; + let settings: Any<'py> = provider_instance(&value, &name, &file_path, project_dir)?; + discovered.push(discovered_class.call( + (), + Some(&py_kwargs!(py; + file_path = &file_path, + relative_path = relative_path, + name = name, + provider_class = &value, + settings = settings, + )?), + )?); + } + } + Ok(PyTuple::new(py, discovered)?.into_any()) +} + +/// `ProviderInputError(message)`. +fn input_error(py: Python<'_>, message: String) -> PyResult { + Ok(PyErr::from_value( + attr(py, PROVIDER_EXCEPTIONS, "ProviderInputError")?.call1((message,))?, + )) +} + +/// The class name in snake case: an underscore before every ASCII capital after the first +/// character, then lowercased. +fn snake_case(class_name: &str) -> String { + let mut snake: String = String::with_capacity(class_name.len() + 4); + for (index, character) in class_name.chars().enumerate() { + if index > 0 && character.is_ascii_uppercase() { + snake.push('_'); + } + snake.push(character); + } + snake.to_lowercase() +} + +/// Whether `name` is a lower snake_case Python identifier. +fn is_provider_name(name: &str) -> bool { + let mut characters = name.chars(); + characters + .next() + .is_some_and(|first| first.is_ascii_lowercase() || first == '_') + && characters.all(|rest| rest.is_ascii_lowercase() || rest.is_ascii_digit() || rest == '_') +} + +/// The runtime name of a provider class: its `provider_name`, or its class name in snake case. +#[pyfunction] +pub(super) fn provider_name<'py>(provider_class: Any<'py>) -> PyResult> { + let py: Python<'py> = provider_class.py(); + let class_name: Any<'py> = provider_class.getattr("__name__")?; + let mut raw_name: Any<'py> = provider_class.getattr("provider_name")?; + if raw_name.is_none() { + raw_name = + PyString::new(py, &snake_case(class_name.cast::()?.to_str()?)).into_any(); + } + if !raw_name.is_truthy()? { + return Err(input_error( + py, + format!("provider {} resolved to an empty name", text(&class_name)?), + )?); + } + let Ok(name) = raw_name.cast::() else { + attr(py, "re", "fullmatch")?.call1((PROVIDER_NAME_PATTERN, &raw_name))?; + return Ok(raw_name); + }; + if !is_provider_name(name.to_str()?) { + return Err(input_error( + py, + format!( + "provider {} has invalid provider name '{}'; provider names must be lower \ + snake_case Python identifiers", + text(&class_name)?, + text(&raw_name)? + ), + )?); + } + Ok(raw_name) +} diff --git a/crates/sqltide-python/src/sensors/registry.rs b/crates/sqltide-python/src/sensors/registry.rs index e92c4d8d..95243655 100644 --- a/crates/sqltide-python/src/sensors/registry.rs +++ b/crates/sqltide-python/src/sensors/registry.rs @@ -6,7 +6,8 @@ use pyo3::prelude::*; use pyo3::types::{PyCFunction, PyDict, PyFrozenSet, PyList, PyTuple}; use super::dispatch::event_source_for; -use super::support::{SENSOR_EXCEPTIONS, SENSOR_MODELS, SENSOR_TYPES, text}; +use super::providers::discover_provider_classes; +use super::support::{SENSOR_EXCEPTIONS, SENSOR_MODELS, SENSOR_TYPES, chained_to_handled, text}; use crate::dev_server::support::attr; use crate::py_kwargs; @@ -350,12 +351,7 @@ fn compile_sensors<'py>( ) -> PyResult>> { let py: Python<'py> = project_dir.py(); let sensor_paths: Vec> = discover_sensor_paths(&project_dir)?; - let providers: Any<'py> = attr( - py, - "sqltide.provider.main._discover_provider_classes", - "discover_provider_classes", - )? - .call((), Some(&py_kwargs!(py; project_dir = &project_dir)?))?; + let providers: Any<'py> = discover_provider_classes(&project_dir)?; if sensor_paths.is_empty() && !providers.is_truthy()? { return Ok(None); } @@ -582,8 +578,35 @@ fn polling_sensor<'py>( ) } +/// Reject a `SensorRetryPolicy` without an attempt or with a negative backoff. +#[pyfunction] +#[pyo3(signature = (*, max_attempts, backoff_seconds))] +fn validate_sensor_retry_policy(max_attempts: Any<'_>, backoff_seconds: Any<'_>) -> PyResult<()> { + let py: Python<'_> = max_attempts.py(); + if max_attempts.lt(1)? { + return Err(chained_to_handled( + py, + plain_error( + py, + "SensorRetryPolicy.max_attempts must be at least 1".to_owned(), + ), + )); + } + if backoff_seconds.lt(0)? { + return Err(chained_to_handled( + py, + plain_error( + py, + "SensorRetryPolicy.backoff_seconds must not be negative".to_owned(), + ), + )); + } + Ok(()) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(compile_sensors, module)?)?; + module.add_function(wrap_pyfunction!(validate_sensor_retry_policy, module)?)?; module.add_function(wrap_pyfunction!(event_sensor, module)?)?; module.add_function(wrap_pyfunction!(polling_sensor, module)?) } diff --git a/crates/sqltide-python/src/sensors/support.rs b/crates/sqltide-python/src/sensors/support.rs index 1d084717..a16fd10d 100644 --- a/crates/sqltide-python/src/sensors/support.rs +++ b/crates/sqltide-python/src/sensors/support.rs @@ -15,6 +15,23 @@ pub(crate) const SENSOR_MODELS: &str = "sqltide.sensors.models"; pub(crate) const SENSOR_TYPES: &str = "sqltide.sensors.types"; pub(crate) const SENSOR_EXCEPTIONS: &str = "sqltide.sensors.exceptions"; +/// Chain `error` to the exception being handled, as a Python `raise` inside an `except` does. +pub(crate) fn chained_to_handled(py: Python<'_>, error: PyErr) -> PyErr { + let handled: Bound<'_, PyAny> = match py + .import("sys") + .and_then(|sys| sys.call_method0("exception")) + { + Ok(handled) => handled, + Err(failed) => return failed, + }; + if !handled.is_none() + && let Err(failed) = error.value(py).setattr("__context__", handled) + { + return failed; + } + error +} + /// `str(value)`. pub(crate) fn text(value: &Bound<'_, PyAny>) -> PyResult { Ok(value.str()?.to_string()) diff --git a/crates/sqltide-python/src/shapes/mod.rs b/crates/sqltide-python/src/shapes/mod.rs index 45a49d98..8e52325e 100644 --- a/crates/sqltide-python/src/shapes/mod.rs +++ b/crates/sqltide-python/src/shapes/mod.rs @@ -20,6 +20,7 @@ mod compiler_analysis; mod compiler_project; mod executor_ops; mod planner_execution; +mod sensors_dev; /// The callable a `default_factory` field calls for each new instance. pub(crate) type Factory = for<'py> fn(Python<'py>) -> PyResult>; @@ -868,6 +869,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { planner_execution::register(module)?; executor_ops::register(module)?; cli_service::register(module)?; + sensors_dev::register(module)?; lazy_constants( module, &[ diff --git a/crates/sqltide-python/src/shapes/sensors_dev.rs b/crates/sqltide-python/src/shapes/sensors_dev.rs new file mode 100644 index 00000000..22f346e6 --- /dev/null +++ b/crates/sqltide-python/src/shapes/sensors_dev.rs @@ -0,0 +1,350 @@ +//! Internal shapes of the sensor runtime, the event catalog's observation rows and the dev +//! server, plus the sensor value enums. + +use pyo3::prelude::*; +use pyo3::types::{PyBytes, PyDict, PyString, PyTuple}; +use sha2::{Digest, Sha256}; + +use super::{Coerce, Given, add_class, factory, path, shapes, str_enum}; + +shapes! { + module = "sqltide.sensors.models"; + + /// One registered authored sensor with retained source identity. + LoadedSensor { + name, + kind, + declaration, + file_path, + relative_path, + source, + definition_line, + description(default = Given::None), + timeout_seconds(default = Given::Float(60.0)), + } + @impl { + /// Fingerprint the sensor identity and retained implementation source. + #[getter] + fn identity_fingerprint(&self, py: Python<'_>) -> PyResult { + // An f-string formats each part with `format(part, "")`, which a `str` subclass may + // override independently of `__str__`. + let format: Bound<'_, PyAny> = py.import("builtins")?.getattr("format")?; + let parts: [Bound<'_, PyAny>; 3] = [ + format.call1((self.name.bind(py),))?, + format.call1((self.relative_path.bind(py).call_method0("as_posix")?,))?, + format.call1((self.source.bind(py),))?, + ]; + let payload: Bound<'_, PyAny> = PyString::new(py, "\0") + .call_method1("join", (parts,))? + .call_method1("encode", ("utf-8",))?; + Ok(Sha256::digest(payload.cast::()?.as_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect()) + } + + #[getter] + fn default_status(&self, py: Python<'_>) -> PyResult> { + path(py, &self.declaration, &["default_status"]) + } + + #[getter] + fn retry_policy(&self, py: Python<'_>) -> PyResult> { + path(py, &self.declaration, &["retry_policy"]) + } + } + + /// Compiled project automation: loaded sensors plus discovered providers. + CompiledSensors { registry, providers(default = Given::EmptyTuple) } + + /// One (completed_at, id) position in an ordered observation stream. + SensorStreamPosition { completed_at, result_id } + + /// Reduced durable-step state for one (sensor, event, key). + StepMarker { status, result_json, attempt } + + /// Reduced tick history for one (sensor, event). + TickAttemptState { failed_attempts, last_failed_at, last_error_message, resolved } + + /// One reduced tick with its terminal state preferred over started rows. + SensorTickView { + tick_id, + sensor_name, + definition_fingerprint, + kind, + event_id, + event_kind, + attempt, + status, + started_at, + completed_at, + error_message, + skip_reason, + cursor, + } + + /// Latest polling tick timing and cursor for one sensor. + PollingTickState { last_started_at, last_success_at, cursor } + + /// Terminal outcome of one isolated sensor handler invocation. + SensorEvaluation { + status, + skip_reason(default = Given::None), + error_message(default = Given::None), + cursor(default = Given::None), + } + + /// Resolution of one matched event delivery within a dispatch pass. + SensorDeliveryOutcome { + resolved, + evaluation(default = Given::None), + dead_lettered(default = Given::Bool(false)), + } + + /// Events derived from one positioned observation row. + SensorEventDelivery { position, events } + + /// Counters describing one dispatch pass. + SensorDispatchSummary { + lease_acquired(default = Given::Bool(true)), + evaluated(default = Given::Int(0)), + succeeded(default = Given::Int(0)), + skipped(default = Given::Int(0)), + failed(default = Given::Int(0)), + dead_lettered(default = Given::Int(0)), + } + + /// Deterministic name-keyed registry of loaded project sensors. + SensorRegistry { + sensors(default = Given::Factory(factory::), coerce = Coerce::ReadOnlyMapping), + } + @impl { + fn ordered<'py>(&self, py: Python<'py>) -> PyResult> { + let sensors: &Bound<'py, PyAny> = self.sensors.bind(py); + let names: Bound<'py, PyAny> = py + .import("builtins")? + .getattr("sorted")? + .call1((sensors,))?; + let mut ordered: Vec> = Vec::new(); + for name in names.try_iter()? { + ordered.push(sensors.get_item(name?)?); + } + PyTuple::new(py, ordered) + } + } +} + +shapes! { + module = "sqltide.events.models"; + + /// One persisted quality node result row viewed by the event catalog. + NodeResultObservation { + result_id, + invocation_id, + node_kind, + node_name, + binding_key, + target_identity, + trigger, + status, + severity, + failure_count, + completed_at, + scheduled_for, + payload_json, + error_message, + } + + /// One persisted terminal invocation row viewed by the event catalog. + InvocationObservation { + invocation_id, + command, + mode, + outcome, + exit_code, + target_identity, + deployment_id, + selected_node_count, + error_message, + completed_at, + } +} + +shapes! { + module = "sqltide.dev_server.models"; + + /// Approximate progress derived from committed target offset frontiers. + ReplayOffsetProgress { + percentage, + eta_seconds, + completed_span, + total_span, + observed_partitions, + total_partitions, + } + + /// Resolved invocation values retained across dev reloads, previews, and builds. + DevExecutionContext @no_repr { + database(default = Given::None), + selected_target(default = Given::None), + cli_variables(default = Given::EmptyTuple), + environment(default = Given::None), + connection_host(default = Given::None), + connection_port(default = Given::None), + connection_username(default = Given::None), + connection_password(default = Given::None), + run_presumed_failed_after_seconds(default = Given::Int(600)), + connection_factory(default = Given::None), + observation_connection_factory(default = Given::None), + } + + /// A compile failure reduced to what an error page needs. + CompileErrorInfo { + message, + path(default = Given::None), + line(default = Given::None), + column(default = Given::None), + end_line(default = Given::None), + end_column(default = Given::None), + } + + /// One held project compile: either servable definitions or the failure. + CompileOutcome { + state, + version_key, + compiled_at, + analysis(default = Given::None), + timings(default = Given::None), + error(default = Given::None), + } + + /// Shared control-plane inputs for route-level operation authorization. + OperationAuthorizationContext { store, project_dir, selected_target } + + /// Durable control-plane stores and their application ownership. + DevControlStores { accounts, destruction_plans, owns_accounts } + @impl { + fn close(&self, py: Python<'_>) -> PyResult<()> { + self.destruction_plans.bind(py).call_method0("close")?; + if self.owns_accounts.bind(py).is_truthy()? { + self.accounts.bind(py).call_method0("close")?; + } + Ok(()) + } + } + + /// Broker metadata for one topic. + KafkaTopicInfo { name, partition_count, replication_factor, internal } + + /// Best-effort topic inventory for one broker list. + KafkaTopicsSnapshot { topics } + + /// Broker offsets and lag for one topic partition. + KafkaPartitionLag { partition, committed_offset, end_offset, lag_messages } + + /// Best-effort consumer-group lag for one managed Kafka source. + KafkaLagSnapshot { total_messages, partitions } + + /// Row and byte totals for one warehouse relation. + RelationStorage { rows(default = Given::Int(0)), bytes(default = Given::Int(0)) } + + /// Terminal evidence from one server-owned build cancellation attempt. + BuildCancellationOutcome { + invocation_id, + query_id, + requested_at, + status, + process_exit_code, + warehouse_supported, + query_found, + warehouse_termination_confirmed, + error_message(default = Given::None), + } +} + +fn enums(module: &Bound<'_, PyModule>) -> PyResult<()> { + const SENSOR_TYPES: &str = "sqltide.sensors.types"; + str_enum( + module, + "SensorKind", + SENSOR_TYPES, + "SensorKind", + Some("Kinds of authored sensors."), + &[("EVENT", "event"), ("POLLING", "polling")], + )?; + str_enum( + module, + "DefaultSensorStatus", + SENSOR_TYPES, + "DefaultSensorStatus", + Some("Authored default sensor status declared in code."), + &[("RUNNING", "running"), ("STOPPED", "stopped")], + )?; + str_enum( + module, + "SensorOverrideStatus", + SENSOR_TYPES, + "SensorOverrideStatus", + Some("Runtime override state persisted by operator actions."), + &[ + ("RUNNING", "running"), + ("STOPPED", "stopped"), + ("DECLARED_IN_CODE", "declared_in_code"), + ], + )?; + str_enum( + module, + "SensorTickStatus", + SENSOR_TYPES, + "SensorTickStatus", + Some("Persisted status of one sensor tick attempt."), + &[ + ("STARTED", "started"), + ("SKIPPED", "skipped"), + ("SUCCEEDED", "succeeded"), + ("FAILED", "failed"), + ("DEAD_LETTERED", "dead_lettered"), + ], + )?; + str_enum( + module, + "StepPolicy", + SENSOR_TYPES, + "StepPolicy", + Some("Durable step re-execution policy."), + &[ + ("AT_LEAST_ONCE", "at_least_once"), + ("AT_MOST_ONCE", "at_most_once"), + ], + ) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + enums(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module) +} diff --git a/python-boundary.toml b/python-boundary.toml index 795fc1a4..ff6e0b9a 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -8,7 +8,7 @@ # still carries planning, validation, rendering or other movable logic counts as movable until # that logic moves to Rust. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 2685 +remaining_lines = 1774 [[stays]] path = "adapter/classes/adapter.py" @@ -105,6 +105,11 @@ path = "events/__init__.py" rule = 3 note = "Rule 3: Public sqltide.events facade that documented sensors import; re-exports only." +[[stays]] +path = "events/models.py" +rule = 3 +note = "Rule 3: AuditCompleted and RunCompleted, the event classes sqltide.events re-exports, that user sensors pass to event_sensor(on=...) and receive as ctx.event, that native events_from_node_result and events_from_invocation construct, and that native dispatch matches by identity, stay Python frozen dataclasses; data only. NodeResultObservation and InvocationObservation are native re-exports." + [[stays]] path = "events/types.py" rule = 3 @@ -185,11 +190,6 @@ path = "provider/main/build_provider_session.py" rule = 2 note = "Rule 2: Instantiates user Provider subclasses into a session." -[[stays]] -path = "provider/main/_discover_provider_classes.py" -rule = 2 -note = "Rule 2: Entry point of the shim that imports project provider modules." - [[stays]] path = "provider/main/invoke_with_providers.py" rule = 2 @@ -200,6 +200,11 @@ path = "provider/models.py" rule = 2 note = "Rule 2: Record holding a live user Provider class and its settings instance." +[[stays]] +path = "providers.py" +rule = 3 +note = "Rule 3: The documented Provider base class that project providers subclass, built on pydantic-settings BaseSettings with its model config and the no-op setup and teardown hooks the rule-2 provider session calls; name() returns the native provider_name for the class, which native provider discovery reads." + [[stays]] path = "sensors/__init__.py" rule = 3 @@ -220,12 +225,27 @@ path = "sensors/classes/polling_sensor_context.py" rule = 3 note = "Rule 3: PollingSensorContext that user polling sensors receive, re-exported by sqltide.sensors; read-only properties." +[[stays]] +path = "sensors/models.py" +rule = 3 +note = "Rule 3: SensorRetryPolicy, SkipReason and PollingSensorResult, which user sensors construct and return and sqltide.sensors re-exports (the rule-2 evaluation shim matches the returned SkipReason and PollingSensorResult), and EventSensorDeclaration and PollingSensorDeclaration, which the native event_sensor and polling_sensor decorators bind in place of the user function and whose __call__ forwards to it, stay Python frozen dataclasses; SensorRetryPolicy validation runs in the native validate_sensor_retry_policy. The other names here are native re-exports of the sensor runtime shapes." + +[[stays]] +path = "sensors/types.py" +rule = 1 +note = "Rule 1: The data-only alias SensorFunction, the live user sensor function the rule-3 EventSensorDeclaration and PollingSensorDeclaration in sensors/models.py hold and the native event_sensor and polling_sensor decorators wrap; the data-only alias SensorDeclaration, the union of those two declarations that native LoadedSensor holds; and the logic-free SensorStepStore protocol for the store the native DurableStepRunner takes, implemented by native InMemoryStepStore and RepositoryStepStore. The sensor enums here are native re-exports; no logic." + [[stays]] path = "auth/models.py" rule = 4 note = "Rule 4: pydantic request bodies LoginRequest, CreateUserRequest, UpdateUserRequest, PasswordResetRequest, RoleRequest and ProjectRoleRequest, which the native dev server validates with pydantic for the account routes. They hold only field declarations with Field constraints and defaults and no custom validators or model config; the other names here are native re-exports of the account shapes." +[[stays]] +path = "dev_server/models.py" +rule = 4 +note = "Rule 4: pydantic request bodies ChecksRunRequest, AuditBatchRunRequest, BuildRunRequest, MessageQueryMode, MessageQueryPredicate, MessageQueryCursor, MessagesQueryRequest, MessageRecordRequest, MessageFacetsRequest, DeploymentCleanupRequest, DestructionPlanRequest, DestructionExecutionRequest, SensorStatusRequest and SensorDeadLetterActionRequest, which the native dev server routes validate with pydantic. They hold only field declarations with Field constraints and defaults, a shared MessageQueryMode() default and model_config extra=\"forbid\" on the two destruction requests, and no custom validators; the other names here are native re-exports of the dev server shapes." + [[stays]] path = "dev_server/types.py" rule = 1 diff --git a/src/sqltide/__init__.py b/src/sqltide/__init__.py index f978be49..e69de29b 100644 --- a/src/sqltide/__init__.py +++ b/src/sqltide/__init__.py @@ -1 +0,0 @@ -"""SQLTide package.""" diff --git a/src/sqltide/_native.pyi b/src/sqltide/_native.pyi index 8c46433a..62ae6a84 100644 --- a/src/sqltide/_native.pyi +++ b/src/sqltide/_native.pyi @@ -110,23 +110,15 @@ from sqltide.compiler.discovery.models import ( from sqltide.compiler.macros.models import MacroContext from sqltide.compiler.macros.types import MacroFunction from sqltide.dev_server.models import ( - BuildCancellationOutcome, - CompileErrorInfo, - CompileOutcome, - DevExecutionContext, - KafkaLagSnapshot, - KafkaTopicsSnapshot, MessageFacetsRequest, MessagesQueryRequest, - ReplayOffsetProgress, ) from sqltide.dev_server.types import CompileAuthorizationGuard, DevServerReporter from sqltide.events.models import ( AuditCompleted, - InvocationObservation, - NodeResultObservation, RunCompleted, ) +from sqltide.events.types import SensorEvent from sqltide.executor.destruction.types import ( DestructionClock, DestructionPlanningConnection, @@ -142,27 +134,16 @@ from sqltide.executor.workflow.types import ( WorkflowPhase, ) from sqltide.provider.models import DiscoveredProvider +from sqltide.providers import Provider from sqltide.sensors.models import ( - CompiledSensors, EventSensorDeclaration, - LoadedSensor, PollingSensorDeclaration, - PollingTickState, - SensorDispatchSummary, - SensorRegistry, SensorRetryPolicy, - SensorStreamPosition, - SensorTickView, - StepMarker, - TickAttemptState, ) from sqltide.sensors.types import ( - DefaultSensorStatus, + SensorDeclaration, SensorFunction, - SensorOverrideStatus, SensorStepStore, - SensorTickStatus, - StepPolicy, ) # Compiler analysis shapes: SQL analysis, graph, Rules, quality, audit and test discovery. @@ -3032,6 +3013,277 @@ CANCEL_GRACE_SECONDS: float TERMINATE_GRACE_SECONDS: float ACTIVITY_WINDOW_SECONDS: int +# Sensor runtime, event observation and dev server shapes. + +class SensorKind(StrEnum): + EVENT = "event" + POLLING = "polling" + +class DefaultSensorStatus(StrEnum): + RUNNING = "running" + STOPPED = "stopped" + +class SensorOverrideStatus(StrEnum): + RUNNING = "running" + STOPPED = "stopped" + DECLARED_IN_CODE = "declared_in_code" + +class SensorTickStatus(StrEnum): + STARTED = "started" + SKIPPED = "skipped" + SUCCEEDED = "succeeded" + FAILED = "failed" + DEAD_LETTERED = "dead_lettered" + +class StepPolicy(StrEnum): + AT_LEAST_ONCE = "at_least_once" + AT_MOST_ONCE = "at_most_once" + +@final +@dataclass(frozen=True) +class LoadedSensor: + name: str + kind: SensorKind + declaration: SensorDeclaration + file_path: Path + relative_path: Path + source: str + definition_line: int + description: str | None = None + timeout_seconds: float = ... + @property + def identity_fingerprint(self) -> str: ... + @property + def default_status(self) -> DefaultSensorStatus: ... + @property + def retry_policy(self) -> SensorRetryPolicy: ... + +@final +@dataclass(frozen=True) +class CompiledSensors: + registry: SensorRegistry + providers: tuple[DiscoveredProvider, ...] = () + +@final +@dataclass(frozen=True) +class SensorStreamPosition: + completed_at: str + result_id: str + +@final +@dataclass(frozen=True) +class StepMarker: + status: str + result_json: str | None + attempt: int + +@final +@dataclass(frozen=True) +class TickAttemptState: + failed_attempts: int + last_failed_at: str | None + last_error_message: str | None + resolved: bool + +@final +@dataclass(frozen=True) +class SensorTickView: + tick_id: str + sensor_name: str + definition_fingerprint: str + kind: str + event_id: str | None + event_kind: str | None + attempt: int + status: str + started_at: str + completed_at: str | None + error_message: str | None + skip_reason: str | None + cursor: str | None + +@final +@dataclass(frozen=True) +class PollingTickState: + last_started_at: str | None + last_success_at: str | None + cursor: str | None + +@final +@dataclass(frozen=True) +class SensorEvaluation: + status: SensorTickStatus + skip_reason: str | None = None + error_message: str | None = None + cursor: str | None = None + +@final +@dataclass(frozen=True) +class SensorDeliveryOutcome: + resolved: bool + evaluation: SensorEvaluation | None = None + dead_lettered: bool = False + +@final +@dataclass(frozen=True) +class SensorEventDelivery: + position: SensorStreamPosition + events: tuple[SensorEvent, ...] + +@final +@dataclass(frozen=True) +class SensorDispatchSummary: + lease_acquired: bool = True + evaluated: int = 0 + succeeded: int = 0 + skipped: int = 0 + failed: int = 0 + dead_lettered: int = 0 + +@final +@dataclass(frozen=True) +class SensorRegistry: + sensors: Mapping[str, LoadedSensor] = ... + def ordered(self) -> tuple[LoadedSensor, ...]: ... + +@final +@dataclass(frozen=True) +class NodeResultObservation: + result_id: str + invocation_id: str + node_kind: QualityNodeKind + node_name: str + binding_key: str + target_identity: str + trigger: str + status: QualityResultStatus + severity: str | None + failure_count: int + completed_at: str + scheduled_for: str | None + payload_json: str + error_message: str | None + +@final +@dataclass(frozen=True) +class InvocationObservation: + invocation_id: str + command: str + mode: str | None + outcome: str + exit_code: int + target_identity: str + deployment_id: str | None + selected_node_count: int + error_message: str | None + completed_at: str + +@final +@dataclass(frozen=True) +class ReplayOffsetProgress: + percentage: float + eta_seconds: float | None + completed_span: int + total_span: int + observed_partitions: int + total_partitions: int + +@final +@dataclass(frozen=True, repr=False) +class DevExecutionContext: + database: str | None = None + selected_target: str | None = None + cli_variables: tuple[tuple[str, object], ...] = () + environment: Mapping[str, str] | None = None + connection_host: str | None = None + connection_port: int | None = None + connection_username: str | None = None + connection_password: str | None = None + run_presumed_failed_after_seconds: int = ... + connection_factory: Callable[[], AdapterConnection] | None = None + observation_connection_factory: Callable[[], AdapterConnection] | None = None + +@final +@dataclass(frozen=True) +class CompileErrorInfo: + message: str + path: str | None = None + line: int | None = None + column: int | None = None + end_line: int | None = None + end_column: int | None = None + +@final +@dataclass(frozen=True) +class CompileOutcome: + state: CompileStateKind + version_key: str + compiled_at: str + analysis: CompileAnalysis | None = None + timings: CompilationTimings | None = None + error: CompileErrorInfo | None = None + +@final +@dataclass(frozen=True) +class OperationAuthorizationContext: + store: ControlStore + project_dir: Path + selected_target: str | None + +@final +@dataclass(frozen=True) +class DevControlStores: + accounts: ControlStore + destruction_plans: RelationalDestructionPlanStore + owns_accounts: bool + def close(self) -> None: ... + +@final +@dataclass(frozen=True) +class KafkaTopicInfo: + name: str + partition_count: int + replication_factor: int + internal: bool + +@final +@dataclass(frozen=True) +class KafkaTopicsSnapshot: + topics: tuple[KafkaTopicInfo, ...] + +@final +@dataclass(frozen=True) +class KafkaPartitionLag: + partition: int + committed_offset: int | None + end_offset: int + lag_messages: int | None + +@final +@dataclass(frozen=True) +class KafkaLagSnapshot: + total_messages: int | None + partitions: tuple[KafkaPartitionLag, ...] + +@final +@dataclass(frozen=True) +class RelationStorage: + rows: int = 0 + bytes: int = 0 + +@final +@dataclass(frozen=True) +class BuildCancellationOutcome: + invocation_id: str + query_id: str | None + requested_at: str + status: Literal["cancelled", "cancellation_failed"] + process_exit_code: int + warehouse_supported: bool | None + query_found: bool | None + warehouse_termination_confirmed: bool | None + error_message: str | None = None + type _Span = tuple[int, int, int, int, int, int] class SqlScanError(ValueError): @@ -3107,6 +3359,8 @@ def polling_sensor( def effective_sensor_status( *, sensor: LoadedSensor, overrides: Mapping[str, SensorOverrideStatus] ) -> SensorOverrideStatus: ... +def validate_sensor_retry_policy(*, max_attempts: int, backoff_seconds: float) -> None: ... +def provider_name(provider_class: type[Provider]) -> str: ... def start_invocation(*, invocation_id: str | None = None) -> tuple[str, str, int]: ... def logical_project_identity(*, project_dir: Path) -> str: ... def build_invocation_record( diff --git a/src/sqltide/adapter/__init__.py b/src/sqltide/adapter/__init__.py index 69b49cae..e69de29b 100644 --- a/src/sqltide/adapter/__init__.py +++ b/src/sqltide/adapter/__init__.py @@ -1 +0,0 @@ -"""Neutral warehouse adapter contract.""" diff --git a/src/sqltide/adapter/classes/__init__.py b/src/sqltide/adapter/classes/__init__.py index b78c00fe..e69de29b 100644 --- a/src/sqltide/adapter/classes/__init__.py +++ b/src/sqltide/adapter/classes/__init__.py @@ -1 +0,0 @@ -"""Neutral adapter contract classes.""" diff --git a/src/sqltide/adapters/__init__.py b/src/sqltide/adapters/__init__.py index a73580e7..e69de29b 100644 --- a/src/sqltide/adapters/__init__.py +++ b/src/sqltide/adapters/__init__.py @@ -1 +0,0 @@ -"""Built-in warehouse adapter implementations.""" diff --git a/src/sqltide/adapters/clickhouse/__init__.py b/src/sqltide/adapters/clickhouse/__init__.py index f56c4d69..e69de29b 100644 --- a/src/sqltide/adapters/clickhouse/__init__.py +++ b/src/sqltide/adapters/clickhouse/__init__.py @@ -1 +0,0 @@ -"""ClickHouse adapter implementation.""" diff --git a/src/sqltide/adapters/clickhouse/classes/__init__.py b/src/sqltide/adapters/clickhouse/classes/__init__.py index 523ea672..e69de29b 100644 --- a/src/sqltide/adapters/clickhouse/classes/__init__.py +++ b/src/sqltide/adapters/clickhouse/classes/__init__.py @@ -1 +0,0 @@ -"""ClickHouse adapter classes.""" diff --git a/src/sqltide/auth/__init__.py b/src/sqltide/auth/__init__.py index f384f258..e69de29b 100644 --- a/src/sqltide/auth/__init__.py +++ b/src/sqltide/auth/__init__.py @@ -1 +0,0 @@ -"""SQLTide authentication and account control plane.""" diff --git a/src/sqltide/auth/_helpers/__init__.py b/src/sqltide/auth/_helpers/__init__.py index 8e0cc41a..e69de29b 100644 --- a/src/sqltide/auth/_helpers/__init__.py +++ b/src/sqltide/auth/_helpers/__init__.py @@ -1 +0,0 @@ -"""Internal authentication helpers.""" diff --git a/src/sqltide/auth/classes/__init__.py b/src/sqltide/auth/classes/__init__.py index b0d10bd9..e69de29b 100644 --- a/src/sqltide/auth/classes/__init__.py +++ b/src/sqltide/auth/classes/__init__.py @@ -1 +0,0 @@ -"""Authentication runtime classes.""" diff --git a/src/sqltide/authorization/__init__.py b/src/sqltide/authorization/__init__.py index 7b9c4cc9..e69de29b 100644 --- a/src/sqltide/authorization/__init__.py +++ b/src/sqltide/authorization/__init__.py @@ -1 +0,0 @@ -"""Centralized operational authorization.""" diff --git a/src/sqltide/cli/__init__.py b/src/sqltide/cli/__init__.py index a3ba1f69..e69de29b 100644 --- a/src/sqltide/cli/__init__.py +++ b/src/sqltide/cli/__init__.py @@ -1 +0,0 @@ -"""CLI package.""" diff --git a/src/sqltide/cli/admin/__init__.py b/src/sqltide/cli/admin/__init__.py index b9ccb690..e69de29b 100644 --- a/src/sqltide/cli/admin/__init__.py +++ b/src/sqltide/cli/admin/__init__.py @@ -1 +0,0 @@ -"""Account administration CLI.""" diff --git a/src/sqltide/cli/build/__init__.py b/src/sqltide/cli/build/__init__.py index d05c6978..e69de29b 100644 --- a/src/sqltide/cli/build/__init__.py +++ b/src/sqltide/cli/build/__init__.py @@ -1 +0,0 @@ -"""Direct build command.""" diff --git a/src/sqltide/cli/compile/__init__.py b/src/sqltide/cli/compile/__init__.py index fb8fb6c9..e69de29b 100644 --- a/src/sqltide/cli/compile/__init__.py +++ b/src/sqltide/cli/compile/__init__.py @@ -1 +0,0 @@ -"""Compile command package.""" diff --git a/src/sqltide/cli/destruction/__init__.py b/src/sqltide/cli/destruction/__init__.py index 79d48a42..e69de29b 100644 --- a/src/sqltide/cli/destruction/__init__.py +++ b/src/sqltide/cli/destruction/__init__.py @@ -1 +0,0 @@ -"""Interactive destructive CLI commands.""" diff --git a/src/sqltide/cli/entry/__init__.py b/src/sqltide/cli/entry/__init__.py index 8b14e645..e69de29b 100644 --- a/src/sqltide/cli/entry/__init__.py +++ b/src/sqltide/cli/entry/__init__.py @@ -1 +0,0 @@ -"""CLI root entry package.""" diff --git a/src/sqltide/cli/plan/__init__.py b/src/sqltide/cli/plan/__init__.py index f3fdc3ef..e69de29b 100644 --- a/src/sqltide/cli/plan/__init__.py +++ b/src/sqltide/cli/plan/__init__.py @@ -1 +0,0 @@ -"""Plan command package.""" diff --git a/src/sqltide/cli/selection/__init__.py b/src/sqltide/cli/selection/__init__.py index 851c40c9..e69de29b 100644 --- a/src/sqltide/cli/selection/__init__.py +++ b/src/sqltide/cli/selection/__init__.py @@ -1 +0,0 @@ -"""CLI selection.""" diff --git a/src/sqltide/compiler/__init__.py b/src/sqltide/compiler/__init__.py index 5729082c..e69de29b 100644 --- a/src/sqltide/compiler/__init__.py +++ b/src/sqltide/compiler/__init__.py @@ -1 +0,0 @@ -"""Compilation from authored specs into desired state.""" diff --git a/src/sqltide/compiler/access/__init__.py b/src/sqltide/compiler/access/__init__.py index c0fa5ddb..e69de29b 100644 --- a/src/sqltide/compiler/access/__init__.py +++ b/src/sqltide/compiler/access/__init__.py @@ -1 +0,0 @@ -"""Compiled project operational access policy.""" diff --git a/src/sqltide/compiler/audit_discovery/__init__.py b/src/sqltide/compiler/audit_discovery/__init__.py index ea53e6e6..e69de29b 100644 --- a/src/sqltide/compiler/audit_discovery/__init__.py +++ b/src/sqltide/compiler/audit_discovery/__init__.py @@ -1 +0,0 @@ -"""SQL audit discovery package.""" diff --git a/src/sqltide/compiler/compile/__init__.py b/src/sqltide/compiler/compile/__init__.py index 3c0ce6ca..e69de29b 100644 --- a/src/sqltide/compiler/compile/__init__.py +++ b/src/sqltide/compiler/compile/__init__.py @@ -1 +0,0 @@ -"""Pipeline compilation.""" diff --git a/src/sqltide/compiler/discovery/__init__.py b/src/sqltide/compiler/discovery/__init__.py index f786c710..e69de29b 100644 --- a/src/sqltide/compiler/discovery/__init__.py +++ b/src/sqltide/compiler/discovery/__init__.py @@ -1 +0,0 @@ -"""Pipeline discovery.""" diff --git a/src/sqltide/compiler/graph/__init__.py b/src/sqltide/compiler/graph/__init__.py index 799a52d4..e69de29b 100644 --- a/src/sqltide/compiler/graph/__init__.py +++ b/src/sqltide/compiler/graph/__init__.py @@ -1 +0,0 @@ -"""Logical and realized dependency graph behavior.""" diff --git a/src/sqltide/compiler/macros/__init__.py b/src/sqltide/compiler/macros/__init__.py index 6864bdfd..e69de29b 100644 --- a/src/sqltide/compiler/macros/__init__.py +++ b/src/sqltide/compiler/macros/__init__.py @@ -1 +0,0 @@ -"""Authored Python SQL macro helpers.""" diff --git a/src/sqltide/compiler/macros/_helpers/__init__.py b/src/sqltide/compiler/macros/_helpers/__init__.py index 0cd3ffb0..e69de29b 100644 --- a/src/sqltide/compiler/macros/_helpers/__init__.py +++ b/src/sqltide/compiler/macros/_helpers/__init__.py @@ -1 +0,0 @@ -"""Internal support code for authored Python SQL macros.""" diff --git a/src/sqltide/compiler/macros/main/__init__.py b/src/sqltide/compiler/macros/main/__init__.py index 9561535c..e69de29b 100644 --- a/src/sqltide/compiler/macros/main/__init__.py +++ b/src/sqltide/compiler/macros/main/__init__.py @@ -1 +0,0 @@ -"""Public macro expansion entry points.""" diff --git a/src/sqltide/compiler/planner/__init__.py b/src/sqltide/compiler/planner/__init__.py index cdf461ec..e69de29b 100644 --- a/src/sqltide/compiler/planner/__init__.py +++ b/src/sqltide/compiler/planner/__init__.py @@ -1 +0,0 @@ -"""Planning surfaces for desired and actual state.""" diff --git a/src/sqltide/compiler/quality/__init__.py b/src/sqltide/compiler/quality/__init__.py index d1b83681..e69de29b 100644 --- a/src/sqltide/compiler/quality/__init__.py +++ b/src/sqltide/compiler/quality/__init__.py @@ -1 +0,0 @@ -"""Stable compiled identities for audits and SQL tests.""" diff --git a/src/sqltide/compiler/rules/__init__.py b/src/sqltide/compiler/rules/__init__.py index 8b026a30..e69de29b 100644 --- a/src/sqltide/compiler/rules/__init__.py +++ b/src/sqltide/compiler/rules/__init__.py @@ -1 +0,0 @@ -"""SQLClean Rules and compiler-verified source formatting.""" diff --git a/src/sqltide/compiler/sql_analysis/__init__.py b/src/sqltide/compiler/sql_analysis/__init__.py index 515c7fba..e69de29b 100644 --- a/src/sqltide/compiler/sql_analysis/__init__.py +++ b/src/sqltide/compiler/sql_analysis/__init__.py @@ -1 +0,0 @@ -"""Mandatory SQL analysis boundary.""" diff --git a/src/sqltide/compiler/test_discovery/__init__.py b/src/sqltide/compiler/test_discovery/__init__.py index 4b716624..e69de29b 100644 --- a/src/sqltide/compiler/test_discovery/__init__.py +++ b/src/sqltide/compiler/test_discovery/__init__.py @@ -1 +0,0 @@ -"""Discovery for SQL-native unit test files.""" diff --git a/src/sqltide/compiler/testing/__init__.py b/src/sqltide/compiler/testing/__init__.py index 8badce33..e69de29b 100644 --- a/src/sqltide/compiler/testing/__init__.py +++ b/src/sqltide/compiler/testing/__init__.py @@ -1 +0,0 @@ -"""Compiler-side SQL test assembly.""" diff --git a/src/sqltide/dev_server/models.py b/src/sqltide/dev_server/models.py index ad177f2c..5ff29996 100644 --- a/src/sqltide/dev_server/models.py +++ b/src/sqltide/dev_server/models.py @@ -1,95 +1,23 @@ -"""Dev server result models.""" +"""Dev server request bodies and result models.""" from __future__ import annotations -from collections.abc import Callable, Mapping -from dataclasses import dataclass -from pathlib import Path from typing import ClassVar, Literal from pydantic import BaseModel, ConfigDict, Field -from sqltide._native import ControlStore, RelationalDestructionPlanStore -from sqltide.adapter.classes.adapter_connection import AdapterConnection -from sqltide.compiler.discovery.constants import DEFAULT_RUN_PRESUMED_FAILED_AFTER_SECONDS -from sqltide.compiler.pipeline.models import CompilationTimings, CompileAnalysis -from sqltide.dev_server.types import CompileStateKind - - -@dataclass(frozen=True) -class ReplayOffsetProgress: - """Approximate progress derived from committed target offset frontiers.""" - - percentage: float - eta_seconds: float | None - completed_span: int - total_span: int - observed_partitions: int - total_partitions: int - - -@dataclass(frozen=True, repr=False) -class DevExecutionContext: - """Resolved invocation values retained across dev reloads, previews, and builds.""" - - database: str | None = None - selected_target: str | None = None - cli_variables: tuple[tuple[str, object], ...] = () - environment: Mapping[str, str] | None = None - connection_host: str | None = None - connection_port: int | None = None - connection_username: str | None = None - connection_password: str | None = None - run_presumed_failed_after_seconds: int = DEFAULT_RUN_PRESUMED_FAILED_AFTER_SECONDS - connection_factory: Callable[[], AdapterConnection] | None = None - observation_connection_factory: Callable[[], AdapterConnection] | None = None - - -@dataclass(frozen=True) -class CompileErrorInfo: - """A compile failure reduced to what an error page needs.""" - - message: str - path: str | None = None - line: int | None = None - column: int | None = None - end_line: int | None = None - end_column: int | None = None - - -@dataclass(frozen=True) -class CompileOutcome: - """One held project compile: either servable definitions or the failure.""" - - state: CompileStateKind - version_key: str - compiled_at: str - analysis: CompileAnalysis | None = None - timings: CompilationTimings | None = None - error: CompileErrorInfo | None = None - - -@dataclass(frozen=True) -class OperationAuthorizationContext: - """Shared control-plane inputs for route-level operation authorization.""" - - store: ControlStore - project_dir: Path - selected_target: str | None - - -@dataclass(frozen=True) -class DevControlStores: - """Durable control-plane stores and their application ownership.""" - - accounts: ControlStore - destruction_plans: RelationalDestructionPlanStore - owns_accounts: bool - - def close(self) -> None: - self.destruction_plans.close() - if self.owns_accounts: - self.accounts.close() +from sqltide._native import BuildCancellationOutcome as BuildCancellationOutcome +from sqltide._native import CompileErrorInfo as CompileErrorInfo +from sqltide._native import CompileOutcome as CompileOutcome +from sqltide._native import DevControlStores as DevControlStores +from sqltide._native import DevExecutionContext as DevExecutionContext +from sqltide._native import KafkaLagSnapshot as KafkaLagSnapshot +from sqltide._native import KafkaPartitionLag as KafkaPartitionLag +from sqltide._native import KafkaTopicInfo as KafkaTopicInfo +from sqltide._native import KafkaTopicsSnapshot as KafkaTopicsSnapshot +from sqltide._native import OperationAuthorizationContext as OperationAuthorizationContext +from sqltide._native import RelationStorage as RelationStorage +from sqltide._native import ReplayOffsetProgress as ReplayOffsetProgress class ChecksRunRequest(BaseModel): @@ -173,64 +101,6 @@ class MessageFacetsRequest(BaseModel): facetPath: list[str | int] = [] # noqa: N815 - wire format is camelCase -@dataclass(frozen=True) -class KafkaTopicInfo: - """Broker metadata for one topic.""" - - name: str - partition_count: int - replication_factor: int - internal: bool - - -@dataclass(frozen=True) -class KafkaTopicsSnapshot: - """Best-effort topic inventory for one broker list.""" - - topics: tuple[KafkaTopicInfo, ...] - - -@dataclass(frozen=True) -class KafkaPartitionLag: - """Broker offsets and lag for one topic partition.""" - - partition: int - committed_offset: int | None - end_offset: int - lag_messages: int | None - - -@dataclass(frozen=True) -class KafkaLagSnapshot: - """Best-effort consumer-group lag for one managed Kafka source.""" - - total_messages: int | None - partitions: tuple[KafkaPartitionLag, ...] - - -@dataclass(frozen=True) -class RelationStorage: - """Row and byte totals for one warehouse relation.""" - - rows: int = 0 - bytes: int = 0 - - -@dataclass(frozen=True) -class BuildCancellationOutcome: - """Terminal evidence from one server-owned build cancellation attempt.""" - - invocation_id: str - query_id: str | None - requested_at: str - status: Literal["cancelled", "cancellation_failed"] - process_exit_code: int - warehouse_supported: bool | None - query_found: bool | None - warehouse_termination_confirmed: bool | None - error_message: str | None = None - - class DeploymentCleanupRequest(BaseModel): """Janitor apply request from the development UI.""" diff --git a/src/sqltide/diagnostics/__init__.py b/src/sqltide/diagnostics/__init__.py index ba3d780c..e69de29b 100644 --- a/src/sqltide/diagnostics/__init__.py +++ b/src/sqltide/diagnostics/__init__.py @@ -1 +0,0 @@ -"""Structured compiler and runtime diagnostics.""" diff --git a/src/sqltide/events/models.py b/src/sqltide/events/models.py index 6b1b58a2..3e69bcef 100644 --- a/src/sqltide/events/models.py +++ b/src/sqltide/events/models.py @@ -4,47 +4,12 @@ from dataclasses import dataclass -from sqltide.compiler.quality.types import QualityNodeKind +from sqltide._native import InvocationObservation as InvocationObservation +from sqltide._native import NodeResultObservation as NodeResultObservation from sqltide.events.types import AuditTransition, ObservedCommand from sqltide.executor.auditing.types import QualityResultStatus -@dataclass(frozen=True) -class NodeResultObservation: - """One persisted quality node result row viewed by the event catalog.""" - - result_id: str - invocation_id: str - node_kind: QualityNodeKind - node_name: str - binding_key: str - target_identity: str - trigger: str - status: QualityResultStatus - severity: str | None - failure_count: int - completed_at: str - scheduled_for: str | None - payload_json: str - error_message: str | None - - -@dataclass(frozen=True) -class InvocationObservation: - """One persisted terminal invocation row viewed by the event catalog.""" - - invocation_id: str - command: str - mode: str | None - outcome: str - exit_code: int - target_identity: str - deployment_id: str | None - selected_node_count: int - error_message: str | None - completed_at: str - - @dataclass(frozen=True) class AuditCompleted: """One audit attempt reached a terminal status.""" diff --git a/src/sqltide/executor/__init__.py b/src/sqltide/executor/__init__.py index c2518142..e69de29b 100644 --- a/src/sqltide/executor/__init__.py +++ b/src/sqltide/executor/__init__.py @@ -1 +0,0 @@ -"""Plan application logic.""" diff --git a/src/sqltide/executor/auditing/__init__.py b/src/sqltide/executor/auditing/__init__.py index add8c813..e69de29b 100644 --- a/src/sqltide/executor/auditing/__init__.py +++ b/src/sqltide/executor/auditing/__init__.py @@ -1 +0,0 @@ -"""SQL audit execution package.""" diff --git a/src/sqltide/executor/backfill/__init__.py b/src/sqltide/executor/backfill/__init__.py index 76a8abe8..e69de29b 100644 --- a/src/sqltide/executor/backfill/__init__.py +++ b/src/sqltide/executor/backfill/__init__.py @@ -1 +0,0 @@ -"""Backfill executor entrypoints.""" diff --git a/src/sqltide/executor/deployment/__init__.py b/src/sqltide/executor/deployment/__init__.py index b8d2628d..e69de29b 100644 --- a/src/sqltide/executor/deployment/__init__.py +++ b/src/sqltide/executor/deployment/__init__.py @@ -1 +0,0 @@ -"""Virtual deployment lifecycle operations.""" diff --git a/src/sqltide/executor/direct/__init__.py b/src/sqltide/executor/direct/__init__.py index e28d6b49..e69de29b 100644 --- a/src/sqltide/executor/direct/__init__.py +++ b/src/sqltide/executor/direct/__init__.py @@ -1 +0,0 @@ -"""Direct-mode build execution.""" diff --git a/src/sqltide/executor/doctor/__init__.py b/src/sqltide/executor/doctor/__init__.py index ad26bbeb..e69de29b 100644 --- a/src/sqltide/executor/doctor/__init__.py +++ b/src/sqltide/executor/doctor/__init__.py @@ -1 +0,0 @@ -"""Doctor execution entrypoints.""" diff --git a/src/sqltide/executor/observability/__init__.py b/src/sqltide/executor/observability/__init__.py index 551dbf39..e69de29b 100644 --- a/src/sqltide/executor/observability/__init__.py +++ b/src/sqltide/executor/observability/__init__.py @@ -1 +0,0 @@ -"""Non-authoritative terminal observation recording.""" diff --git a/src/sqltide/executor/promotion/__init__.py b/src/sqltide/executor/promotion/__init__.py index f59641c0..e69de29b 100644 --- a/src/sqltide/executor/promotion/__init__.py +++ b/src/sqltide/executor/promotion/__init__.py @@ -1 +0,0 @@ -"""Publish execution entrypoints.""" diff --git a/src/sqltide/executor/readiness/__init__.py b/src/sqltide/executor/readiness/__init__.py index 4dc75bff..e69de29b 100644 --- a/src/sqltide/executor/readiness/__init__.py +++ b/src/sqltide/executor/readiness/__init__.py @@ -1 +0,0 @@ -"""Deployment readiness execution.""" diff --git a/src/sqltide/executor/reconcile/__init__.py b/src/sqltide/executor/reconcile/__init__.py index 9c19ca2f..e69de29b 100644 --- a/src/sqltide/executor/reconcile/__init__.py +++ b/src/sqltide/executor/reconcile/__init__.py @@ -1 +0,0 @@ -"""Package.""" diff --git a/src/sqltide/executor/repair/__init__.py b/src/sqltide/executor/repair/__init__.py index 25707e63..e69de29b 100644 --- a/src/sqltide/executor/repair/__init__.py +++ b/src/sqltide/executor/repair/__init__.py @@ -1 +0,0 @@ -"""Repair execution entrypoints.""" diff --git a/src/sqltide/executor/testing/__init__.py b/src/sqltide/executor/testing/__init__.py index 4311cf27..e69de29b 100644 --- a/src/sqltide/executor/testing/__init__.py +++ b/src/sqltide/executor/testing/__init__.py @@ -1 +0,0 @@ -"""Runtime execution for SQL-native tests.""" diff --git a/src/sqltide/executor/workflow/__init__.py b/src/sqltide/executor/workflow/__init__.py index d981634d..e69de29b 100644 --- a/src/sqltide/executor/workflow/__init__.py +++ b/src/sqltide/executor/workflow/__init__.py @@ -1 +0,0 @@ -"""Warehouse workflow authority.""" diff --git a/src/sqltide/executor/workflow/main/__init__.py b/src/sqltide/executor/workflow/main/__init__.py index 2d96ec52..e69de29b 100644 --- a/src/sqltide/executor/workflow/main/__init__.py +++ b/src/sqltide/executor/workflow/main/__init__.py @@ -1 +0,0 @@ -"""Workflow execution entry boundaries.""" diff --git a/src/sqltide/provider/__init__.py b/src/sqltide/provider/__init__.py index 8e08a6f0..e69de29b 100644 --- a/src/sqltide/provider/__init__.py +++ b/src/sqltide/provider/__init__.py @@ -1 +0,0 @@ -"""Provider internals.""" diff --git a/src/sqltide/provider/_helpers/__init__.py b/src/sqltide/provider/_helpers/__init__.py index 243a221f..e69de29b 100644 --- a/src/sqltide/provider/_helpers/__init__.py +++ b/src/sqltide/provider/_helpers/__init__.py @@ -1 +0,0 @@ -"""Provider internal helpers.""" diff --git a/src/sqltide/provider/_helpers/discovery.py b/src/sqltide/provider/_helpers/discovery.py deleted file mode 100644 index e53493ac..00000000 --- a/src/sqltide/provider/_helpers/discovery.py +++ /dev/null @@ -1,156 +0,0 @@ -"""Apache-2.0: SQLBuild compiler/discovery/_helpers/filesystem/core.py@7625d22e2716.""" - -from __future__ import annotations - -import importlib.util -import inspect -import sys -from importlib.machinery import ModuleSpec -from pathlib import Path -from types import ModuleType - -from pydantic import ValidationError - -from sqltide.provider.exceptions import ProviderDiscoveryError, ProviderInputError -from sqltide.provider.models import DiscoveredProvider -from sqltide.providers import Provider - -_PYTHON_INIT_MODULE_STEM: str = "__init__" - - -def discover_provider_classes(*, project_dir: Path) -> tuple[DiscoveredProvider, ...]: - """Discover provider classes under providers/.""" - - providers_root: Path = project_dir / "providers" - if not providers_root.is_dir(): - return () - - discovered_providers: list[DiscoveredProvider] = [] - seen_names: dict[str, Path] = {} - file_path: Path - for file_path in sorted(providers_root.rglob("*.py")): - if file_path.stem == _PYTHON_INIT_MODULE_STEM or file_path.name.startswith("_"): - continue - module: ModuleType = _load_provider_module(file_path=file_path, project_dir=project_dir) - for _, value in inspect.getmembers(module, inspect.isclass): - if value.__module__ != module.__name__: - continue - if value is Provider or not issubclass(value, Provider) or inspect.isabstract(value): - continue - provider_class: type[Provider] = value - provider_name: str = _provider_name( - provider_class=provider_class, - file_path=file_path, - project_dir=project_dir, - ) - existing_path: Path | None = seen_names.get(provider_name) - if existing_path is not None: - raise ProviderDiscoveryError( - f"Duplicate provider name '{provider_name}' found in " - f"{existing_path.relative_to(project_dir)} and " - f"{file_path.relative_to(project_dir)}" - ) - seen_names[provider_name] = file_path - discovered_providers.append( - DiscoveredProvider( - file_path=file_path, - relative_path=file_path.relative_to(project_dir), - name=provider_name, - provider_class=provider_class, - settings=_provider_instance( - provider_class=provider_class, - provider_name=provider_name, - file_path=file_path, - project_dir=project_dir, - ), - ) - ) - return tuple(discovered_providers) - - -def _provider_name(*, provider_class: type[Provider], file_path: Path, project_dir: Path) -> str: - try: - return provider_class.name() - except ProviderInputError as error: - raise ProviderDiscoveryError( - f"Provider class {provider_class.__name__} in {file_path.relative_to(project_dir)} " - f"has an invalid provider name: {error}" - ) from error - - -def _provider_instance( - *, provider_class: type[Provider], provider_name: str, file_path: Path, project_dir: Path -) -> Provider: - try: - return provider_class() - except ValidationError as error: - relative_path: Path = file_path.relative_to(project_dir) - raise ProviderDiscoveryError( - f"Provider '{provider_name}' in {relative_path} has invalid settings:\n" - f"{_format_provider_validation_error(error)}" - ) from error - - -def _format_provider_validation_error(error: ValidationError) -> str: - details: list[str] = [] - for item in error.errors(include_input=False): - location: object = item.get("loc", ()) - location_text: str = ".".join(str(part) for part in location) if location else "" - message: object = item.get("msg", "invalid value") - error_type: object = item.get("type", "validation_error") - details.append(f"{location_text}: {message} [{error_type}]") - return "\n".join(details) or "invalid provider settings" - - -def _load_provider_module(*, file_path: Path, project_dir: Path) -> ModuleType: - module_name: str = ".".join(file_path.relative_to(project_dir).with_suffix("").parts) - _evict_stale_project_package_modules( - root_module=module_name.split(".", maxsplit=1)[0], - project_dir=project_dir, - ) - existing_module: ModuleType | None = sys.modules.get(module_name) - if existing_module is not None: - existing_file: object = getattr(existing_module, "__file__", None) - if isinstance(existing_file, str) and Path(existing_file).resolve() == file_path.resolve(): - return existing_module - sys.modules.pop(module_name, None) - spec: ModuleSpec | None = importlib.util.spec_from_file_location(module_name, file_path) - if spec is None or spec.loader is None: - raise ProviderDiscoveryError(f"Could not load provider file {file_path}") - module: ModuleType = importlib.util.module_from_spec(spec) - old_path: list[str] = list(sys.path) - sys.path.insert(0, str(project_dir)) - try: - sys.modules[module_name] = module - spec.loader.exec_module(module) - except Exception as error: - raise ProviderDiscoveryError( - f"Failed to import provider file {file_path.relative_to(project_dir)}: {error}" - ) from error - finally: - sys.path = old_path - return module - - -def _evict_stale_project_package_modules(*, root_module: str, project_dir: Path) -> None: - root_path: Path = (project_dir / root_module).resolve() - module_name: str - module: ModuleType - for module_name, module in tuple(sys.modules.items()): - if module_name != root_module and not module_name.startswith(f"{root_module}."): - continue - module_file: object = getattr(module, "__file__", None) - if isinstance(module_file, str): - try: - Path(module_file).resolve().relative_to(project_dir.resolve()) - continue - except ValueError: - sys.modules.pop(module_name, None) - continue - module_paths: object = getattr(module, "__path__", None) - if module_paths is None: - sys.modules.pop(module_name, None) - continue - if any(Path(path).resolve() == root_path for path in module_paths): - continue - sys.modules.pop(module_name, None) diff --git a/src/sqltide/provider/classes/__init__.py b/src/sqltide/provider/classes/__init__.py index 0377b33b..e69de29b 100644 --- a/src/sqltide/provider/classes/__init__.py +++ b/src/sqltide/provider/classes/__init__.py @@ -1 +0,0 @@ -"""Provider lifecycle classes.""" diff --git a/src/sqltide/provider/main/__init__.py b/src/sqltide/provider/main/__init__.py index d58a38b6..e69de29b 100644 --- a/src/sqltide/provider/main/__init__.py +++ b/src/sqltide/provider/main/__init__.py @@ -1 +0,0 @@ -"""Provider entry points for other SQLTide domains.""" diff --git a/src/sqltide/provider/main/_discover_provider_classes.py b/src/sqltide/provider/main/_discover_provider_classes.py deleted file mode 100644 index fb3617be..00000000 --- a/src/sqltide/provider/main/_discover_provider_classes.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Provider discovery entry point for other SQLTide domains.""" - -from __future__ import annotations - -from pathlib import Path - -from sqltide.provider._helpers.discovery import ( - discover_provider_classes as _discover_provider_classes, -) -from sqltide.provider.models import DiscoveredProvider - - -def discover_provider_classes(*, project_dir: Path) -> tuple[DiscoveredProvider, ...]: - """Discover provider classes under providers/.""" - - return _discover_provider_classes(project_dir=project_dir) diff --git a/src/sqltide/providers.py b/src/sqltide/providers.py index 2813685e..31354e31 100644 --- a/src/sqltide/providers.py +++ b/src/sqltide/providers.py @@ -2,12 +2,11 @@ from __future__ import annotations -import re from typing import Any, ClassVar from pydantic_settings import BaseSettings, SettingsConfigDict -from sqltide.provider.exceptions import ProviderInputError +import sqltide._native as _native class Provider(BaseSettings): @@ -23,18 +22,7 @@ class Provider(BaseSettings): def name(cls) -> str: """Return the runtime provider name for this provider class.""" - raw_name: str | None = cls.provider_name - if raw_name is None: - raw_name = re.sub(r"(? None: """Prepare the provider for runtime use.""" diff --git a/src/sqltide/sensors/_helpers/__init__.py b/src/sqltide/sensors/_helpers/__init__.py index a8e461a6..e69de29b 100644 --- a/src/sqltide/sensors/_helpers/__init__.py +++ b/src/sqltide/sensors/_helpers/__init__.py @@ -1 +0,0 @@ -"""Sensor internal helpers.""" diff --git a/src/sqltide/sensors/classes/__init__.py b/src/sqltide/sensors/classes/__init__.py index 778b2d4a..e69de29b 100644 --- a/src/sqltide/sensors/classes/__init__.py +++ b/src/sqltide/sensors/classes/__init__.py @@ -1 +0,0 @@ -"""Sensor runtime classes.""" diff --git a/src/sqltide/sensors/models.py b/src/sqltide/sensors/models.py index fc37756b..3c19e355 100644 --- a/src/sqltide/sensors/models.py +++ b/src/sqltide/sensors/models.py @@ -1,24 +1,23 @@ -"""Authored sensor declarations and the loaded sensor registry.""" +"""Authored sensor declarations and policies, plus the native sensor runtime shapes.""" from __future__ import annotations -from collections.abc import Mapping -from dataclasses import dataclass, field -from hashlib import sha256 -from pathlib import Path -from types import MappingProxyType - -from sqltide.events.types import SensorEvent -from sqltide.provider.models import DiscoveredProvider -from sqltide.sensors.constants import DEFAULT_SENSOR_TIMEOUT_SECONDS -from sqltide.sensors.exceptions import SensorError -from sqltide.sensors.types import ( - DefaultSensorStatus, - SensorDeclaration, - SensorFunction, - SensorKind, - SensorTickStatus, -) +from dataclasses import dataclass + +import sqltide._native as _native +from sqltide._native import CompiledSensors as CompiledSensors +from sqltide._native import LoadedSensor as LoadedSensor +from sqltide._native import PollingTickState as PollingTickState +from sqltide._native import SensorDeliveryOutcome as SensorDeliveryOutcome +from sqltide._native import SensorDispatchSummary as SensorDispatchSummary +from sqltide._native import SensorEvaluation as SensorEvaluation +from sqltide._native import SensorEventDelivery as SensorEventDelivery +from sqltide._native import SensorRegistry as SensorRegistry +from sqltide._native import SensorStreamPosition as SensorStreamPosition +from sqltide._native import SensorTickView as SensorTickView +from sqltide._native import StepMarker as StepMarker +from sqltide._native import TickAttemptState as TickAttemptState +from sqltide.sensors.types import DefaultSensorStatus, SensorFunction @dataclass(frozen=True) @@ -29,10 +28,9 @@ class SensorRetryPolicy: backoff_seconds: float = 0.0 def __post_init__(self) -> None: - if self.max_attempts < 1: - raise SensorError("SensorRetryPolicy.max_attempts must be at least 1") - if self.backoff_seconds < 0: - raise SensorError("SensorRetryPolicy.backoff_seconds must not be negative") + _native.validate_sensor_retry_policy( + max_attempts=self.max_attempts, backoff_seconds=self.backoff_seconds + ) @dataclass(frozen=True) @@ -80,148 +78,3 @@ class PollingSensorDeclaration: def __call__(self, *args: object, **kwargs: object) -> object: return self.function(*args, **kwargs) - - -@dataclass(frozen=True) -class LoadedSensor: - """One registered authored sensor with retained source identity.""" - - name: str - kind: SensorKind - declaration: SensorDeclaration - file_path: Path - relative_path: Path - source: str - definition_line: int - description: str | None = None - timeout_seconds: float = DEFAULT_SENSOR_TIMEOUT_SECONDS - - @property - def identity_fingerprint(self) -> str: - """Fingerprint the sensor identity and retained implementation source.""" - - payload: str = f"{self.name}\0{self.relative_path.as_posix()}\0{self.source}" - return sha256(payload.encode("utf-8")).hexdigest() - - @property - def default_status(self) -> DefaultSensorStatus: - return self.declaration.default_status - - @property - def retry_policy(self) -> SensorRetryPolicy: - return self.declaration.retry_policy - - -@dataclass(frozen=True) -class CompiledSensors: - """Compiled project automation: loaded sensors plus discovered providers.""" - - registry: SensorRegistry - providers: tuple[DiscoveredProvider, ...] = () - - -@dataclass(frozen=True) -class SensorStreamPosition: - """One (completed_at, id) position in an ordered observation stream.""" - - completed_at: str - result_id: str - - -@dataclass(frozen=True) -class StepMarker: - """Reduced durable-step state for one (sensor, event, key).""" - - status: str - result_json: str | None - attempt: int - - -@dataclass(frozen=True) -class TickAttemptState: - """Reduced tick history for one (sensor, event).""" - - failed_attempts: int - last_failed_at: str | None - last_error_message: str | None - resolved: bool - - -@dataclass(frozen=True) -class SensorTickView: - """One reduced tick with its terminal state preferred over started rows.""" - - tick_id: str - sensor_name: str - definition_fingerprint: str - kind: str - event_id: str | None - event_kind: str | None - attempt: int - status: str - started_at: str - completed_at: str | None - error_message: str | None - skip_reason: str | None - cursor: str | None - - -@dataclass(frozen=True) -class PollingTickState: - """Latest polling tick timing and cursor for one sensor.""" - - last_started_at: str | None - last_success_at: str | None - cursor: str | None - - -@dataclass(frozen=True) -class SensorEvaluation: - """Terminal outcome of one isolated sensor handler invocation.""" - - status: SensorTickStatus - skip_reason: str | None = None - error_message: str | None = None - cursor: str | None = None - - -@dataclass(frozen=True) -class SensorDeliveryOutcome: - """Resolution of one matched event delivery within a dispatch pass.""" - - resolved: bool - evaluation: SensorEvaluation | None = None - dead_lettered: bool = False - - -@dataclass(frozen=True) -class SensorEventDelivery: - """Events derived from one positioned observation row.""" - - position: SensorStreamPosition - events: tuple[SensorEvent, ...] - - -@dataclass(frozen=True) -class SensorDispatchSummary: - """Counters describing one dispatch pass.""" - - lease_acquired: bool = True - evaluated: int = 0 - succeeded: int = 0 - skipped: int = 0 - failed: int = 0 - dead_lettered: int = 0 - - -@dataclass(frozen=True) -class SensorRegistry: - """Deterministic name-keyed registry of loaded project sensors.""" - - sensors: Mapping[str, LoadedSensor] = field(default_factory=dict) - - def __post_init__(self) -> None: - object.__setattr__(self, "sensors", MappingProxyType(dict(self.sensors))) - - def ordered(self) -> tuple[LoadedSensor, ...]: - return tuple(self.sensors[name] for name in sorted(self.sensors)) diff --git a/src/sqltide/sensors/types.py b/src/sqltide/sensors/types.py index 8be99a55..35cfc458 100644 --- a/src/sqltide/sensors/types.py +++ b/src/sqltide/sensors/types.py @@ -3,9 +3,14 @@ from __future__ import annotations from collections.abc import Callable -from enum import StrEnum from typing import TYPE_CHECKING, Protocol +from sqltide._native import DefaultSensorStatus as DefaultSensorStatus +from sqltide._native import SensorKind as SensorKind +from sqltide._native import SensorOverrideStatus as SensorOverrideStatus +from sqltide._native import SensorTickStatus as SensorTickStatus +from sqltide._native import StepPolicy as StepPolicy + if TYPE_CHECKING: from sqltide.sensors.models import ( EventSensorDeclaration, @@ -17,45 +22,6 @@ type SensorDeclaration = EventSensorDeclaration | PollingSensorDeclaration -class SensorKind(StrEnum): - """Kinds of authored sensors.""" - - EVENT = "event" - POLLING = "polling" - - -class DefaultSensorStatus(StrEnum): - """Authored default sensor status declared in code.""" - - RUNNING = "running" - STOPPED = "stopped" - - -class SensorOverrideStatus(StrEnum): - """Runtime override state persisted by operator actions.""" - - RUNNING = "running" - STOPPED = "stopped" - DECLARED_IN_CODE = "declared_in_code" - - -class SensorTickStatus(StrEnum): - """Persisted status of one sensor tick attempt.""" - - STARTED = "started" - SKIPPED = "skipped" - SUCCEEDED = "succeeded" - FAILED = "failed" - DEAD_LETTERED = "dead_lettered" - - -class StepPolicy(StrEnum): - """Durable step re-execution policy.""" - - AT_LEAST_ONCE = "at_least_once" - AT_MOST_ONCE = "at_most_once" - - class SensorStepStore(Protocol): """Persisted step marker storage keyed by (sensor, event, step).""" diff --git a/tests/unit/src/sqltide/compiler/discovery/_test_types.py b/tests/unit/src/sqltide/compiler/discovery/_test_types.py index ea016f8c..9664f8d5 100644 --- a/tests/unit/src/sqltide/compiler/discovery/_test_types.py +++ b/tests/unit/src/sqltide/compiler/discovery/_test_types.py @@ -344,3 +344,11 @@ class ProjectConnectionSettingsTestCase: class ProjectProductionTargetTestCase: description: str expected_production_target: bool + + +@dataclass(frozen=True) +class ModelStepErrorTestCase: + description: str + step_class: type + arguments: dict[str, object] + expected_message: str diff --git a/tests/unit/src/sqltide/compiler/discovery/test_models.py b/tests/unit/src/sqltide/compiler/discovery/test_models.py new file mode 100644 index 00000000..c5bb3ace --- /dev/null +++ b/tests/unit/src/sqltide/compiler/discovery/test_models.py @@ -0,0 +1,70 @@ +"""Behavior tests for the authored model step specifications.""" + +import pytest + +from sqltide.compiler.discovery.exceptions import ProjectSpecError +from sqltide.compiler.discovery.models import TransformStep, ViewStep +from tests.unit.src.sqltide.compiler.discovery._test_types import ModelStepErrorTestCase + +QUERY_OR_FILE_MESSAGE: str = "Exactly one of 'query' or 'sql_file' must be provided" +TRANSFORM_ARGUMENTS: dict[str, object] = { + "name": "orders", + "source": "raw_orders", + "engine": "MergeTree", +} + + +@pytest.mark.parametrize( + "test_case", + [ + ModelStepErrorTestCase( + description="a transform needs a query or a SQL file", + step_class=TransformStep, + arguments={**TRANSFORM_ARGUMENTS, "order_by": ("order_id",)}, + expected_message=QUERY_OR_FILE_MESSAGE, + ), + ModelStepErrorTestCase( + description="a transform rejects both a query and a SQL file", + step_class=TransformStep, + arguments={ + **TRANSFORM_ARGUMENTS, + "order_by": ("order_id",), + "query": "SELECT 1", + "sql_file": "orders.sql", + }, + expected_message=QUERY_OR_FILE_MESSAGE, + ), + ModelStepErrorTestCase( + description="a transform needs at least one ORDER BY expression", + step_class=TransformStep, + arguments={**TRANSFORM_ARGUMENTS, "order_by": [], "query": "SELECT 1"}, + expected_message="TransformStep must declare at least one ORDER BY expression", + ), + ModelStepErrorTestCase( + description="the query check runs before the ORDER BY check", + step_class=TransformStep, + arguments={**TRANSFORM_ARGUMENTS, "order_by": (), "sql_file": ""}, + expected_message=QUERY_OR_FILE_MESSAGE, + ), + ModelStepErrorTestCase( + description="a view needs a query or a SQL file", + step_class=ViewStep, + arguments={"name": "orders_view", "query": ""}, + expected_message=QUERY_OR_FILE_MESSAGE, + ), + ModelStepErrorTestCase( + description="a view rejects both a query and a SQL file", + step_class=ViewStep, + arguments={"name": "orders_view", "query": "SELECT 1", "sql_file": "view.sql"}, + expected_message=QUERY_OR_FILE_MESSAGE, + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_model_step_when_constructing_then_project_spec_error_is_raised( + test_case: ModelStepErrorTestCase, +) -> None: + with pytest.raises(ProjectSpecError) as error_info: + _ = test_case.step_class(**test_case.arguments) + + assert error_info.value.args == (test_case.expected_message,) diff --git a/tests/unit/src/sqltide/sensors/_helpers/_test_types.py b/tests/unit/src/sqltide/sensors/_helpers/_test_types.py index 1ac9024b..79279034 100644 --- a/tests/unit/src/sqltide/sensors/_helpers/_test_types.py +++ b/tests/unit/src/sqltide/sensors/_helpers/_test_types.py @@ -28,3 +28,10 @@ class SensorRegistryErrorTestCase: files: tuple[tuple[str, str], ...] expected_error_fragment: str expected_location_line: int | None = None + + +@dataclass(frozen=True) +class SensorFingerprintTestCase: + description: str + files: tuple[tuple[str, str], ...] + expected_fingerprints: tuple[tuple[str, str], ...] diff --git a/tests/unit/src/sqltide/sensors/_helpers/helpers.py b/tests/unit/src/sqltide/sensors/_helpers/helpers.py index c06ff12a..068004cd 100644 --- a/tests/unit/src/sqltide/sensors/_helpers/helpers.py +++ b/tests/unit/src/sqltide/sensors/_helpers/helpers.py @@ -29,6 +29,44 @@ def kafka_lag_watch(ctx): \"\"\"Watch consumer lag.\"\"\" """ +RETRYING_SENSOR_SOURCE: str = """ +from sqltide.events import RunCompleted +from sqltide.sensors import ( + DefaultSensorStatus, + SensorRetryPolicy, + StepPolicy, + event_sensor, +) + + +@event_sensor( + on=RunCompleted, + targets={"prod"}, + default_status=DefaultSensorStatus.STOPPED, + retry_policy=SensorRetryPolicy(max_attempts=3, backoff_seconds=30), + timeout_seconds=12.5, +) +def announce_run(ctx): + \"\"\"Announce completed runs.\"\"\" + ctx.step("notify", lambda: ctx.event.invocation_id, policy=StepPolicy.AT_MOST_ONCE) +""" + +PAIRED_SENSORS_SOURCE: str = """ +from sqltide.events import AuditCompleted +from sqltide.sensors import SkipReason, event_sensor, polling_sensor + + +@event_sensor(on=AuditCompleted, maximum_event_age_seconds=60) +def stale_audits(ctx): + \"\"\"Café orders: skip stale audit results.\"\"\" + return SkipReason("stale") + + +@polling_sensor(minimum_interval_seconds=5) +def inventory_poll(ctx): + return None +""" + def write_project_files(*, project_dir: Path, files: tuple[tuple[str, str], ...]) -> None: for relative_path, contents in files: diff --git a/tests/unit/src/sqltide/sensors/_helpers/test_registry.py b/tests/unit/src/sqltide/sensors/_helpers/test_registry.py index a568b61d..bc97a089 100644 --- a/tests/unit/src/sqltide/sensors/_helpers/test_registry.py +++ b/tests/unit/src/sqltide/sensors/_helpers/test_registry.py @@ -7,12 +7,15 @@ from sqltide.sensors.exceptions import SensorError from sqltide.sensors.models import SensorRegistry from tests.unit.src.sqltide.sensors._helpers._test_types import ( + SensorFingerprintTestCase, SensorRegistryErrorTestCase, SensorRegistryLoadTestCase, ) from tests.unit.src.sqltide.sensors._helpers.helpers import ( LAG_SENSOR_SOURCE, + PAIRED_SENSORS_SOURCE, QUALITY_SENSOR_SOURCE, + RETRYING_SENSOR_SOURCE, load_registry, write_project_files, ) @@ -88,6 +91,69 @@ def test_given_unchanged_sources_when_reloading_then_fingerprints_are_stable( ) +@pytest.mark.parametrize( + "test_case", + [ + SensorFingerprintTestCase( + description="an event sensor hashes its name, path and module source", + files=(("sensors/quality.py", QUALITY_SENSOR_SOURCE),), + expected_fingerprints=( + ( + "quality_alerts", + "b0d8ddb99aa4f7c889765b017e571e215902f120a009d2fbc97e1ae6396bb4fb", + ), + ), + ), + SensorFingerprintTestCase( + description="a polling sensor hashes its name, path and module source", + files=(("sensors/lag.py", LAG_SENSOR_SOURCE),), + expected_fingerprints=( + ( + "kafka_lag_watch", + "4618e4883fc4b13596ee900059f4b3b0a5b1ed8e2b2902fdb37c49b6d6453fa9", + ), + ), + ), + SensorFingerprintTestCase( + description="a nested sensor with retries, steps and a stopped status", + files=(("sensors/alerts/retrying.py", RETRYING_SENSOR_SOURCE),), + expected_fingerprints=( + ( + "announce_run", + "f16b2d3541a1050043ccf0021655fe68e8f410e51d809710339787c618eb20c1", + ), + ), + ), + SensorFingerprintTestCase( + description="sensors sharing a non-ASCII module differ by name", + files=(("sensors/paired.py", PAIRED_SENSORS_SOURCE),), + expected_fingerprints=( + ( + "inventory_poll", + "e49e2925abfe4569ae920de86136b12e9678a2ba2e8456d217d65bfd28c2c8c8", + ), + ( + "stale_audits", + "0fadd05926fc89caf1abf02f8c30657ffcc1d954298279d8bc9cb943201bbadb", + ), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_authored_sensors_when_loading_then_persisted_fingerprints_are_unchanged( + test_case: SensorFingerprintTestCase, tmp_path: Path +) -> None: + write_project_files(project_dir=tmp_path, files=test_case.files) + + registry: SensorRegistry = load_registry(project_dir=tmp_path) + + assert ( + tuple((sensor.name, sensor.identity_fingerprint) for sensor in registry.ordered()) + == test_case.expected_fingerprints + ) + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqltide/sensors/_test_types.py b/tests/unit/src/sqltide/sensors/_test_types.py new file mode 100644 index 00000000..6027e832 --- /dev/null +++ b/tests/unit/src/sqltide/sensors/_test_types.py @@ -0,0 +1,9 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class SensorRetryPolicyErrorTestCase: + description: str + max_attempts: int + backoff_seconds: float + expected_message: str diff --git a/tests/unit/src/sqltide/sensors/classes/_test_types.py b/tests/unit/src/sqltide/sensors/classes/_test_types.py index 453e9cee..d19b44dc 100644 --- a/tests/unit/src/sqltide/sensors/classes/_test_types.py +++ b/tests/unit/src/sqltide/sensors/classes/_test_types.py @@ -86,3 +86,12 @@ class StepRunnerTestCase: expected_first_value: object expected_second_value: object expected_call_count: int + + +@dataclass(frozen=True) +class CompiledSensorDispatchTestCase: + description: str + sensor_files: tuple[tuple[str, str], ...] + checkpoints: dict[tuple[str, str], SensorStreamPosition] + expected_ticks: tuple[tuple[str, str], ...] + expected_summary: SensorDispatchSummary diff --git a/tests/unit/src/sqltide/sensors/classes/helpers.py b/tests/unit/src/sqltide/sensors/classes/helpers.py index b6c61d05..d5a889a5 100644 --- a/tests/unit/src/sqltide/sensors/classes/helpers.py +++ b/tests/unit/src/sqltide/sensors/classes/helpers.py @@ -90,3 +90,19 @@ def __init__(self) -> None: def __call__(self) -> object: self.calls += 1 raise RuntimeError("slack unavailable") + + +CATALOG_EVENT_HANDLERS_SOURCE: str = """ +from sqltide.events import AuditCompleted, RunCompleted +from sqltide.sensors import DefaultSensorStatus, event_sensor + + +@event_sensor(on=AuditCompleted, default_status=DefaultSensorStatus.RUNNING) +def audit_watch(ctx): + assert type(ctx.event) is AuditCompleted + + +@event_sensor(on=RunCompleted, default_status=DefaultSensorStatus.RUNNING) +def run_watch(ctx): + assert type(ctx.event) is RunCompleted +""" diff --git a/tests/unit/src/sqltide/sensors/classes/test_sensor_dispatcher.py b/tests/unit/src/sqltide/sensors/classes/test_sensor_dispatcher.py index 01ce1155..8e057f47 100644 --- a/tests/unit/src/sqltide/sensors/classes/test_sensor_dispatcher.py +++ b/tests/unit/src/sqltide/sensors/classes/test_sensor_dispatcher.py @@ -1,5 +1,7 @@ """Behavior tests for one sequential sensor dispatch pass.""" +from pathlib import Path + import pytest from sqltide._native import SensorDispatcher @@ -12,8 +14,13 @@ TickAttemptState, ) from sqltide.sensors.types import SensorOverrideStatus, SensorTickStatus -from tests.unit.src.sqltide.events.helpers import build_node_result_observation +from tests.unit.src.sqltide.events.helpers import ( + build_invocation_observation, + build_node_result_observation, +) +from tests.unit.src.sqltide.sensors._helpers.helpers import load_registry, write_project_files from tests.unit.src.sqltide.sensors.classes._test_types import ( + CompiledSensorDispatchTestCase, DeadLetterActionTestCase, DispatcherScenarioTestCase, EventCheckpointInitializationTestCase, @@ -22,6 +29,7 @@ StreamHeadBoundTestCase, ) from tests.unit.src.sqltide.sensors.classes.helpers import ( + CATALOG_EVENT_HANDLERS_SOURCE, EPOCH_POSITION, ROW_POSITION, backoff_sensor, @@ -526,3 +534,46 @@ def test_given_dead_letter_when_skipping_then_a_skipped_tick_resolves_it( test_case.expected_tick_attempts ) assert repository.recorded_ticks[0].skip_reason == test_case.expected_skip_reason + + +@pytest.mark.parametrize( + "test_case", + [ + CompiledSensorDispatchTestCase( + description="sensors on events imported from sqltide.events receive those events", + sensor_files=(("sensors/catalog.py", CATALOG_EVENT_HANDLERS_SOURCE),), + checkpoints={ + ("audit_watch", "node_results"): EPOCH_POSITION, + ("run_watch", "invocations"): EPOCH_POSITION, + }, + expected_ticks=( + ("audit_watch", "started"), + ("audit_watch", "succeeded"), + ("run_watch", "started"), + ("run_watch", "succeeded"), + ), + expected_summary=SensorDispatchSummary(evaluated=2, succeeded=2), + ) + ], + ids=lambda case: case.description, +) +def test_given_compiled_catalog_event_sensors_when_dispatching_then_each_receives_its_event( + test_case: CompiledSensorDispatchTestCase, tmp_path: Path +) -> None: + write_project_files(project_dir=tmp_path, files=test_case.sensor_files) + registry: SensorRegistry = load_registry(project_dir=tmp_path) + repository: FakeSensorStateRepository = FakeSensorStateRepository( + checkpoints=test_case.checkpoints, + node_results=(build_node_result_observation(),), + invocations=(build_invocation_observation(),), + newest_position=SensorStreamPosition(completed_at="2024-01-01 00:00:09.000", result_id="z"), + ) + dispatcher: SensorDispatcher = SensorDispatcher(repository=repository, event_target="prod") + + summary: SensorDispatchSummary = dispatcher.dispatch_once(registry=registry, target="prod") + + assert summary == test_case.expected_summary + assert ( + tuple((tick.sensor_name, tick.status) for tick in repository.recorded_ticks) + == test_case.expected_ticks + ) diff --git a/tests/unit/src/sqltide/sensors/main/_test_types.py b/tests/unit/src/sqltide/sensors/main/_test_types.py index e579bf97..396eca91 100644 --- a/tests/unit/src/sqltide/sensors/main/_test_types.py +++ b/tests/unit/src/sqltide/sensors/main/_test_types.py @@ -17,3 +17,33 @@ class CompileSensorsErrorTestCase: sensor_files: tuple[tuple[str, str], ...] reserved_names: frozenset[str] expected_error_fragment: str + + +@dataclass(frozen=True) +class CompileSensorsEventTypeTestCase: + description: str + sensor_files: tuple[tuple[str, str], ...] + expected_event_types: tuple[tuple[str, type], ...] + + +@dataclass(frozen=True) +class CompileSensorsProviderErrorTestCase: + description: str + provider_files: tuple[tuple[str, str], ...] + expected_message: str + expected_cause_type: type + + +@dataclass(frozen=True) +class CompileSensorsProviderPathTestCase: + description: str + provider_files: tuple[tuple[str, str], ...] + expected_error_type: type[BaseException] + expected_context_type_name: str + + +@dataclass(frozen=True) +class CompileSensorsProviderReloadTestCase: + description: str + project_provider_names: tuple[tuple[str, str], ...] + expected_provider_names: tuple[str, ...] diff --git a/tests/unit/src/sqltide/sensors/main/helpers.py b/tests/unit/src/sqltide/sensors/main/helpers.py index 5f7dee38..6a9aa99a 100644 --- a/tests/unit/src/sqltide/sensors/main/helpers.py +++ b/tests/unit/src/sqltide/sensors/main/helpers.py @@ -1,6 +1,9 @@ from pathlib import Path from textwrap import dedent +from sqltide._native import compile_sensors +from sqltide.sensors.models import CompiledSensors + QUALITY_SENSOR_SOURCE: str = """ from sqltide.events import AuditCompleted from sqltide.sensors import event_sensor @@ -32,9 +35,130 @@ def homemade(ctx): \"\"\"React to a non-catalog event.\"\"\" """ +CATALOG_EVENT_SENSORS_SOURCE: str = """ +from sqltide.events import AuditCompleted, RunCompleted +from sqltide.sensors import event_sensor + + +@event_sensor(on=AuditCompleted) +def audit_watch(ctx): + \"\"\"React to audit completions.\"\"\" + + +@event_sensor(on=RunCompleted) +def run_watch(ctx): + \"\"\"React to run completions.\"\"\" +""" + +LOOKALIKE_EVENT_SENSOR_SOURCE: str = """ +from sqltide.sensors import event_sensor + + +class AuditCompleted: + pass + + +@event_sensor(on=AuditCompleted) +def lookalike(ctx): + \"\"\"React to a class that only shares the catalog event name.\"\"\" +""" + +DUPLICATE_PROVIDER_SOURCE: str = PROVIDER_SOURCE + +UNSET_SETTINGS_PROVIDER_SOURCE: str = """ +from sqltide.providers import Provider + + +class OrdersWebhook(Provider): + orders_webhook_url: str + orders_webhook_retry_limit: int = 3 + orders_webhook_timeout: float +""" + +UNPRINTABLE_ERROR_PROVIDER_SOURCE: str = """ +class UnprintableError(Exception): + def __str__(self) -> str: + raise ZeroDivisionError("unprintable") + + +raise UnprintableError() +""" + +BROKEN_PROVIDER_SOURCE: str = """ +VALUE = 1 +raise RuntimeError("cannot import") +""" + +INVALID_NAME_PROVIDER_SOURCE: str = """ +from typing import ClassVar + +from sqltide.providers import Provider + + +class BadName(Provider): + provider_name: ClassVar[str | None] = "Not Valid" +""" + +MIXED_PROVIDER_SOURCE: str = """ +from abc import abstractmethod + +from providers.zeta import OpsSlack +from sqltide.providers import Provider + + +class Helper: + pass + + +class AbstractNotifier(Provider): + @abstractmethod + def send(self, message: str) -> None: ... + + +class ConsoleNotifier(AbstractNotifier): + def send(self, message: str) -> None: + del message + + +class AuditLog(Provider): + pass +""" + +SHARED_NAME_PROVIDER_SOURCE: str = """ +from typing import ClassVar + +from providers._shared import PROVIDER_NAME +from sqltide.providers import Provider + + +class OpsSlack(Provider): + provider_name: ClassVar[str | None] = PROVIDER_NAME +""" + def write_project_files(*, project_dir: Path, files: tuple[tuple[str, str], ...]) -> None: for relative_path, contents in files: path: Path = project_dir / relative_path path.parent.mkdir(parents=True, exist_ok=True) path.write_text(dedent(contents).strip() + "\n", encoding="utf-8") + + +def compile_provider_names_in_turn( + *, root: Path, project_provider_names: tuple[tuple[str, str], ...] +) -> tuple[str, ...]: + names: list[str] = [] + for project_name, provider_name in project_provider_names: + project_dir: Path = root / project_name + write_project_files( + project_dir=project_dir, + files=( + ("providers/_shared.py", f'PROVIDER_NAME = "{provider_name}"'), + ("providers/slack.py", SHARED_NAME_PROVIDER_SOURCE), + ), + ) + compiled: CompiledSensors | None = compile_sensors( + project_dir=project_dir, reserved_names=frozenset() + ) + assert compiled is not None + names.extend(provider.name for provider in compiled.providers) + return tuple(names) diff --git a/tests/unit/src/sqltide/sensors/main/test_compile_sensors.py b/tests/unit/src/sqltide/sensors/main/test_compile_sensors.py index 6ed4525e..e9aa1032 100644 --- a/tests/unit/src/sqltide/sensors/main/test_compile_sensors.py +++ b/tests/unit/src/sqltide/sensors/main/test_compile_sensors.py @@ -1,20 +1,37 @@ """Behavior tests for compiling project sensors and providers.""" +import sys from pathlib import Path import pytest +from pydantic import ValidationError from sqltide._native import compile_sensors +from sqltide.events import AuditCompleted, RunCompleted +from sqltide.provider.exceptions import ProviderDiscoveryError, ProviderInputError from sqltide.sensors.exceptions import SensorError from sqltide.sensors.models import CompiledSensors from tests.unit.src.sqltide.sensors.main._test_types import ( CompileSensorsErrorTestCase, + CompileSensorsEventTypeTestCase, + CompileSensorsProviderErrorTestCase, + CompileSensorsProviderPathTestCase, + CompileSensorsProviderReloadTestCase, CompileSensorsTestCase, ) from tests.unit.src.sqltide.sensors.main.helpers import ( + BROKEN_PROVIDER_SOURCE, + CATALOG_EVENT_SENSORS_SOURCE, + DUPLICATE_PROVIDER_SOURCE, + INVALID_NAME_PROVIDER_SOURCE, + LOOKALIKE_EVENT_SENSOR_SOURCE, + MIXED_PROVIDER_SOURCE, PROVIDER_SOURCE, QUALITY_SENSOR_SOURCE, + UNPRINTABLE_ERROR_PROVIDER_SOURCE, + UNSET_SETTINGS_PROVIDER_SOURCE, UNSUPPORTED_EVENT_SENSOR_SOURCE, + compile_provider_names_in_turn, write_project_files, ) @@ -99,6 +116,12 @@ def test_given_authored_automation_when_compiling_then_artifact_is_complete( reserved_names=frozenset(), expected_error_fragment="Unsupported sensor event type 'HomemadeEvent'", ), + CompileSensorsErrorTestCase( + description="a class that only shares a catalog event name is not that event", + sensor_files=(("sensors/lookalike.py", LOOKALIKE_EVENT_SENSOR_SOURCE),), + reserved_names=frozenset(), + expected_error_fragment="Unsupported sensor event type 'AuditCompleted'", + ), ], ids=lambda case: case.description, ) @@ -111,3 +134,175 @@ def test_given_invalid_sensors_when_compiling_then_diagnostics_surface( _ = compile_sensors(project_dir=tmp_path, reserved_names=test_case.reserved_names) assert error_info.value.location is not None + + +@pytest.mark.parametrize( + "test_case", + [ + CompileSensorsEventTypeTestCase( + description="sensors on imported catalog events keep the catalog classes", + sensor_files=(("sensors/catalog.py", CATALOG_EVENT_SENSORS_SOURCE),), + expected_event_types=(("audit_watch", AuditCompleted), ("run_watch", RunCompleted)), + ) + ], + ids=lambda case: case.description, +) +def test_given_catalog_event_sensors_when_compiling_then_event_types_are_the_catalog_classes( + test_case: CompileSensorsEventTypeTestCase, tmp_path: Path +) -> None: + write_project_files(project_dir=tmp_path, files=test_case.sensor_files) + + compiled: CompiledSensors | None = compile_sensors( + project_dir=tmp_path, reserved_names=frozenset() + ) + + assert compiled is not None + observed: tuple[tuple[str, object], ...] = tuple( + (sensor.name, getattr(sensor.declaration, "event_type", None)) + for sensor in compiled.registry.ordered() + ) + assert len(observed) == len(test_case.expected_event_types) + assert all( + name == expected_name and event_type is expected_type + for (name, event_type), (expected_name, expected_type) in zip( + observed, test_case.expected_event_types, strict=True + ) + ) + + +@pytest.mark.parametrize( + "test_case", + [ + CompileSensorsTestCase( + description="providers are discovered by file then class name, skipping the rest", + sensor_files=(), + provider_files=( + ("providers/zeta.py", PROVIDER_SOURCE), + ("providers/alpha.py", MIXED_PROVIDER_SOURCE), + ("providers/_private.py", PROVIDER_SOURCE), + ("providers/__init__.py", ""), + ), + expected_provider_names=("audit_log", "console_notifier", "ops_slack"), + ) + ], + ids=lambda case: case.description, +) +def test_given_mixed_provider_modules_when_compiling_then_only_concrete_providers_are_found( + test_case: CompileSensorsTestCase, tmp_path: Path +) -> None: + write_project_files(project_dir=tmp_path, files=test_case.provider_files) + + compiled: CompiledSensors | None = compile_sensors( + project_dir=tmp_path, reserved_names=test_case.reserved_names + ) + + assert compiled is not None + assert ( + tuple(provider.name for provider in compiled.providers) == test_case.expected_provider_names + ) + + +@pytest.mark.parametrize( + "test_case", + [ + CompileSensorsProviderErrorTestCase( + description="provider names must be unique across provider files", + provider_files=( + ("providers/alerts.py", DUPLICATE_PROVIDER_SOURCE), + ("providers/slack.py", DUPLICATE_PROVIDER_SOURCE), + ), + expected_message=( + "Duplicate provider name 'ops_slack' found in providers/alerts.py and " + "providers/slack.py" + ), + expected_cause_type=type(None), + ), + CompileSensorsProviderErrorTestCase( + description="unset provider settings list every failing field", + provider_files=(("providers/orders.py", UNSET_SETTINGS_PROVIDER_SOURCE),), + expected_message=( + "Provider 'orders_webhook' in providers/orders.py has invalid settings:\n" + "orders_webhook_url: Field required [missing]\n" + "orders_webhook_timeout: Field required [missing]" + ), + expected_cause_type=ValidationError, + ), + CompileSensorsProviderErrorTestCase( + description="provider import failures name the provider file", + provider_files=(("providers/broken.py", BROKEN_PROVIDER_SOURCE),), + expected_message="Failed to import provider file providers/broken.py: cannot import", + expected_cause_type=RuntimeError, + ), + CompileSensorsProviderErrorTestCase( + description="invalid provider names name the provider class and file", + provider_files=(("providers/bad.py", INVALID_NAME_PROVIDER_SOURCE),), + expected_message=( + "Provider class BadName in providers/bad.py has an invalid provider name: " + "provider BadName has invalid provider name 'Not Valid'; provider names must " + "be lower snake_case Python identifiers" + ), + expected_cause_type=ProviderInputError, + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_providers_when_compiling_then_discovery_error_explains_the_failure( + test_case: CompileSensorsProviderErrorTestCase, tmp_path: Path +) -> None: + write_project_files(project_dir=tmp_path, files=test_case.provider_files) + + with pytest.raises(ProviderDiscoveryError) as error_info: + _ = compile_sensors(project_dir=tmp_path, reserved_names=frozenset()) + + assert error_info.value.args == (test_case.expected_message,) + assert type(error_info.value.__cause__) is test_case.expected_cause_type + + +@pytest.mark.parametrize( + "test_case", + [ + CompileSensorsProviderPathTestCase( + description="an import error whose message cannot be rendered still restores sys.path", + provider_files=(("providers/unprintable.py", UNPRINTABLE_ERROR_PROVIDER_SOURCE),), + expected_error_type=ZeroDivisionError, + expected_context_type_name="UnprintableError", + ) + ], + ids=lambda case: case.description, +) +def test_given_unprintable_provider_import_error_when_compiling_then_sys_path_is_restored( + test_case: CompileSensorsProviderPathTestCase, tmp_path: Path +) -> None: + write_project_files(project_dir=tmp_path, files=test_case.provider_files) + path_before: list[str] = list(sys.path) + + with pytest.raises(test_case.expected_error_type) as error_info: + _ = compile_sensors(project_dir=tmp_path, reserved_names=frozenset()) + + assert sys.path == path_before + assert type(error_info.value.__context__).__name__ == test_case.expected_context_type_name + + +@pytest.mark.parametrize( + "test_case", + [ + CompileSensorsProviderReloadTestCase( + description="a second project reloads its own provider package modules", + project_provider_names=( + ("first", "alpha_slack"), + ("second", "beta_slack"), + ("first", "alpha_slack"), + ), + expected_provider_names=("alpha_slack", "beta_slack", "alpha_slack"), + ) + ], + ids=lambda case: case.description, +) +def test_given_two_projects_when_compiling_in_turn_then_stale_provider_modules_are_evicted( + test_case: CompileSensorsProviderReloadTestCase, tmp_path: Path +) -> None: + names: tuple[str, ...] = compile_provider_names_in_turn( + root=tmp_path, project_provider_names=test_case.project_provider_names + ) + + assert names == test_case.expected_provider_names diff --git a/tests/unit/src/sqltide/sensors/test_models.py b/tests/unit/src/sqltide/sensors/test_models.py new file mode 100644 index 00000000..97d18ae8 --- /dev/null +++ b/tests/unit/src/sqltide/sensors/test_models.py @@ -0,0 +1,44 @@ +"""Behavior tests for the authored sensor policy shapes.""" + +import pytest + +from sqltide.sensors import SensorRetryPolicy +from sqltide.sensors.exceptions import SensorError +from tests.unit.src.sqltide.sensors._test_types import SensorRetryPolicyErrorTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + SensorRetryPolicyErrorTestCase( + description="a retry policy needs at least one attempt", + max_attempts=0, + backoff_seconds=0.0, + expected_message="SensorRetryPolicy.max_attempts must be at least 1", + ), + SensorRetryPolicyErrorTestCase( + description="a retry policy rejects a negative backoff", + max_attempts=3, + backoff_seconds=-0.5, + expected_message="SensorRetryPolicy.backoff_seconds must not be negative", + ), + SensorRetryPolicyErrorTestCase( + description="the attempt count is checked before the backoff", + max_attempts=0, + backoff_seconds=-1.0, + expected_message="SensorRetryPolicy.max_attempts must be at least 1", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_retry_policy_when_constructing_then_sensor_error_names_the_field( + test_case: SensorRetryPolicyErrorTestCase, +) -> None: + with pytest.raises(SensorError) as error_info: + _ = SensorRetryPolicy( + max_attempts=test_case.max_attempts, backoff_seconds=test_case.backoff_seconds + ) + + assert error_info.value.args == (test_case.expected_message,) + assert str(error_info.value) == test_case.expected_message + assert error_info.value.location is None