From 9417ec03f856d7804ac1b00ba4d0ababa32ddc0a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:32:59 +0000 Subject: [PATCH 1/3] refactor: move executor operation shapes to the native core Destruction, readiness, deployment, auditing, observability, janitor, reconcile, testing and doctor shapes, enums, exceptions and constants are native classes; the old modules re-export them. The destruction types module keeps its plan-store and planning-connection protocols and clock alias. Movable Python drops from 4784 to 3747 lines. --- .../sqltide-python/src/shapes/executor_ops.rs | 992 ++++++++++++++++++ crates/sqltide-python/src/shapes/mod.rs | 17 +- .../src/shapes/planner_execution.rs | 2 +- python-boundary.toml | 7 +- scripts/shape_surface/constants.py | 9 + src/sqltide/_native.pyi | 559 +++++++++- src/sqltide/executor/auditing/constants.py | 2 +- src/sqltide/executor/auditing/exceptions.py | 4 +- src/sqltide/executor/auditing/models.py | 70 +- src/sqltide/executor/auditing/types.py | 28 +- src/sqltide/executor/deployment/constants.py | 6 +- src/sqltide/executor/deployment/exceptions.py | 9 +- src/sqltide/executor/deployment/models.py | 78 +- src/sqltide/executor/deployment/types.py | 24 +- src/sqltide/executor/destruction/constants.py | 8 +- .../executor/destruction/exceptions.py | 90 +- src/sqltide/executor/destruction/models.py | 183 +--- src/sqltide/executor/destruction/types.py | 36 +- src/sqltide/executor/doctor/models.py | 28 +- src/sqltide/executor/janitor/exceptions.py | 4 +- src/sqltide/executor/janitor/models.py | 50 +- .../executor/observability/constants.py | 12 +- src/sqltide/executor/observability/models.py | 51 +- src/sqltide/executor/observability/types.py | 19 +- src/sqltide/executor/readiness/constants.py | 4 +- src/sqltide/executor/readiness/exceptions.py | 4 +- src/sqltide/executor/readiness/models.py | 197 +--- src/sqltide/executor/readiness/types.py | 8 +- src/sqltide/executor/reconcile/constants.py | 2 +- src/sqltide/executor/reconcile/exceptions.py | 4 +- src/sqltide/executor/reconcile/models.py | 44 +- src/sqltide/executor/testing/exceptions.py | 4 +- src/sqltide/executor/testing/models.py | 39 +- 33 files changed, 1631 insertions(+), 963 deletions(-) create mode 100644 crates/sqltide-python/src/shapes/executor_ops.rs diff --git a/crates/sqltide-python/src/shapes/executor_ops.rs b/crates/sqltide-python/src/shapes/executor_ops.rs new file mode 100644 index 00000000..2b2700d5 --- /dev/null +++ b/crates/sqltide-python/src/shapes/executor_ops.rs @@ -0,0 +1,992 @@ +//! Shapes of the operational executor commands: destruction, readiness audits, deployment +//! inventory and diffs, SQL audits, run observability, janitor, reconcile, SQL tests and doctor, +//! plus their value enums, exceptions and constants. + +use pyo3::exceptions::{PyRuntimeError, PyTypeError, PyValueError}; +use pyo3::prelude::*; +use pyo3::types::{IntoPyDict, PyFrozenSet, PySet, PyTuple}; + +use super::{ + Coerce, Given, add, add_class, base_init, exception, factory, native_enum, shapes, str_enum, +}; + +const DESTRUCTION_EXCEPTIONS: &str = "sqltide.executor.destruction.exceptions"; +const REPLAY_LINEAGE_MODE: Coerce = Coerce::Maybe(&Coerce::Enum("ReplayLineageMode")); + +/// The native exception `name` raised with `message`. +fn raised(py: Python<'_>, name: &str, message: String) -> PyResult { + Ok(PyErr::from_value( + py.import("sqltide._native")? + .getattr(name)? + .call1((message,))?, + )) +} + +/// `DestructionValidationError(message)`. +fn invalid_destruction(py: Python<'_>, message: &str) -> PyResult { + raised(py, "DestructionValidationError", message.to_owned()) +} + +/// `tuple(sorted(value))`. +fn sorted_tuple<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + let sorted: Bound<'py, PyAny> = py.import("builtins")?.getattr("sorted")?.call1((value,))?; + py.get_type::().call1((sorted,)) +} + +/// `DestructionRequest`: orphans only with pipeline destruction, unique and disjoint pipeline +/// selections, each then kept sorted and deduplicated. +fn destruction_request(values: &mut [Bound<'_, PyAny>]) -> PyResult<()> { + let [ + operation, + _, + _, + _, + pipeline_names, + included, + include_orphans, + ] = values + else { + return Err(PyValueError::new_err( + "destruction request bound the wrong number of values", + )); + }; + let py: Python<'_> = operation.py(); + let set: Bound<'_, PyAny> = py.get_type::().into_any(); + if include_orphans.is_truthy()? + && operation.ne(native_enum(py, "DestructionOperation")?.getattr("DESTROY_PIPELINES")?)? + { + return Err(invalid_destruction( + py, + "Orphan inclusion is only valid for selected pipeline destruction", + )?); + } + if set.call1((&*pipeline_names,))?.len()? != pipeline_names.len()? { + return Err(invalid_destruction( + py, + "Pipeline destruction selection contains duplicate names", + )?); + } + if set.call1((&*included,))?.len()? != included.len()? { + return Err(invalid_destruction( + py, + "Included dependent pipeline selection contains duplicate names", + )?); + } + let overlap: Bound<'_, PyAny> = set + .call1((&*pipeline_names,))? + .bitand(set.call1((&*included,))?)?; + if overlap.is_truthy()? { + return Err(raised( + py, + "DestructionValidationError", + format!( + "Pipelines cannot be both originally selected and included dependants: {}", + sorted_tuple(&overlap)?.repr()? + ), + )?); + } + *pipeline_names = sorted_tuple(&set.call1((&*pipeline_names,))?)?; + *included = sorted_tuple(&set.call1((&*included,))?)?; + Ok(()) +} + +/// `JanitorRequest`: retention and the kept rollback deployments are non-negative. +fn janitor_request(values: &mut [Bound<'_, PyAny>]) -> PyResult<()> { + let [_, _, retention_days, _, minimum_rollback_deployments] = values else { + return Err(PyValueError::new_err( + "janitor request bound the wrong number of values", + )); + }; + let py: Python<'_> = retention_days.py(); + if retention_days.lt(0)? { + return Err(raised( + py, + "JanitorInputError", + "retention_days must be non-negative".to_owned(), + )?); + } + if minimum_rollback_deployments.lt(0)? { + return Err(raised( + py, + "JanitorInputError", + "minimum_rollback_deployments must be non-negative".to_owned(), + )?); + } + Ok(()) +} + +/// `relation.total_bytes or 0`, one relation's share of a plan's estimated bytes. +#[pyfunction] +fn relation_bytes<'py>(relation: &Bound<'py, PyAny>) -> PyResult> { + let total: Bound<'py, PyAny> = relation.getattr("total_bytes")?; + if total.is_truthy()? { + return Ok(total); + } + Ok(0_i64.into_pyobject(relation.py())?.into_any()) +} + +/// How many audits failed at `severity` without being deferred. +fn failure_count(results: &Bound<'_, PyAny>, severity: &str) -> PyResult { + let py: Python<'_> = results.py(); + let wanted: Bound<'_, PyAny> = native_enum(py, "AuditSeverity")?.getattr(severity)?; + let mut count: usize = 0; + for result in results.try_iter()? { + let result: Bound<'_, PyAny> = result?; + if result.getattr("severity")?.eq(&wanted)? + && !result.getattr("passed")?.is_truthy()? + && !result.getattr("deferred")?.is_truthy()? + { + count += 1; + } + } + Ok(count) +} + +shapes! { + module = "sqltide.executor.destruction.models"; + + DestructionRequest { + operation(coerce = Coerce::Enum("DestructionOperation")), + target, + database, + metadata_database, + pipeline_names(default = Given::EmptyTuple), + included_dependent_pipeline_names(default = Given::EmptyTuple), + include_orphans(default = Given::Bool(false)), + } + @post_init(destruction_request) + + /// A historically published pipeline absent from the compiled project. + InactivePipeline { name, model_count, resource_count, last_published_at } + + DestructionRelationEvidence { + database, + name, + kind(coerce = Coerce::Enum("DestructionRelationKind")), + exists, + total_bytes, + active_parts, + catalog_fingerprint, + logical_names, + pipeline_names, + ownership, + dependency_relation_names, + } + + DestructionPlan { + plan_id, + operation, + target, + database, + metadata_database, + requested_pipeline_names, + included_dependent_pipeline_names, + affected_pipeline_names, + affected_model_names, + affected_source_names, + relations, + challenges, + preserves_sources, + preserves_replay_data, + manifest_fingerprint, + plan_fingerprint, + created_at, + expires_at, + relation_drop_size_limit(default = Given::None), + relation_drop_size_server_limit(default = Given::None), + relation_drop_size_override(default = Given::None), + relation_drop_size_policy_observed(default = Given::Bool(false)), + include_orphans(default = Given::Bool(false)), + } + @impl { + #[getter] + fn estimated_bytes<'py>(&self, py: Python<'py>) -> PyResult> { + let builtins: Bound<'py, PyModule> = py.import("builtins")?; + let shares: Bound<'py, PyAny> = builtins + .getattr("map")? + .call1((wrap_pyfunction!(relation_bytes, py)?, self.relations.bind(py)))?; + builtins.getattr("sum")?.call1((shares,)) + } + } + + /// Terminal recorded outcome for one consumed frozen plan. + DestructionExecutionResult { + invocation_id, + outcome, + completed_statement_sequences, + pending_statement_sequences, + remaining_relation_names, + error_message, + residual_catalog_status(default = Given::Str("observed")), + residual_catalog_error(default = Given::None), + } + + /// Authenticated actor identity recorded with destructive execution evidence. + DestructionActor { actor_id, actor_name } + + /// Immutable identity and connections shared by destructive recording phases. + DestructionRecordingContext { + plan, + actor_id, + actor_name, + reviewed_at, + confirmed_at, + challenge_responses, + connection, + observation_connection, + project_dir, + started, + statements, + sink, + } +} + +shapes! { + module = "sqltide.executor.readiness.models"; + + /// Effective thresholds used to assess one staged deployment. + DeploymentReadinessThresholds { + maximum_lag_seconds(default = Given::Float(30.0)), + minimum_staged_row_ratio(default = Given::Float(0.5)), + } + + /// Input required to audit a staged backfill deployment. + DeploymentAuditRequest { + deployment_id, + metadata_database, + default_database, + thresholds(default = Given::Factory(factory::)), + } + + /// A deployment candidate surfaced when audit resolution is ambiguous. + AuditDeploymentCandidate { + deployment_id, + created_at(default = Given::None), + deployment_status(default = Given::None), + root_names(default = Given::EmptyTuple), + } + + /// Persisted deployment metadata used by metadata-backed flows. + LoadedAuditDeployment { + deployment_id, + created_at, + status, + replay_lineage_mode(coerce = REPLAY_LINEAGE_MODE), + warning_codes, + root_keys, + prepared_object_mappings, + } + + /// Partitioned catch-up metrics for offset-based replay. + OffsetCatchupSummary { + active_partition_count, + staged_partition_count, + partitions_compared, + missing_staged_partition_count, + missing_freshness_partition_count, + lagging_partition_count, + max_offset_gap, + average_offset_gap, + lag_boundary_column, + max_lag_seconds, + average_lag_seconds, + } + + /// Scalar catch-up metrics for timestamp-based replay. + ScalarCatchupSummary { + active_min_value, + active_max_value, + staged_min_value, + staged_max_value, + lag_seconds, + } + + /// Audit summary for one rebuilt root target. + RootAuditResult { + root_key, + staged_physical_name, + staged_exists, + active_exists, + active_row_count, + staged_row_count, + row_delta, + row_ratio, + assessment(coerce = Coerce::Enum("AuditAssessment")), + replay_lineage_mode(coerce = REPLAY_LINEAGE_MODE), + offset_catchup_summary, + scalar_catchup_summary, + state, + replay_source_name, + replay_source_row_count, + warnings(default = Given::EmptyTuple), + } + + /// Structured audit result for a staged backfill deployment. + DeploymentAuditResult { + deployment_id, + deployment_status, + assessment(coerce = Coerce::Enum("AuditAssessment")), + replay_lineage_mode(coerce = REPLAY_LINEAGE_MODE), + warning_codes, + root_results, + quality_check_results(default = Given::EmptyTuple), + thresholds(default = Given::Factory(factory::)), + } +} + +shapes! { + module = "sqltide.executor.deployment.models"; + + /// One deployment reconstructed from authoritative warehouse evidence. + DeploymentSummary { + deployment_id, + state, + created_at, + persisted_status, + root_names, + physical_relation_names, + missing_physical_relation_names, + active_binding_names, + latest_published_at, + } + + /// Deterministically ordered deployment summaries for one target database. + DeploymentInventory { database, deployments } + + /// One active/deployment comparison expression. + DeploymentDiffRequest { database, metadata_database, comparison } + + /// One catalog column included in a deployment comparison. + DeploymentDiffColumn { name, r#type, default_expression } + + /// Schema and count comparison for one logical relation. + DeploymentDiffRelation { + database, + logical_name, + status(coerce = Coerce::Enum("DeploymentDiffStatus")), + from_physical_name, + to_physical_name, + from_columns, + to_columns, + from_row_count, + to_row_count, + } + + /// Resolved endpoints and per-relation deployment differences. + DeploymentDiffResult { database, from_endpoint, to_endpoint, relations } +} + +shapes! { + module = "sqltide.executor.auditing.models"; + + /// One executed SQL audit and its outcome. + SqlAuditResult { + file_path, + referenced_model_names, + severity, + passed, + failing_row_count, + sample_column_names, + sample_rows, + description(default = Given::None), + name(default = Given::None), + error_message(default = Given::None), + deferred_until(default = Given::None), + missing_relation_names(default = Given::EmptyTuple), + } + @impl { + /// Return whether execution was deferred for warmup or materialization. + #[getter] + fn deferred(&self, py: Python<'_>) -> PyResult { + Ok(!self.deferred_until.is_none(py) + || self.missing_relation_names.bind(py).is_truthy()?) + } + } + + /// Aggregate outcome for a batch of SQL audits. + SqlAuditRunResult { audit_results } + @impl { + #[getter] + fn error_failure_count(&self, py: Python<'_>) -> PyResult { + failure_count(self.audit_results.bind(py), "ERROR") + } + + #[getter] + fn warning_failure_count(&self, py: Python<'_>) -> PyResult { + failure_count(self.audit_results.bind(py), "WARNING") + } + } + + /// Eligibility of one audit relative to its newest referenced model anchor. + AuditWarmupState { + eligible, + anchor, + eligible_at, + missing_model_names(default = Given::EmptyTuple), + } +} + +shapes! { + module = "sqltide.executor.observability.models"; + + /// Pre-execution work completed before the first durable run event. + RunStartupTimings { compile_ms, observability_ms, planning_ms } + @impl { + #[getter] + fn total_ms<'py>(&self, py: Python<'py>) -> PyResult> { + self.compile_ms + .bind(py) + .add(self.observability_ms.bind(py))? + .add(self.planning_ms.bind(py)) + } + } + + /// Command outcome fields captured at one finite CLI boundary. + TerminalInvocation { + project_dir, + target_identity, + command, + mode, + outcome, + exit_code, + materialized_outcome, + deployment_id, + workflow_id, + selected_node_count, + error_message, + summary, + } + + /// Trigger, logical slot, and policy snapshot for one quality attempt. + QualityResultContext { + trigger(coerce = Coerce::Enum("QualityResultTrigger")), + scheduled_for(default = Given::None), + cadence_seconds(default = Given::None), + warmup_seconds(default = Given::Int(0)), + } +} + +shapes! { + module = "sqltide.executor.janitor.models"; + + JanitorPreviewCandidate { + deployment_id, + created_at, + status, + logical_view_names, + physical_object_names, + deletable, + reason, + } + + JanitorPreviewResult { + database, + retention_days, + minimum_rollback_deployments, + candidates, + } + + JanitorApplyResult { + database, + retention_days, + minimum_rollback_deployments, + deleted_deployment_ids, + deleted_object_names, + } + + JanitorRequest { + database, + metadata_database, + retention_days, + apply, + minimum_rollback_deployments(default = Given::Int(2)), + } + @post_init(janitor_request) +} + +shapes! { + module = "sqltide.executor.reconcile.models"; + + ReconcileRejectedTarget { target_key, target_name, reasons } + + ReconcilePreview { database, reconcile_id, eligible_records, rejected_targets } + + ReconcileResult { database, reconcile_id, reconciled_records, rejected_targets } +} + +shapes! { + module = "sqltide.executor.testing.models"; + + /// One executed target comparison inside a SQL-native test scenario. + SqlTestTargetExecutionResult { + target_model_name, + passed, + column_names, + missing_rows, + unexpected_rows, + } + + /// One executed SQL test scenario and its aggregate diff outcome. + SqlTestExecutionResult { + file_path, + passed, + target_results, + executed_sql, + warnings(default = Given::EmptyTuple), + name(default = Given::None), + test_index(default = Given::Int(1)), + error_message(default = Given::None), + } +} + +shapes! { + module = "sqltide.executor.doctor.models"; + + /// Input required to inspect active-view health. + DoctorRequest { default_database } + + /// Diagnosis for one logical managed table. + ActiveViewStatus { + table_name, + state_kind, + active_deployment_id, + candidate_deployment_ids, + } + + /// Read-only diagnosis of active-view health. + DoctorResult { active_views } +} + +/// A native enum: its class, public module, docstring and members. +type EnumSpec = ( + &'static str, + &'static str, + Option<&'static str>, + &'static [(&'static str, &'static str)], +); + +const ENUMS: [EnumSpec; 11] = [ + ( + "DestructionOperation", + "sqltide.executor.destruction.types", + None, + &[ + ("DESTROY_PIPELINES", "destroy_pipelines"), + ("RESET_TARGET", "reset_target"), + ], + ), + ( + "DestructionRelationKind", + "sqltide.executor.destruction.types", + None, + &[ + ("VIEW", "view"), + ("MATERIALIZED_VIEW", "materialized_view"), + ("TABLE", "table"), + ("MANAGED_SOURCE", "managed_source"), + ], + ), + ( + "DestructionOwnership", + "sqltide.executor.destruction.types", + None, + &[ + ("CURRENT_MANIFEST", "current_manifest"), + ("VIRTUAL_PHYSICAL_MAPPING", "virtual_physical_mapping"), + ("PUBLISHED_STABLE_BINDING", "published_stable_binding"), + ("OWNERSHIP_LEDGER", "ownership_ledger"), + ("HISTORICAL_MANIFEST", "historical_manifest"), + ], + ), + ( + "DestructionPlanningErrorCode", + "sqltide.executor.destruction.types", + Some("Why the native planner rejected a destruction request."), + &[ + ("SELECTION", "selection"), + ("UNKNOWN_PIPELINES", "unknown_pipelines"), + ("DEPENDENCY", "dependency"), + ("RESOURCE", "resource"), + ("UNSUPPORTED_HISTORICAL_KIND", "unsupported_historical_kind"), + ("INVALID_OBJECT_TYPE", "invalid_object_type"), + ], + ), + ( + "AuditAssessment", + "sqltide.executor.readiness.types", + None, + &[ + ("READY", "ready"), + ("NOT_READY", "not_ready"), + ("CAUTION", "caution"), + ], + ), + ( + "DeploymentLifecycleState", + "sqltide.executor.deployment.types", + Some("Authoritative state derived from metadata and live catalog evidence."), + &[ + ("ACTIVE", "active"), + ("STAGED", "staged"), + ("SUPERSEDED", "superseded"), + ("INCOMPLETE", "incomplete"), + ("METADATA_MISSING", "metadata_missing"), + ("PHYSICAL_MISSING", "physical_missing"), + ], + ), + ( + "DeploymentDiffStatus", + "sqltide.executor.deployment.types", + Some("Summary classification for one logical relation comparison."), + &[ + ("ADDED", "added"), + ("REMOVED", "removed"), + ("CHANGED", "changed"), + ("UNCHANGED", "unchanged"), + ("PHYSICAL_MISSING", "physical_missing"), + ], + ), + ( + "AuditSeverity", + "sqltide.executor.auditing.types", + Some("How a failing audit should be reported."), + &[("ERROR", "error"), ("WARNING", "warning")], + ), + ( + "QualityResultStatus", + "sqltide.executor.auditing.types", + Some("Persisted status of one audit or SQL-test attempt."), + &[ + ("PASSED", "passed"), + ("WARNING", "warning"), + ("FAILED", "failed"), + ("ERROR", "error"), + ("DEFERRED", "deferred"), + ], + ), + ( + "QualityResultTrigger", + "sqltide.executor.observability.types", + Some("The execution path that produced one quality result."), + &[ + ("BUILD", "build"), + ("MANUAL", "manual"), + ("SCHEDULED", "scheduled"), + ("DEPLOYMENT", "deployment"), + ], + ), + ( + "MaterializationOutcome", + "sqltide.executor.observability.types", + Some("Whether a direct mutation workflow safely completed."), + &[("APPLIED", "applied"), ("FAILED", "failed")], + ), +]; + +fn enums(module: &Bound<'_, PyModule>) -> PyResult<()> { + for (name, public_module, doc, members) in ENUMS { + str_enum(module, name, public_module, name, doc, members)?; + } + Ok(()) +} + +#[pyfunction] +#[pyo3(name = "__init__")] +fn destruction_dependency_init<'py>( + instance: &Bound<'py, PyAny>, + dependent_pipeline_names: Bound<'py, PyAny>, +) -> PyResult<()> { + let message: String = format!( + "Unselected dependent or shared-source pipelines must be explicitly included: {}", + dependent_pipeline_names.repr()? + ); + instance.setattr("dependent_pipeline_names", dependent_pipeline_names)?; + base_init(instance, [message.into_pyobject(instance.py())?.into_any()]) +} + +#[pyfunction] +#[pyo3(name = "__init__")] +fn destruction_external_dependency_init<'py>( + instance: &Bound<'py, PyAny>, + relation_names: Bound<'py, PyAny>, +) -> PyResult<()> { + let message: String = format!( + "Unmanaged warehouse dependants block destruction: {}", + relation_names.repr()? + ); + instance.setattr("relation_names", relation_names)?; + base_init(instance, [message.into_pyobject(instance.py())?.into_any()]) +} + +/// The destruction errors under `DestructionError` without their own constructor: name, base +/// and docstring. +const DESTRUCTION_ERRORS: [(&str, &str, &str); 10] = [ + ( + "DestructionConsistencyError", + "DestructionError", + "Internal destruction evidence became inconsistent.", + ), + ( + "DestructionPlanNotFoundError", + "DestructionError", + "The requested frozen plan does not exist or was already consumed.", + ), + ( + "DestructionPlanExpiredError", + "DestructionError", + "The requested frozen plan has expired.", + ), + ( + "DestructionPlanNotReviewedError", + "DestructionError", + "The frozen plan has not passed the explicit review gate.", + ), + ( + "DestructionPlanCorruptError", + "DestructionError", + "A durable frozen-plan record is incompatible or corrupt.", + ), + ( + "DestructionChallengeError", + "DestructionError", + "Typed challenge responses do not exactly match the frozen plan.", + ), + ( + "DestructionDriftError", + "DestructionError", + "A freshly planned operation differs from its frozen plan.", + ), + ( + "DestructionRecordingError", + "DestructionError", + "Required destructive-operation evidence could not be persisted.", + ), + ( + "DestructionRecoveryError", + "DestructionError", + "A failed run does not contain valid recoverable destruction intent.", + ), + ( + "DestructionRecoveryNotFoundError", + "DestructionRecoveryError", + "A run is not a recoverable operation in the active project.", + ), +]; + +/// The other executor errors: public module, name, docstring and whether `RuntimeError` rather +/// than `ValueError` is the base. +const EXECUTOR_ERRORS: [(&str, &str, &str, bool); 7] = [ + ( + "sqltide.executor.readiness.exceptions", + "DeploymentAuditExecutionError", + "Raised when audit backfill input or state is invalid.", + false, + ), + ( + "sqltide.executor.deployment.exceptions", + "DeploymentNotFoundError", + "Raised when an explicit deployment identifier is not present.", + false, + ), + ( + "sqltide.executor.deployment.exceptions", + "DeploymentDiffError", + "Raised when a deployment comparison cannot be resolved safely.", + false, + ), + ( + "sqltide.executor.auditing.exceptions", + "AuditExecutionError", + "Raised when auditing input or state is invalid.", + false, + ), + ( + "sqltide.executor.janitor.exceptions", + "JanitorInputError", + "Raised when janitor retention input is invalid.", + false, + ), + ( + "sqltide.executor.reconcile.exceptions", + "ReconcileError", + "Raised when verified reconcile state cannot be persisted safely.", + true, + ), + ( + "sqltide.executor.testing.exceptions", + "SqlTestExecutionError", + "Raised when testing input or state is invalid.", + false, + ), +]; + +fn exceptions(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py: Python<'_> = module.py(); + let value_error: Bound<'_, PyAny> = py.get_type::().into_any(); + let runtime_error: Bound<'_, PyAny> = py.get_type::().into_any(); + let base: Bound<'_, PyAny> = exception( + module, + DESTRUCTION_EXCEPTIONS, + "DestructionError", + "Base error for the destruction domain.", + std::slice::from_ref(&runtime_error), + None, + None, + )?; + let selection: Bound<'_, PyAny> = exception( + module, + DESTRUCTION_EXCEPTIONS, + "DestructionSelectionError", + "The requested pipeline selection is invalid.", + std::slice::from_ref(&base), + None, + None, + )?; + exception( + module, + DESTRUCTION_EXCEPTIONS, + "DestructionValidationError", + "A destruction value violates a domain invariant.", + &[base.clone(), value_error.clone()], + None, + None, + )?; + exception( + module, + DESTRUCTION_EXCEPTIONS, + "DestructionResourceError", + "A manifest resource cannot participate in destruction.", + &[base.clone(), py.get_type::().into_any()], + None, + None, + )?; + exception( + module, + DESTRUCTION_EXCEPTIONS, + "DestructionDependencyError", + "Unselected downstream pipelines make destruction unsafe.", + std::slice::from_ref(&selection), + Some(wrap_pyfunction!(destruction_dependency_init, module)?.into_any()), + None, + )?; + exception( + module, + DESTRUCTION_EXCEPTIONS, + "DestructionExternalDependencyError", + "Warehouse relations outside the owned scope depend on planned relations.", + std::slice::from_ref(&selection), + Some(wrap_pyfunction!(destruction_external_dependency_init, module)?.into_any()), + None, + )?; + for (name, parent, doc) in DESTRUCTION_ERRORS { + let parent: Bound<'_, PyAny> = module.getattr(parent)?; + exception( + module, + DESTRUCTION_EXCEPTIONS, + name, + doc, + &[parent], + None, + None, + )?; + } + for (public_module, name, doc, runtime) in EXECUTOR_ERRORS { + let parent: &Bound<'_, PyAny> = if runtime { + &runtime_error + } else { + &value_error + }; + exception( + module, + public_module, + name, + doc, + std::slice::from_ref(parent), + None, + None, + )?; + } + Ok(()) +} + +const TEXT: [(&str, &str); 5] = [ + ("ACTIVE_DEPLOYMENT_DIFF_ENDPOINT", "active"), + ("RUN_DISPLAY_COMMAND_ENV_VAR", "SQLTIDE_RUN_DISPLAY_COMMAND"), + ("RUN_INVOCATION_ID_ENV_VAR", "SQLTIDE_RUN_INVOCATION_ID"), + ("PROJECT_REVISION_ENV_VAR", "SQLTIDE_PROJECT_REVISION"), + ("RECONCILE_DEPLOYMENT_ID_PREFIX", "reconcile_"), +]; + +fn constants(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py: Python<'_> = module.py(); + add( + module, + "DEFAULT_DESTRUCTION_PLAN_TTL", + py.import("datetime")? + .getattr("timedelta")? + .call((), Some(&[("minutes", 15)].into_py_dict(py)?))?, + )?; + add(module, "DESTRUCTION_PLAN_PAYLOAD_VERSION", 1)?; + add( + module, + "PRODUCTION_TARGET_NAMES", + PyFrozenSet::new(py, ["prod", "production"])?, + )?; + add(module, "ACCEPTABLE_LAG_SECONDS", 30.0)?; + add(module, "MINIMUM_STAGED_ROW_RATIO", 0.5)?; + add(module, "EXPLICIT_DEPLOYMENT_DIFF_ENDPOINT_COUNT", 2)?; + add(module, "AUDIT_SAMPLE_LIMIT", 5)?; + add(module, "MAX_OBSERVATION_JSON_BYTES", 16_384)?; + add(module, "MAX_OBSERVATION_ERROR_LENGTH", 1_048_576)?; + add(module, "HEARTBEAT_INTERVAL_SECONDS", 10.0)?; + for (name, value) in TEXT { + add(module, name, value)?; + } + Ok(()) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + enums(module)?; + exceptions(module)?; + constants(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module) +} diff --git a/crates/sqltide-python/src/shapes/mod.rs b/crates/sqltide-python/src/shapes/mod.rs index 27b34ca2..1d4b936e 100644 --- a/crates/sqltide-python/src/shapes/mod.rs +++ b/crates/sqltide-python/src/shapes/mod.rs @@ -17,6 +17,7 @@ use pyo3::types::{ mod compiler_analysis; mod compiler_project; +mod executor_ops; mod planner_execution; /// The callable a `default_factory` field calls for each new instance. @@ -77,10 +78,11 @@ impl FieldSpec { }; } -/// A check run on the bound and normalized field values, as the rest of `__post_init__` did. -pub(crate) type PostInit = for<'py> fn(&[Bound<'py, PyAny>]) -> PyResult<()>; +/// The rest of `__post_init__`, run on the bound and normalized field values: it may reject them +/// or replace them in place. +pub(crate) type PostInit = for<'py> fn(&mut [Bound<'py, PyAny>]) -> PyResult<()>; -/// One shape: its class name, fields in declaration order and optional post-init check. +/// One shape: its class name, fields in declaration order and optional post-init hook. pub(crate) struct ShapeSpec { pub(crate) qualname: &'static str, pub(crate) fields: &'static [FieldSpec], @@ -104,8 +106,8 @@ pub(crate) const fn unraw(name: &'static str) -> &'static str { /// `Name { field, field(default = Given::None), ... }`, optionally followed by /// `@impl { ... }` holding its extra methods and properties. `Name: Base { ... }` subclasses /// another shape and lists the base fields first, `Name @no_repr { ... }` keeps the object -/// repr, as `@dataclass(repr=False)` did, and `@post_init(check)` after the fields runs a -/// check over the bound values. +/// repr, as `@dataclass(repr=False)` did, and `@post_init(hook)` after the fields runs a hook +/// that may reject or replace the bound values. macro_rules! shapes { (@repr) => { true }; (@repr no_repr) => { false }; @@ -385,7 +387,7 @@ pub(crate) fn bind<'py>( *value = coerce(py, field.coerce, value)?; } if let Some(check) = spec.post_init { - check(&values)?; + check(&mut values)?; } Ok(values) } @@ -853,5 +855,6 @@ pub(crate) fn lazy_constants( pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { compiler_analysis::register(module)?; compiler_project::register(module)?; - planner_execution::register(module) + planner_execution::register(module)?; + executor_ops::register(module) } diff --git a/crates/sqltide-python/src/shapes/planner_execution.rs b/crates/sqltide-python/src/shapes/planner_execution.rs index ab94987c..56ab5453 100644 --- a/crates/sqltide-python/src/shapes/planner_execution.rs +++ b/crates/sqltide-python/src/shapes/planner_execution.rs @@ -84,7 +84,7 @@ fn publish_operation<'py>(value: &Bound<'py, PyAny>) -> PyResult]) -> PyResult<()> { +fn rollback_target(values: &mut [Bound<'_, PyAny>]) -> PyResult<()> { let [deployment_id, previous, ..] = values else { return Err(PyValueError::new_err( "rollback request bound too few values", diff --git a/python-boundary.toml b/python-boundary.toml index 3287d3a6..63748a66 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -8,7 +8,7 @@ # still carries planning, validation, rendering or other movable logic counts as movable until # that logic moves to Rust. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 4784 +remaining_lines = 3747 [[stays]] path = "adapter/classes/adapter.py" @@ -110,6 +110,11 @@ path = "events/types.py" rule = 3 note = "Rule 3: Public AuditTransition and ObservedCommand enums and the SensorEvent alias re-exported by sqltide.events; data only." +[[stays]] +path = "executor/destruction/types.py" +rule = 1 +note = "Rule 1: Logic-free protocols DestructionPlanningConnection (the read-only adapter surface destruction planning calls) and DestructionPlanStore (the reviewed single-use plan store) and the data-only type alias DestructionClock the destruction runtime consumes, beside native re-exports of the destruction value enums; no logic." + [[stays]] path = "executor/kafka_admin/_helpers/offset_reset.py" rule = 4 diff --git a/scripts/shape_surface/constants.py b/scripts/shape_surface/constants.py index 6fc455c1..4a9b81de 100644 --- a/scripts/shape_surface/constants.py +++ b/scripts/shape_surface/constants.py @@ -90,9 +90,17 @@ "sqltide.compiler.discovery.types:PipelineFacts": _UNUSED_SHAPE, "sqltide.compiler.discovery.types:ProjectPolicyFacts": _UNUSED_SHAPE, "sqltide.compiler.test_discovery.models:SqlTestHeader": _UNUSED_SHAPE, + "sqltide.executor.auditing.types:AuditResultStatus": _UNUSED_SHAPE, "sqltide.executor.deployment.types:NativeDeploymentSummary": _NATIVE_INTERCHANGE, "sqltide.executor.deployment.types:NativeInventoryRecord": _NATIVE_INTERCHANGE, + "sqltide.executor.destruction.models:DestructionPlanParts": _UNUSED_SHAPE, "sqltide.executor.destruction.models:RelationalStoredDestructionPlan": _NATIVE_INTERCHANGE, + "sqltide.executor.readiness.models:ColumnNameSystemRow": _UNUSED_SHAPE, + "sqltide.executor.readiness.models:CountQueryRow": _UNUSED_SHAPE, + "sqltide.executor.readiness.models:CreateTableQueryRow": _UNUSED_SHAPE, + "sqltide.executor.readiness.models:OffsetSummaryQueryRow": _UNUSED_SHAPE, + "sqltide.executor.readiness.models:ScalarSummaryQueryRow": _UNUSED_SHAPE, + "sqltide.executor.reconcile.models:ReconcileCandidate": _UNUSED_SHAPE, "sqltide.executor.reconcile.types:NativeObjectKey": _NATIVE_INTERCHANGE, "sqltide.executor.reconcile.types:NativeReconcileObject": _NATIVE_INTERCHANGE, "sqltide.executor.reconcile.types:NativeReconcileOutcome": _NATIVE_INTERCHANGE, @@ -105,6 +113,7 @@ "sqltide.executor.testing.constants:COMPARISON_ROW_VALUES_INDEX": _NATIVE_CONSTANT, "sqltide.executor.testing.constants:MISSING_DIFF_TYPE": _NATIVE_CONSTANT, "sqltide.executor.testing.constants:UNEXPECTED_DIFF_TYPE": _NATIVE_CONSTANT, + "sqltide.executor.testing.models:ComparisonRows": _UNUSED_SHAPE, "sqltide.sensors.constants:DEFAULT_DISPATCH_LEASE_TTL_SECONDS": _NATIVE_CONSTANT, "sqltide.sensors.constants:DEFAULT_EVENT_BATCH_LIMIT": _NATIVE_CONSTANT, "sqltide.sensors.constants:DISPATCH_LEASE_NAME": _NATIVE_CONSTANT, diff --git a/src/sqltide/_native.pyi b/src/sqltide/_native.pyi index e39763b8..f7c8f897 100644 --- a/src/sqltide/_native.pyi +++ b/src/sqltide/_native.pyi @@ -163,49 +163,13 @@ from sqltide.events.models import ( NodeResultObservation, RunCompleted, ) -from sqltide.executor.auditing.models import AuditWarmupState, SqlAuditResult, SqlAuditRunResult -from sqltide.executor.auditing.types import QualityResultStatus -from sqltide.executor.deployment.models import ( - DeploymentDiffRequest, - DeploymentDiffResult, - DeploymentInventory, - DeploymentSummary, -) -from sqltide.executor.destruction.models import ( - DestructionActor, - DestructionExecutionResult, - DestructionPlan, - DestructionRequest, - InactivePipeline, -) from sqltide.executor.destruction.types import ( DestructionClock, DestructionPlanningConnection, DestructionPlanStore, ) -from sqltide.executor.doctor.models import DoctorRequest, DoctorResult -from sqltide.executor.janitor.models import ( - JanitorApplyResult, - JanitorPreviewResult, - JanitorRequest, -) from sqltide.executor.kafka_admin.models import ConsumerGroupOffsetReset from sqltide.executor.kafka_admin.types import KafkaAdminClientFactory -from sqltide.executor.observability.models import ( - QualityResultContext, - RunStartupTimings, - TerminalInvocation, -) -from sqltide.executor.readiness.models import ( - AuditDeploymentCandidate, - DeploymentAuditRequest, - DeploymentAuditResult, - DeploymentReadinessThresholds, - LoadedAuditDeployment, - RootAuditResult, -) -from sqltide.executor.reconcile.models import ReconcilePreview, ReconcileResult -from sqltide.executor.testing.models import SqlTestExecutionResult from sqltide.executor.workflow.models import ( BuildWorkflow, PublishedBuildWorkflow, @@ -2032,6 +1996,529 @@ CLICKHOUSE_CATALOG_RETRY_DELAY_SECONDS: float CLICKHOUSE_MODEL_TABLE_NAME_PREFIX: str STAGED_SOURCE_NAME_PREFIX: str +# Operational executor shapes: destruction, readiness, deployment, auditing, observability, +# janitor, reconcile, SQL tests and doctor. + +class DestructionOperation(StrEnum): + DESTROY_PIPELINES = "destroy_pipelines" + RESET_TARGET = "reset_target" + +class DestructionRelationKind(StrEnum): + VIEW = "view" + MATERIALIZED_VIEW = "materialized_view" + TABLE = "table" + MANAGED_SOURCE = "managed_source" + +class DestructionOwnership(StrEnum): + CURRENT_MANIFEST = "current_manifest" + VIRTUAL_PHYSICAL_MAPPING = "virtual_physical_mapping" + PUBLISHED_STABLE_BINDING = "published_stable_binding" + OWNERSHIP_LEDGER = "ownership_ledger" + HISTORICAL_MANIFEST = "historical_manifest" + +class DestructionPlanningErrorCode(StrEnum): + SELECTION = "selection" + UNKNOWN_PIPELINES = "unknown_pipelines" + DEPENDENCY = "dependency" + RESOURCE = "resource" + UNSUPPORTED_HISTORICAL_KIND = "unsupported_historical_kind" + INVALID_OBJECT_TYPE = "invalid_object_type" + +class AuditAssessment(StrEnum): + READY = "ready" + NOT_READY = "not_ready" + CAUTION = "caution" + +class DeploymentLifecycleState(StrEnum): + ACTIVE = "active" + STAGED = "staged" + SUPERSEDED = "superseded" + INCOMPLETE = "incomplete" + METADATA_MISSING = "metadata_missing" + PHYSICAL_MISSING = "physical_missing" + +class DeploymentDiffStatus(StrEnum): + ADDED = "added" + REMOVED = "removed" + CHANGED = "changed" + UNCHANGED = "unchanged" + PHYSICAL_MISSING = "physical_missing" + +class AuditSeverity(StrEnum): + ERROR = "error" + WARNING = "warning" + +class QualityResultStatus(StrEnum): + PASSED = "passed" + WARNING = "warning" + FAILED = "failed" + ERROR = "error" + DEFERRED = "deferred" + +class QualityResultTrigger(StrEnum): + BUILD = "build" + MANUAL = "manual" + SCHEDULED = "scheduled" + DEPLOYMENT = "deployment" + +class MaterializationOutcome(StrEnum): + APPLIED = "applied" + FAILED = "failed" + +@final +@dataclass(frozen=True) +class DestructionRequest: + operation: DestructionOperation | str + target: str + database: str + metadata_database: str + pipeline_names: tuple[str, ...] = () + included_dependent_pipeline_names: tuple[str, ...] = () + include_orphans: bool = False + +@final +@dataclass(frozen=True) +class InactivePipeline: + name: str + model_count: int + resource_count: int + last_published_at: str + +@final +@dataclass(frozen=True) +class DestructionRelationEvidence: + database: str + name: str + kind: DestructionRelationKind | str + exists: bool + total_bytes: int | None + active_parts: int | None + catalog_fingerprint: str | None + logical_names: tuple[str, ...] + pipeline_names: tuple[str, ...] + ownership: tuple[DestructionOwnership, ...] + dependency_relation_names: tuple[str, ...] + +@final +@dataclass(frozen=True) +class DestructionPlan: + plan_id: str + operation: DestructionOperation + target: str + database: str + metadata_database: str + requested_pipeline_names: tuple[str, ...] + included_dependent_pipeline_names: tuple[str, ...] + affected_pipeline_names: tuple[str, ...] + affected_model_names: tuple[str, ...] + affected_source_names: tuple[str, ...] + relations: tuple[DestructionRelationEvidence, ...] + challenges: tuple[str, ...] + preserves_sources: bool + preserves_replay_data: bool + manifest_fingerprint: str + plan_fingerprint: str + created_at: datetime + expires_at: datetime + relation_drop_size_limit: int | None = None + relation_drop_size_server_limit: int | None = None + relation_drop_size_override: int | None = None + relation_drop_size_policy_observed: bool = False + include_orphans: bool = False + @property + def estimated_bytes(self) -> int: ... + +@final +@dataclass(frozen=True) +class DestructionExecutionResult: + invocation_id: str + outcome: str + completed_statement_sequences: tuple[int, ...] + pending_statement_sequences: tuple[int, ...] + remaining_relation_names: tuple[str, ...] | None + error_message: str | None + residual_catalog_status: str = "observed" + residual_catalog_error: str | None = None + +@final +@dataclass(frozen=True) +class DestructionActor: + actor_id: str + actor_name: str + +@final +@dataclass(frozen=True) +class DestructionRecordingContext: + plan: DestructionPlan + actor_id: str + actor_name: str + reviewed_at: datetime + confirmed_at: datetime + challenge_responses: tuple[str, ...] + connection: AdapterConnection + observation_connection: AdapterConnection + project_dir: Path + started: tuple[str, str, int] + statements: tuple[WarehouseStatement, ...] + sink: RunEventSink + +@final +@dataclass(frozen=True) +class DeploymentReadinessThresholds: + maximum_lag_seconds: float = ... + minimum_staged_row_ratio: float = ... + +@final +@dataclass(frozen=True) +class DeploymentAuditRequest: + deployment_id: str | None + metadata_database: str + default_database: str + thresholds: DeploymentReadinessThresholds = field(default_factory=DeploymentReadinessThresholds) + +@final +@dataclass(frozen=True) +class AuditDeploymentCandidate: + deployment_id: str + created_at: str | None = None + deployment_status: str | None = None + root_names: tuple[str, ...] = () + +@final +@dataclass(frozen=True) +class LoadedAuditDeployment: + deployment_id: str + created_at: str + status: str + replay_lineage_mode: ReplayLineageMode | str | None + warning_codes: tuple[str, ...] + root_keys: tuple[ObjectKey, ...] + prepared_object_mappings: tuple[tuple[ObjectKey, str], ...] + +@final +@dataclass(frozen=True) +class OffsetCatchupSummary: + active_partition_count: int + staged_partition_count: int + partitions_compared: int + missing_staged_partition_count: int + missing_freshness_partition_count: int + lagging_partition_count: int + max_offset_gap: int + average_offset_gap: float + lag_boundary_column: str | None + max_lag_seconds: float | None + average_lag_seconds: float | None + +@final +@dataclass(frozen=True) +class ScalarCatchupSummary: + active_min_value: str | None + active_max_value: str | None + staged_min_value: str | None + staged_max_value: str | None + lag_seconds: float | None + +@final +@dataclass(frozen=True) +class RootAuditResult: + root_key: ObjectKey + staged_physical_name: str + staged_exists: bool + active_exists: bool + active_row_count: int | None + staged_row_count: int | None + row_delta: int | None + row_ratio: float | None + assessment: AuditAssessment | str + replay_lineage_mode: ReplayLineageMode | str | None + offset_catchup_summary: OffsetCatchupSummary | None + scalar_catchup_summary: ScalarCatchupSummary | None + state: str + replay_source_name: str | None + replay_source_row_count: int | None + warnings: tuple[str, ...] = () + +@final +@dataclass(frozen=True) +class DeploymentAuditResult: + deployment_id: str + deployment_status: str + assessment: AuditAssessment | str + replay_lineage_mode: ReplayLineageMode | str | None + warning_codes: tuple[str, ...] + root_results: tuple[RootAuditResult, ...] + quality_check_results: tuple[SqlAuditResult, ...] = () + thresholds: DeploymentReadinessThresholds = field(default_factory=DeploymentReadinessThresholds) + +@final +@dataclass(frozen=True) +class DeploymentSummary: + deployment_id: str + state: DeploymentLifecycleState + created_at: str | None + persisted_status: str | None + root_names: tuple[str, ...] + physical_relation_names: tuple[str, ...] + missing_physical_relation_names: tuple[str, ...] + active_binding_names: tuple[str, ...] + latest_published_at: str | None + +@final +@dataclass(frozen=True) +class DeploymentInventory: + database: str + deployments: tuple[DeploymentSummary, ...] + +@final +@dataclass(frozen=True) +class DeploymentDiffRequest: + database: str + metadata_database: str + comparison: str + +@final +@dataclass(frozen=True) +class DeploymentDiffColumn: + name: str + type: str + default_expression: str | None + +@final +@dataclass(frozen=True) +class DeploymentDiffRelation: + database: str + logical_name: str + status: DeploymentDiffStatus | str + from_physical_name: str | None + to_physical_name: str | None + from_columns: tuple[DeploymentDiffColumn, ...] + to_columns: tuple[DeploymentDiffColumn, ...] + from_row_count: int | None + to_row_count: int | None + +@final +@dataclass(frozen=True) +class DeploymentDiffResult: + database: str + from_endpoint: str + to_endpoint: str + relations: tuple[DeploymentDiffRelation, ...] + +@final +@dataclass(frozen=True) +class SqlAuditResult: + file_path: Path + referenced_model_names: tuple[str, ...] + severity: str + passed: bool + failing_row_count: int + sample_column_names: tuple[str, ...] + sample_rows: tuple[tuple[object, ...], ...] + description: str | None = None + name: str | None = None + error_message: str | None = None + deferred_until: str | None = None + missing_relation_names: tuple[str, ...] = () + @property + def deferred(self) -> bool: ... + +@final +@dataclass(frozen=True) +class SqlAuditRunResult: + audit_results: tuple[SqlAuditResult, ...] + @property + def error_failure_count(self) -> int: ... + @property + def warning_failure_count(self) -> int: ... + +@final +@dataclass(frozen=True) +class AuditWarmupState: + eligible: bool + anchor: str | None + eligible_at: str | None + missing_model_names: tuple[str, ...] = () + +@final +@dataclass(frozen=True) +class RunStartupTimings: + compile_ms: int + observability_ms: int + planning_ms: int + @property + def total_ms(self) -> int: ... + +@final +@dataclass(frozen=True) +class TerminalInvocation: + project_dir: Path + target_identity: str + command: str + mode: str | None + outcome: str + exit_code: int + materialized_outcome: str | None + deployment_id: str | None + workflow_id: str | None + selected_node_count: int + error_message: str | None + summary: dict[str, object] + +@final +@dataclass(frozen=True) +class QualityResultContext: + trigger: QualityResultTrigger | str + scheduled_for: str | None = None + cadence_seconds: int | None = None + warmup_seconds: int = 0 + +@final +@dataclass(frozen=True) +class JanitorPreviewCandidate: + deployment_id: str + created_at: str + status: str + logical_view_names: tuple[str, ...] + physical_object_names: tuple[str, ...] + deletable: bool + reason: str + +@final +@dataclass(frozen=True) +class JanitorPreviewResult: + database: str + retention_days: int + minimum_rollback_deployments: int + candidates: tuple[JanitorPreviewCandidate, ...] + +@final +@dataclass(frozen=True) +class JanitorApplyResult: + database: str + retention_days: int + minimum_rollback_deployments: int + deleted_deployment_ids: tuple[str, ...] + deleted_object_names: tuple[str, ...] + +@final +@dataclass(frozen=True) +class JanitorRequest: + database: str + metadata_database: str + retention_days: int + apply: bool + minimum_rollback_deployments: int = 2 + +@final +@dataclass(frozen=True) +class ReconcileRejectedTarget: + target_key: ObjectKey + target_name: str + reasons: tuple[str, ...] + +@final +@dataclass(frozen=True) +class ReconcilePreview: + database: str + reconcile_id: str + eligible_records: tuple[ObjectStateRecord, ...] + rejected_targets: tuple[ReconcileRejectedTarget, ...] + +@final +@dataclass(frozen=True) +class ReconcileResult: + database: str + reconcile_id: str + reconciled_records: tuple[ObjectStateRecord, ...] + rejected_targets: tuple[ReconcileRejectedTarget, ...] + +@final +@dataclass(frozen=True) +class SqlTestTargetExecutionResult: + target_model_name: str + passed: bool + column_names: tuple[str, ...] + missing_rows: tuple[tuple[object, ...], ...] + unexpected_rows: tuple[tuple[object, ...], ...] + +@final +@dataclass(frozen=True) +class SqlTestExecutionResult: + file_path: Path + passed: bool + target_results: tuple[SqlTestTargetExecutionResult, ...] + executed_sql: str + warnings: tuple[str, ...] = () + name: str | None = None + test_index: int = 1 + error_message: str | None = None + +@final +@dataclass(frozen=True) +class DoctorRequest: + default_database: str + +@final +@dataclass(frozen=True) +class ActiveViewStatus: + table_name: str + state_kind: str + active_deployment_id: str | None + candidate_deployment_ids: tuple[str, ...] + +@final +@dataclass(frozen=True) +class DoctorResult: + active_views: tuple[ActiveViewStatus, ...] + +class DestructionError(RuntimeError): ... +class DestructionSelectionError(DestructionError): ... +class DestructionValidationError(DestructionError, ValueError): ... +class DestructionResourceError(DestructionError, TypeError): ... +class DestructionConsistencyError(DestructionError): ... + +class DestructionDependencyError(DestructionSelectionError): + dependent_pipeline_names: tuple[str, ...] + def __init__(self, dependent_pipeline_names: tuple[str, ...]) -> None: ... + +class DestructionExternalDependencyError(DestructionSelectionError): + relation_names: tuple[str, ...] + def __init__(self, relation_names: tuple[str, ...]) -> None: ... + +class DestructionPlanNotFoundError(DestructionError): ... +class DestructionPlanExpiredError(DestructionError): ... +class DestructionPlanNotReviewedError(DestructionError): ... +class DestructionPlanCorruptError(DestructionError): ... +class DestructionChallengeError(DestructionError): ... +class DestructionDriftError(DestructionError): ... +class DestructionRecordingError(DestructionError): ... +class DestructionRecoveryError(DestructionError): ... +class DestructionRecoveryNotFoundError(DestructionRecoveryError): ... +class DeploymentAuditExecutionError(ValueError): ... +class DeploymentNotFoundError(ValueError): ... +class DeploymentDiffError(ValueError): ... +class AuditExecutionError(ValueError): ... +class JanitorInputError(ValueError): ... +class ReconcileError(RuntimeError): ... +class SqlTestExecutionError(ValueError): ... + +DEFAULT_DESTRUCTION_PLAN_TTL: timedelta +DESTRUCTION_PLAN_PAYLOAD_VERSION: int +PRODUCTION_TARGET_NAMES: frozenset[str] +ACCEPTABLE_LAG_SECONDS: float +MINIMUM_STAGED_ROW_RATIO: float +ACTIVE_DEPLOYMENT_DIFF_ENDPOINT: str +EXPLICIT_DEPLOYMENT_DIFF_ENDPOINT_COUNT: int +AUDIT_SAMPLE_LIMIT: int +MAX_OBSERVATION_JSON_BYTES: int +MAX_OBSERVATION_ERROR_LENGTH: int +HEARTBEAT_INTERVAL_SECONDS: float +RUN_DISPLAY_COMMAND_ENV_VAR: str +RUN_INVOCATION_ID_ENV_VAR: str +PROJECT_REVISION_ENV_VAR: str +RECONCILE_DEPLOYMENT_ID_PREFIX: str + type _Span = tuple[int, int, int, int, int, int] class SqlScanError(ValueError): diff --git a/src/sqltide/executor/auditing/constants.py b/src/sqltide/executor/auditing/constants.py index 51ffd420..dec5a1c4 100644 --- a/src/sqltide/executor/auditing/constants.py +++ b/src/sqltide/executor/auditing/constants.py @@ -1,3 +1,3 @@ """Stable constants for SQL audit execution.""" -AUDIT_SAMPLE_LIMIT: int = 5 +from sqltide._native import AUDIT_SAMPLE_LIMIT as AUDIT_SAMPLE_LIMIT diff --git a/src/sqltide/executor/auditing/exceptions.py b/src/sqltide/executor/auditing/exceptions.py index 19eb7b41..5ac8a374 100644 --- a/src/sqltide/executor/auditing/exceptions.py +++ b/src/sqltide/executor/auditing/exceptions.py @@ -1,5 +1,3 @@ """Auditing exceptions.""" - -class AuditExecutionError(ValueError): - """Raised when auditing input or state is invalid.""" +from sqltide._native import AuditExecutionError as AuditExecutionError diff --git a/src/sqltide/executor/auditing/models.py b/src/sqltide/executor/auditing/models.py index abb85364..640c6eb4 100644 --- a/src/sqltide/executor/auditing/models.py +++ b/src/sqltide/executor/auditing/models.py @@ -1,69 +1,5 @@ """Runtime models for SQL audit execution.""" -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path - -from sqltide.executor.auditing.types import AuditSeverity - - -@dataclass(frozen=True) -class SqlAuditResult: - """One executed SQL audit and its outcome.""" - - file_path: Path - referenced_model_names: tuple[str, ...] - severity: str - passed: bool - failing_row_count: int - sample_column_names: tuple[str, ...] - sample_rows: tuple[tuple[object, ...], ...] - description: str | None = None - name: str | None = None - error_message: str | None = None - deferred_until: str | None = None - missing_relation_names: tuple[str, ...] = () - - @property - def deferred(self) -> bool: - """Return whether execution was deferred for warmup or materialization.""" - - return self.deferred_until is not None or bool(self.missing_relation_names) - - -@dataclass(frozen=True) -class SqlAuditRunResult: - """Aggregate outcome for a batch of SQL audits.""" - - audit_results: tuple[SqlAuditResult, ...] - - @property - def error_failure_count(self) -> int: - return sum( - 1 - for audit_result in self.audit_results - if audit_result.severity == AuditSeverity.ERROR - and not audit_result.passed - and not audit_result.deferred - ) - - @property - def warning_failure_count(self) -> int: - return sum( - 1 - for audit_result in self.audit_results - if audit_result.severity == AuditSeverity.WARNING - and not audit_result.passed - and not audit_result.deferred - ) - - -@dataclass(frozen=True) -class AuditWarmupState: - """Eligibility of one audit relative to its newest referenced model anchor.""" - - eligible: bool - anchor: str | None - eligible_at: str | None - missing_model_names: tuple[str, ...] = () +from sqltide._native import AuditWarmupState as AuditWarmupState +from sqltide._native import SqlAuditResult as SqlAuditResult +from sqltide._native import SqlAuditRunResult as SqlAuditRunResult diff --git a/src/sqltide/executor/auditing/types.py b/src/sqltide/executor/auditing/types.py index 29c4312f..1a256942 100644 --- a/src/sqltide/executor/auditing/types.py +++ b/src/sqltide/executor/auditing/types.py @@ -1,28 +1,4 @@ """Live audit runtime domain types.""" -from enum import StrEnum - - -class AuditSeverity(StrEnum): - """How a failing audit should be reported.""" - - ERROR = "error" - WARNING = "warning" - - -class AuditResultStatus(StrEnum): - """Operator-facing status label for one audit result.""" - - PASS = "PASS" - WARN = "WARN" - FAIL = "FAIL" - - -class QualityResultStatus(StrEnum): - """Persisted status of one audit or SQL-test attempt.""" - - PASSED = "passed" - WARNING = "warning" - FAILED = "failed" - ERROR = "error" - DEFERRED = "deferred" +from sqltide._native import AuditSeverity as AuditSeverity +from sqltide._native import QualityResultStatus as QualityResultStatus diff --git a/src/sqltide/executor/deployment/constants.py b/src/sqltide/executor/deployment/constants.py index 062930af..487d7152 100644 --- a/src/sqltide/executor/deployment/constants.py +++ b/src/sqltide/executor/deployment/constants.py @@ -1,4 +1,6 @@ """Deployment lifecycle constants.""" -ACTIVE_DEPLOYMENT_DIFF_ENDPOINT: str = "active" -EXPLICIT_DEPLOYMENT_DIFF_ENDPOINT_COUNT: int = 2 +from sqltide._native import ACTIVE_DEPLOYMENT_DIFF_ENDPOINT as ACTIVE_DEPLOYMENT_DIFF_ENDPOINT +from sqltide._native import ( + EXPLICIT_DEPLOYMENT_DIFF_ENDPOINT_COUNT as EXPLICIT_DEPLOYMENT_DIFF_ENDPOINT_COUNT, +) diff --git a/src/sqltide/executor/deployment/exceptions.py b/src/sqltide/executor/deployment/exceptions.py index 325e35d0..735e7103 100644 --- a/src/sqltide/executor/deployment/exceptions.py +++ b/src/sqltide/executor/deployment/exceptions.py @@ -1,9 +1,4 @@ """Deployment lifecycle exceptions.""" - -class DeploymentNotFoundError(ValueError): - """Raised when an explicit deployment identifier is not present.""" - - -class DeploymentDiffError(ValueError): - """Raised when a deployment comparison cannot be resolved safely.""" +from sqltide._native import DeploymentDiffError as DeploymentDiffError +from sqltide._native import DeploymentNotFoundError as DeploymentNotFoundError diff --git a/src/sqltide/executor/deployment/models.py b/src/sqltide/executor/deployment/models.py index e4449298..b05eab92 100644 --- a/src/sqltide/executor/deployment/models.py +++ b/src/sqltide/executor/deployment/models.py @@ -1,74 +1,8 @@ """Deployment inventory models.""" -from dataclasses import dataclass - -from sqltide.executor.deployment.types import DeploymentDiffStatus, DeploymentLifecycleState - - -@dataclass(frozen=True) -class DeploymentSummary: - """One deployment reconstructed from authoritative warehouse evidence.""" - - deployment_id: str - state: DeploymentLifecycleState - created_at: str | None - persisted_status: str | None - root_names: tuple[str, ...] - physical_relation_names: tuple[str, ...] - missing_physical_relation_names: tuple[str, ...] - active_binding_names: tuple[str, ...] - latest_published_at: str | None - - -@dataclass(frozen=True) -class DeploymentInventory: - """Deterministically ordered deployment summaries for one target database.""" - - database: str - deployments: tuple[DeploymentSummary, ...] - - -@dataclass(frozen=True) -class DeploymentDiffRequest: - """One active/deployment comparison expression.""" - - database: str - metadata_database: str - comparison: str - - -@dataclass(frozen=True) -class DeploymentDiffColumn: - """One catalog column included in a deployment comparison.""" - - name: str - type: str - default_expression: str | None - - -@dataclass(frozen=True) -class DeploymentDiffRelation: - """Schema and count comparison for one logical relation.""" - - database: str - logical_name: str - status: DeploymentDiffStatus | str - from_physical_name: str | None - to_physical_name: str | None - from_columns: tuple[DeploymentDiffColumn, ...] - to_columns: tuple[DeploymentDiffColumn, ...] - from_row_count: int | None - to_row_count: int | None - - def __post_init__(self) -> None: - object.__setattr__(self, "status", DeploymentDiffStatus(self.status)) - - -@dataclass(frozen=True) -class DeploymentDiffResult: - """Resolved endpoints and per-relation deployment differences.""" - - database: str - from_endpoint: str - to_endpoint: str - relations: tuple[DeploymentDiffRelation, ...] +from sqltide._native import DeploymentDiffColumn as DeploymentDiffColumn +from sqltide._native import DeploymentDiffRelation as DeploymentDiffRelation +from sqltide._native import DeploymentDiffRequest as DeploymentDiffRequest +from sqltide._native import DeploymentDiffResult as DeploymentDiffResult +from sqltide._native import DeploymentInventory as DeploymentInventory +from sqltide._native import DeploymentSummary as DeploymentSummary diff --git a/src/sqltide/executor/deployment/types.py b/src/sqltide/executor/deployment/types.py index bc11da34..a25e2998 100644 --- a/src/sqltide/executor/deployment/types.py +++ b/src/sqltide/executor/deployment/types.py @@ -1,24 +1,4 @@ """Deployment lifecycle types.""" -from enum import StrEnum - - -class DeploymentLifecycleState(StrEnum): - """Authoritative state derived from metadata and live catalog evidence.""" - - ACTIVE = "active" - STAGED = "staged" - SUPERSEDED = "superseded" - INCOMPLETE = "incomplete" - METADATA_MISSING = "metadata_missing" - PHYSICAL_MISSING = "physical_missing" - - -class DeploymentDiffStatus(StrEnum): - """Summary classification for one logical relation comparison.""" - - ADDED = "added" - REMOVED = "removed" - CHANGED = "changed" - UNCHANGED = "unchanged" - PHYSICAL_MISSING = "physical_missing" +from sqltide._native import DeploymentDiffStatus as DeploymentDiffStatus +from sqltide._native import DeploymentLifecycleState as DeploymentLifecycleState diff --git a/src/sqltide/executor/destruction/constants.py b/src/sqltide/executor/destruction/constants.py index b5af5bed..73d3bf61 100644 --- a/src/sqltide/executor/destruction/constants.py +++ b/src/sqltide/executor/destruction/constants.py @@ -1,7 +1,5 @@ """Constants for destructive execution.""" -from datetime import timedelta - -DEFAULT_DESTRUCTION_PLAN_TTL: timedelta = timedelta(minutes=15) -DESTRUCTION_PLAN_PAYLOAD_VERSION: int = 1 -PRODUCTION_TARGET_NAMES: frozenset[str] = frozenset({"prod", "production"}) +from sqltide._native import DEFAULT_DESTRUCTION_PLAN_TTL as DEFAULT_DESTRUCTION_PLAN_TTL +from sqltide._native import DESTRUCTION_PLAN_PAYLOAD_VERSION as DESTRUCTION_PLAN_PAYLOAD_VERSION +from sqltide._native import PRODUCTION_TARGET_NAMES as PRODUCTION_TARGET_NAMES diff --git a/src/sqltide/executor/destruction/exceptions.py b/src/sqltide/executor/destruction/exceptions.py index 9a26c357..82bf93ae 100644 --- a/src/sqltide/executor/destruction/exceptions.py +++ b/src/sqltide/executor/destruction/exceptions.py @@ -1,76 +1,18 @@ """Failures raised by destruction planning and confirmation.""" - -class DestructionError(RuntimeError): - """Base error for the destruction domain.""" - - -class DestructionSelectionError(DestructionError): - """The requested pipeline selection is invalid.""" - - -class DestructionValidationError(DestructionError, ValueError): - """A destruction value violates a domain invariant.""" - - -class DestructionResourceError(DestructionError, TypeError): - """A manifest resource cannot participate in destruction.""" - - -class DestructionConsistencyError(DestructionError): - """Internal destruction evidence became inconsistent.""" - - -class DestructionDependencyError(DestructionSelectionError): - """Unselected downstream pipelines make destruction unsafe.""" - - def __init__(self, dependent_pipeline_names: tuple[str, ...]) -> None: - self.dependent_pipeline_names = dependent_pipeline_names - super().__init__( - "Unselected dependent or shared-source pipelines must be explicitly included: " - f"{dependent_pipeline_names!r}" - ) - - -class DestructionExternalDependencyError(DestructionSelectionError): - """Warehouse relations outside the owned scope depend on planned relations.""" - - def __init__(self, relation_names: tuple[str, ...]) -> None: - self.relation_names: tuple[str, ...] = relation_names - super().__init__(f"Unmanaged warehouse dependants block destruction: {relation_names!r}") - - -class DestructionPlanNotFoundError(DestructionError): - """The requested frozen plan does not exist or was already consumed.""" - - -class DestructionPlanExpiredError(DestructionError): - """The requested frozen plan has expired.""" - - -class DestructionPlanNotReviewedError(DestructionError): - """The frozen plan has not passed the explicit review gate.""" - - -class DestructionPlanCorruptError(DestructionError): - """A durable frozen-plan record is incompatible or corrupt.""" - - -class DestructionChallengeError(DestructionError): - """Typed challenge responses do not exactly match the frozen plan.""" - - -class DestructionDriftError(DestructionError): - """A freshly planned operation differs from its frozen plan.""" - - -class DestructionRecordingError(DestructionError): - """Required destructive-operation evidence could not be persisted.""" - - -class DestructionRecoveryError(DestructionError): - """A failed run does not contain valid recoverable destruction intent.""" - - -class DestructionRecoveryNotFoundError(DestructionRecoveryError): - """A run is not a recoverable operation in the active project.""" +from sqltide._native import DestructionChallengeError as DestructionChallengeError +from sqltide._native import DestructionConsistencyError as DestructionConsistencyError +from sqltide._native import DestructionDependencyError as DestructionDependencyError +from sqltide._native import DestructionDriftError as DestructionDriftError +from sqltide._native import DestructionError as DestructionError +from sqltide._native import DestructionExternalDependencyError as DestructionExternalDependencyError +from sqltide._native import DestructionPlanCorruptError as DestructionPlanCorruptError +from sqltide._native import DestructionPlanExpiredError as DestructionPlanExpiredError +from sqltide._native import DestructionPlanNotFoundError as DestructionPlanNotFoundError +from sqltide._native import DestructionPlanNotReviewedError as DestructionPlanNotReviewedError +from sqltide._native import DestructionRecordingError as DestructionRecordingError +from sqltide._native import DestructionRecoveryError as DestructionRecoveryError +from sqltide._native import DestructionRecoveryNotFoundError as DestructionRecoveryNotFoundError +from sqltide._native import DestructionResourceError as DestructionResourceError +from sqltide._native import DestructionSelectionError as DestructionSelectionError +from sqltide._native import DestructionValidationError as DestructionValidationError diff --git a/src/sqltide/executor/destruction/models.py b/src/sqltide/executor/destruction/models.py index 42b41532..f45f7144 100644 --- a/src/sqltide/executor/destruction/models.py +++ b/src/sqltide/executor/destruction/models.py @@ -1,178 +1,9 @@ """Immutable destruction requests, evidence, and plans.""" -from __future__ import annotations - -from dataclasses import dataclass -from datetime import datetime, timedelta -from pathlib import Path - -from sqltide._native import RunEventSink -from sqltide.adapter.classes.adapter_connection import AdapterConnection -from sqltide.executor.destruction.exceptions import DestructionValidationError -from sqltide.executor.destruction.types import ( - DestructionOperation, - DestructionOwnership, - DestructionRelationKind, -) -from sqltide.executor.workflow.models import WarehouseStatement - - -@dataclass(frozen=True) -class DestructionRequest: - operation: DestructionOperation | str - target: str - database: str - metadata_database: str - pipeline_names: tuple[str, ...] = () - included_dependent_pipeline_names: tuple[str, ...] = () - include_orphans: bool = False - - def __post_init__(self) -> None: - object.__setattr__(self, "operation", DestructionOperation(self.operation)) - if self.include_orphans and self.operation != DestructionOperation.DESTROY_PIPELINES: - raise DestructionValidationError( - "Orphan inclusion is only valid for selected pipeline destruction" - ) - if len(set(self.pipeline_names)) != len(self.pipeline_names): - raise DestructionValidationError( - "Pipeline destruction selection contains duplicate names" - ) - if len(set(self.included_dependent_pipeline_names)) != len( - self.included_dependent_pipeline_names - ): - raise DestructionValidationError( - "Included dependent pipeline selection contains duplicate names" - ) - overlap: set[str] = set(self.pipeline_names) & set(self.included_dependent_pipeline_names) - if overlap: - raise DestructionValidationError( - "Pipelines cannot be both originally selected and included dependants: " - f"{tuple(sorted(overlap))!r}" - ) - object.__setattr__(self, "pipeline_names", tuple(sorted(set(self.pipeline_names)))) - object.__setattr__( - self, - "included_dependent_pipeline_names", - tuple(sorted(set(self.included_dependent_pipeline_names))), - ) - - -@dataclass(frozen=True) -class InactivePipeline: - """A historically published pipeline absent from the compiled project.""" - - name: str - model_count: int - resource_count: int - last_published_at: str - - -@dataclass(frozen=True) -class DestructionRelationEvidence: - database: str - name: str - kind: DestructionRelationKind | str - exists: bool - total_bytes: int | None - active_parts: int | None - catalog_fingerprint: str | None - logical_names: tuple[str, ...] - pipeline_names: tuple[str, ...] - ownership: tuple[DestructionOwnership, ...] - dependency_relation_names: tuple[str, ...] - - def __post_init__(self) -> None: - object.__setattr__(self, "kind", DestructionRelationKind(self.kind)) - - -@dataclass(frozen=True) -class DestructionPlan: - plan_id: str - operation: DestructionOperation - target: str - database: str - metadata_database: str - requested_pipeline_names: tuple[str, ...] - included_dependent_pipeline_names: tuple[str, ...] - affected_pipeline_names: tuple[str, ...] - affected_model_names: tuple[str, ...] - affected_source_names: tuple[str, ...] - relations: tuple[DestructionRelationEvidence, ...] - challenges: tuple[str, ...] - preserves_sources: bool - preserves_replay_data: bool - manifest_fingerprint: str - plan_fingerprint: str - created_at: datetime - expires_at: datetime - relation_drop_size_limit: int | None = None - relation_drop_size_server_limit: int | None = None - relation_drop_size_override: int | None = None - relation_drop_size_policy_observed: bool = False - include_orphans: bool = False - - @property - def estimated_bytes(self) -> int: - return sum(relation.total_bytes or 0 for relation in self.relations) - - -@dataclass(frozen=True) -class DestructionExecutionResult: - """Terminal recorded outcome for one consumed frozen plan.""" - - invocation_id: str - outcome: str - completed_statement_sequences: tuple[int, ...] - pending_statement_sequences: tuple[int, ...] - remaining_relation_names: tuple[str, ...] | None - error_message: str | None - residual_catalog_status: str = "observed" - residual_catalog_error: str | None = None - - -@dataclass(frozen=True) -class DestructionActor: - """Authenticated actor identity recorded with destructive execution evidence.""" - - actor_id: str - actor_name: str - - -@dataclass(frozen=True) -class DestructionRecordingContext: - """Immutable identity and connections shared by destructive recording phases.""" - - plan: DestructionPlan - actor_id: str - actor_name: str - reviewed_at: datetime - confirmed_at: datetime - challenge_responses: tuple[str, ...] - connection: AdapterConnection - observation_connection: AdapterConnection - project_dir: Path - started: tuple[str, str, int] - statements: tuple[WarehouseStatement, ...] - sink: RunEventSink - - -@dataclass(frozen=True) -class DestructionPlanParts: - """Internal values assembled before a frozen plan receives its fingerprint.""" - - plan_id: str | None - created_at: datetime - ttl: timedelta - requested_pipeline_names: tuple[str, ...] - included_dependent_pipeline_names: tuple[str, ...] - affected_pipeline_names: tuple[str, ...] - affected_model_names: tuple[str, ...] - affected_source_names: tuple[str, ...] - relations: tuple[DestructionRelationEvidence, ...] - challenges: tuple[str, ...] - manifest_fingerprint: str - relation_drop_size_limit: int | None - relation_drop_size_server_limit: int | None - relation_drop_size_override: int | None - relation_drop_size_policy_observed: bool - include_orphans: bool +from sqltide._native import DestructionActor as DestructionActor +from sqltide._native import DestructionExecutionResult as DestructionExecutionResult +from sqltide._native import DestructionPlan as DestructionPlan +from sqltide._native import DestructionRecordingContext as DestructionRecordingContext +from sqltide._native import DestructionRelationEvidence as DestructionRelationEvidence +from sqltide._native import DestructionRequest as DestructionRequest +from sqltide._native import InactivePipeline as InactivePipeline diff --git a/src/sqltide/executor/destruction/types.py b/src/sqltide/executor/destruction/types.py index 8e9d437b..83d1840a 100644 --- a/src/sqltide/executor/destruction/types.py +++ b/src/sqltide/executor/destruction/types.py @@ -4,9 +4,12 @@ from collections.abc import Callable from datetime import datetime -from enum import StrEnum from typing import TYPE_CHECKING, Protocol +from sqltide._native import DestructionOperation as DestructionOperation +from sqltide._native import DestructionOwnership as DestructionOwnership +from sqltide._native import DestructionPlanningErrorCode as DestructionPlanningErrorCode +from sqltide._native import DestructionRelationKind as DestructionRelationKind from sqltide.adapter.models import ( AdapterDeploymentInventory, AdapterManifestSnapshot, @@ -18,37 +21,6 @@ from sqltide.executor.destruction.models import DestructionPlan -class DestructionOperation(StrEnum): - DESTROY_PIPELINES = "destroy_pipelines" - RESET_TARGET = "reset_target" - - -class DestructionRelationKind(StrEnum): - VIEW = "view" - MATERIALIZED_VIEW = "materialized_view" - TABLE = "table" - MANAGED_SOURCE = "managed_source" - - -class DestructionOwnership(StrEnum): - CURRENT_MANIFEST = "current_manifest" - VIRTUAL_PHYSICAL_MAPPING = "virtual_physical_mapping" - PUBLISHED_STABLE_BINDING = "published_stable_binding" - OWNERSHIP_LEDGER = "ownership_ledger" - HISTORICAL_MANIFEST = "historical_manifest" - - -class DestructionPlanningErrorCode(StrEnum): - """Why the native planner rejected a destruction request.""" - - SELECTION = "selection" - UNKNOWN_PIPELINES = "unknown_pipelines" - DEPENDENCY = "dependency" - RESOURCE = "resource" - UNSUPPORTED_HISTORICAL_KIND = "unsupported_historical_kind" - INVALID_OBJECT_TYPE = "invalid_object_type" - - class DestructionPlanningConnection(Protocol): """The complete, read-only adapter surface available to planning.""" diff --git a/src/sqltide/executor/doctor/models.py b/src/sqltide/executor/doctor/models.py index 0cafc42c..c18dab11 100644 --- a/src/sqltide/executor/doctor/models.py +++ b/src/sqltide/executor/doctor/models.py @@ -1,27 +1,5 @@ """Doctor runtime models.""" -from dataclasses import dataclass - - -@dataclass(frozen=True) -class DoctorRequest: - """Input required to inspect active-view health.""" - - default_database: str - - -@dataclass(frozen=True) -class ActiveViewStatus: - """Diagnosis for one logical managed table.""" - - table_name: str - state_kind: str - active_deployment_id: str | None - candidate_deployment_ids: tuple[str, ...] - - -@dataclass(frozen=True) -class DoctorResult: - """Read-only diagnosis of active-view health.""" - - active_views: tuple[ActiveViewStatus, ...] +from sqltide._native import ActiveViewStatus as ActiveViewStatus +from sqltide._native import DoctorRequest as DoctorRequest +from sqltide._native import DoctorResult as DoctorResult diff --git a/src/sqltide/executor/janitor/exceptions.py b/src/sqltide/executor/janitor/exceptions.py index 5816bc1a..6b1a5090 100644 --- a/src/sqltide/executor/janitor/exceptions.py +++ b/src/sqltide/executor/janitor/exceptions.py @@ -1,5 +1,3 @@ """Janitor input errors.""" - -class JanitorInputError(ValueError): - """Raised when janitor retention input is invalid.""" +from sqltide._native import JanitorInputError as JanitorInputError diff --git a/src/sqltide/executor/janitor/models.py b/src/sqltide/executor/janitor/models.py index 77fde4a1..82329e01 100644 --- a/src/sqltide/executor/janitor/models.py +++ b/src/sqltide/executor/janitor/models.py @@ -1,48 +1,6 @@ """Runtime models for janitor preview.""" -from dataclasses import dataclass - -from sqltide.executor.janitor.exceptions import JanitorInputError - - -@dataclass(frozen=True) -class JanitorPreviewCandidate: - deployment_id: str - created_at: str - status: str - logical_view_names: tuple[str, ...] - physical_object_names: tuple[str, ...] - deletable: bool - reason: str - - -@dataclass(frozen=True) -class JanitorPreviewResult: - database: str - retention_days: int - minimum_rollback_deployments: int - candidates: tuple[JanitorPreviewCandidate, ...] - - -@dataclass(frozen=True) -class JanitorApplyResult: - database: str - retention_days: int - minimum_rollback_deployments: int - deleted_deployment_ids: tuple[str, ...] - deleted_object_names: tuple[str, ...] - - -@dataclass(frozen=True) -class JanitorRequest: - database: str - metadata_database: str - retention_days: int - apply: bool - minimum_rollback_deployments: int = 2 - - def __post_init__(self) -> None: - if self.retention_days < 0: - raise JanitorInputError("retention_days must be non-negative") - if self.minimum_rollback_deployments < 0: - raise JanitorInputError("minimum_rollback_deployments must be non-negative") +from sqltide._native import JanitorApplyResult as JanitorApplyResult +from sqltide._native import JanitorPreviewCandidate as JanitorPreviewCandidate +from sqltide._native import JanitorPreviewResult as JanitorPreviewResult +from sqltide._native import JanitorRequest as JanitorRequest diff --git a/src/sqltide/executor/observability/constants.py b/src/sqltide/executor/observability/constants.py index b648504e..85acfade 100644 --- a/src/sqltide/executor/observability/constants.py +++ b/src/sqltide/executor/observability/constants.py @@ -1,8 +1,8 @@ """Boundaries for persisted terminal observations.""" -MAX_OBSERVATION_JSON_BYTES: int = 16_384 -MAX_OBSERVATION_ERROR_LENGTH: int = 1_048_576 -HEARTBEAT_INTERVAL_SECONDS: float = 10.0 -RUN_DISPLAY_COMMAND_ENV_VAR: str = "SQLTIDE_RUN_DISPLAY_COMMAND" -RUN_INVOCATION_ID_ENV_VAR: str = "SQLTIDE_RUN_INVOCATION_ID" -PROJECT_REVISION_ENV_VAR: str = "SQLTIDE_PROJECT_REVISION" +from sqltide._native import HEARTBEAT_INTERVAL_SECONDS as HEARTBEAT_INTERVAL_SECONDS +from sqltide._native import MAX_OBSERVATION_ERROR_LENGTH as MAX_OBSERVATION_ERROR_LENGTH +from sqltide._native import MAX_OBSERVATION_JSON_BYTES as MAX_OBSERVATION_JSON_BYTES +from sqltide._native import PROJECT_REVISION_ENV_VAR as PROJECT_REVISION_ENV_VAR +from sqltide._native import RUN_DISPLAY_COMMAND_ENV_VAR as RUN_DISPLAY_COMMAND_ENV_VAR +from sqltide._native import RUN_INVOCATION_ID_ENV_VAR as RUN_INVOCATION_ID_ENV_VAR diff --git a/src/sqltide/executor/observability/models.py b/src/sqltide/executor/observability/models.py index 16a3fffd..1d80172d 100644 --- a/src/sqltide/executor/observability/models.py +++ b/src/sqltide/executor/observability/models.py @@ -1,50 +1,5 @@ """Inputs used to construct terminal invocation observations.""" -from dataclasses import dataclass -from pathlib import Path - -from sqltide.executor.observability.types import QualityResultTrigger - - -@dataclass(frozen=True) -class RunStartupTimings: - """Pre-execution work completed before the first durable run event.""" - - compile_ms: int - observability_ms: int - planning_ms: int - - @property - def total_ms(self) -> int: - return self.compile_ms + self.observability_ms + self.planning_ms - - -@dataclass(frozen=True) -class TerminalInvocation: - """Command outcome fields captured at one finite CLI boundary.""" - - project_dir: Path - target_identity: str - command: str - mode: str | None - outcome: str - exit_code: int - materialized_outcome: str | None - deployment_id: str | None - workflow_id: str | None - selected_node_count: int - error_message: str | None - summary: dict[str, object] - - -@dataclass(frozen=True) -class QualityResultContext: - """Trigger, logical slot, and policy snapshot for one quality attempt.""" - - trigger: QualityResultTrigger | str - scheduled_for: str | None = None - cadence_seconds: int | None = None - warmup_seconds: int = 0 - - def __post_init__(self) -> None: - object.__setattr__(self, "trigger", QualityResultTrigger(self.trigger)) +from sqltide._native import QualityResultContext as QualityResultContext +from sqltide._native import RunStartupTimings as RunStartupTimings +from sqltide._native import TerminalInvocation as TerminalInvocation diff --git a/src/sqltide/executor/observability/types.py b/src/sqltide/executor/observability/types.py index 62674528..3a556b75 100644 --- a/src/sqltide/executor/observability/types.py +++ b/src/sqltide/executor/observability/types.py @@ -1,19 +1,4 @@ """Closed observability value domains.""" -from enum import StrEnum - - -class QualityResultTrigger(StrEnum): - """The execution path that produced one quality result.""" - - BUILD = "build" - MANUAL = "manual" - SCHEDULED = "scheduled" - DEPLOYMENT = "deployment" - - -class MaterializationOutcome(StrEnum): - """Whether a direct mutation workflow safely completed.""" - - APPLIED = "applied" - FAILED = "failed" +from sqltide._native import MaterializationOutcome as MaterializationOutcome +from sqltide._native import QualityResultTrigger as QualityResultTrigger diff --git a/src/sqltide/executor/readiness/constants.py b/src/sqltide/executor/readiness/constants.py index a78b028c..323f45ef 100644 --- a/src/sqltide/executor/readiness/constants.py +++ b/src/sqltide/executor/readiness/constants.py @@ -1,4 +1,4 @@ """Stable constants for audit backfill.""" -ACCEPTABLE_LAG_SECONDS: float = 30.0 -MINIMUM_STAGED_ROW_RATIO: float = 0.5 +from sqltide._native import ACCEPTABLE_LAG_SECONDS as ACCEPTABLE_LAG_SECONDS +from sqltide._native import MINIMUM_STAGED_ROW_RATIO as MINIMUM_STAGED_ROW_RATIO diff --git a/src/sqltide/executor/readiness/exceptions.py b/src/sqltide/executor/readiness/exceptions.py index 4a1e59e8..592703c9 100644 --- a/src/sqltide/executor/readiness/exceptions.py +++ b/src/sqltide/executor/readiness/exceptions.py @@ -1,5 +1,3 @@ """Audit Backfill exceptions.""" - -class DeploymentAuditExecutionError(ValueError): - """Raised when audit backfill input or state is invalid.""" +from sqltide._native import DeploymentAuditExecutionError as DeploymentAuditExecutionError diff --git a/src/sqltide/executor/readiness/models.py b/src/sqltide/executor/readiness/models.py index 901a2a1a..06431b59 100644 --- a/src/sqltide/executor/readiness/models.py +++ b/src/sqltide/executor/readiness/models.py @@ -1,191 +1,10 @@ """Audit backfill runtime models.""" -from dataclasses import dataclass, field - -from sqltide.compiler.compile.models import ObjectKey -from sqltide.compiler.discovery.types import ReplayLineageMode -from sqltide.executor.auditing.models import SqlAuditResult -from sqltide.executor.readiness.constants import ( - ACCEPTABLE_LAG_SECONDS, - MINIMUM_STAGED_ROW_RATIO, -) -from sqltide.executor.readiness.types import AuditAssessment - - -@dataclass(frozen=True) -class DeploymentReadinessThresholds: - """Effective thresholds used to assess one staged deployment.""" - - maximum_lag_seconds: float = ACCEPTABLE_LAG_SECONDS - minimum_staged_row_ratio: float = MINIMUM_STAGED_ROW_RATIO - - -@dataclass(frozen=True) -class DeploymentAuditRequest: - """Input required to audit a staged backfill deployment.""" - - deployment_id: str | None - metadata_database: str - default_database: str - thresholds: DeploymentReadinessThresholds = field(default_factory=DeploymentReadinessThresholds) - - -@dataclass(frozen=True) -class AuditDeploymentCandidate: - """A deployment candidate surfaced when audit resolution is ambiguous.""" - - deployment_id: str - created_at: str | None = None - deployment_status: str | None = None - root_names: tuple[str, ...] = () - - -@dataclass(frozen=True) -class LoadedAuditDeployment: - """Persisted deployment metadata used by metadata-backed flows.""" - - deployment_id: str - created_at: str - status: str - replay_lineage_mode: ReplayLineageMode | str | None - warning_codes: tuple[str, ...] - root_keys: tuple[ObjectKey, ...] - prepared_object_mappings: tuple[tuple[ObjectKey, str], ...] - - def __post_init__(self) -> None: - if self.replay_lineage_mode is not None: - object.__setattr__( - self, - "replay_lineage_mode", - ReplayLineageMode(self.replay_lineage_mode), - ) - - -@dataclass(frozen=True) -class OffsetCatchupSummary: - """Partitioned catch-up metrics for offset-based replay.""" - - active_partition_count: int - staged_partition_count: int - partitions_compared: int - missing_staged_partition_count: int - missing_freshness_partition_count: int - lagging_partition_count: int - max_offset_gap: int - average_offset_gap: float - lag_boundary_column: str | None - max_lag_seconds: float | None - average_lag_seconds: float | None - - -@dataclass(frozen=True) -class ScalarCatchupSummary: - """Scalar catch-up metrics for timestamp-based replay.""" - - active_min_value: str | None - active_max_value: str | None - staged_min_value: str | None - staged_max_value: str | None - lag_seconds: float | None - - -@dataclass(frozen=True) -class RootAuditResult: - """Audit summary for one rebuilt root target.""" - - root_key: ObjectKey - staged_physical_name: str - staged_exists: bool - active_exists: bool - active_row_count: int | None - staged_row_count: int | None - row_delta: int | None - row_ratio: float | None - assessment: AuditAssessment | str - replay_lineage_mode: ReplayLineageMode | str | None - offset_catchup_summary: OffsetCatchupSummary | None - scalar_catchup_summary: ScalarCatchupSummary | None - state: str - replay_source_name: str | None - replay_source_row_count: int | None - warnings: tuple[str, ...] = () - - def __post_init__(self) -> None: - object.__setattr__(self, "assessment", AuditAssessment(self.assessment)) - if self.replay_lineage_mode is not None: - object.__setattr__( - self, - "replay_lineage_mode", - ReplayLineageMode(self.replay_lineage_mode), - ) - - -@dataclass(frozen=True) -class DeploymentAuditResult: - """Structured audit result for a staged backfill deployment.""" - - deployment_id: str - deployment_status: str - assessment: AuditAssessment | str - replay_lineage_mode: ReplayLineageMode | str | None - warning_codes: tuple[str, ...] - root_results: tuple[RootAuditResult, ...] - quality_check_results: tuple[SqlAuditResult, ...] = () - thresholds: DeploymentReadinessThresholds = field(default_factory=DeploymentReadinessThresholds) - - def __post_init__(self) -> None: - object.__setattr__(self, "assessment", AuditAssessment(self.assessment)) - if self.replay_lineage_mode is not None: - object.__setattr__( - self, - "replay_lineage_mode", - ReplayLineageMode(self.replay_lineage_mode), - ) - - -@dataclass(frozen=True) -class CountQueryRow: - """Row shape for single-value count projections.""" - - value: int - - -@dataclass(frozen=True) -class ColumnNameSystemRow: - """Row shape for system column-name lookups.""" - - name: str - - -@dataclass(frozen=True) -class OffsetSummaryQueryRow: - """Row shape for offset replay catch-up summaries.""" - - active_partition_count: int - staged_partition_count: int - partitions_compared: int - missing_staged_partition_count: int - missing_freshness_partition_count: int - lagging_partition_count: int - max_offset_gap: int - average_offset_gap: float - max_lag_seconds: float | None - average_lag_seconds: float | None - - -@dataclass(frozen=True) -class ScalarSummaryQueryRow: - """Row shape for timestamp replay catch-up summaries.""" - - active_min_value: str | None - active_max_value: str | None - staged_min_value: str | None - staged_max_value: str | None - lag_seconds: float | None - - -@dataclass(frozen=True) -class CreateTableQueryRow: - """Row shape for create-table DDL lookups from system tables.""" - - create_table_query: str +from sqltide._native import AuditDeploymentCandidate as AuditDeploymentCandidate +from sqltide._native import DeploymentAuditRequest as DeploymentAuditRequest +from sqltide._native import DeploymentAuditResult as DeploymentAuditResult +from sqltide._native import DeploymentReadinessThresholds as DeploymentReadinessThresholds +from sqltide._native import LoadedAuditDeployment as LoadedAuditDeployment +from sqltide._native import OffsetCatchupSummary as OffsetCatchupSummary +from sqltide._native import RootAuditResult as RootAuditResult +from sqltide._native import ScalarCatchupSummary as ScalarCatchupSummary diff --git a/src/sqltide/executor/readiness/types.py b/src/sqltide/executor/readiness/types.py index 30708165..85bae6f0 100644 --- a/src/sqltide/executor/readiness/types.py +++ b/src/sqltide/executor/readiness/types.py @@ -1,9 +1,3 @@ """Audit backfill domain types.""" -from enum import StrEnum - - -class AuditAssessment(StrEnum): - READY = "ready" - NOT_READY = "not_ready" - CAUTION = "caution" +from sqltide._native import AuditAssessment as AuditAssessment diff --git a/src/sqltide/executor/reconcile/constants.py b/src/sqltide/executor/reconcile/constants.py index 1bf5845c..be72fbb9 100644 --- a/src/sqltide/executor/reconcile/constants.py +++ b/src/sqltide/executor/reconcile/constants.py @@ -1,3 +1,3 @@ """Constants for reconcile execution.""" -RECONCILE_DEPLOYMENT_ID_PREFIX: str = "reconcile_" +from sqltide._native import RECONCILE_DEPLOYMENT_ID_PREFIX as RECONCILE_DEPLOYMENT_ID_PREFIX diff --git a/src/sqltide/executor/reconcile/exceptions.py b/src/sqltide/executor/reconcile/exceptions.py index 12af5129..1d83be1d 100644 --- a/src/sqltide/executor/reconcile/exceptions.py +++ b/src/sqltide/executor/reconcile/exceptions.py @@ -1,5 +1,3 @@ """Reconcile execution exceptions.""" - -class ReconcileError(RuntimeError): - """Raised when verified reconcile state cannot be persisted safely.""" +from sqltide._native import ReconcileError as ReconcileError diff --git a/src/sqltide/executor/reconcile/models.py b/src/sqltide/executor/reconcile/models.py index 53669302..c4a38de6 100644 --- a/src/sqltide/executor/reconcile/models.py +++ b/src/sqltide/executor/reconcile/models.py @@ -1,43 +1,5 @@ """Runtime models for reconcile execution.""" -from __future__ import annotations - -from dataclasses import dataclass - -from sqltide.compiler.compile.models import ( - ObjectKey, -) -from sqltide.compiler.planner.models import ( - ObjectStateRecord, -) - - -@dataclass(frozen=True) -class ReconcileCandidate: - key: ObjectKey - target_key: ObjectKey - target_name: str - object_type: str - - -@dataclass(frozen=True) -class ReconcileRejectedTarget: - target_key: ObjectKey - target_name: str - reasons: tuple[str, ...] - - -@dataclass(frozen=True) -class ReconcilePreview: - database: str - reconcile_id: str - eligible_records: tuple[ObjectStateRecord, ...] - rejected_targets: tuple[ReconcileRejectedTarget, ...] - - -@dataclass(frozen=True) -class ReconcileResult: - database: str - reconcile_id: str - reconciled_records: tuple[ObjectStateRecord, ...] - rejected_targets: tuple[ReconcileRejectedTarget, ...] +from sqltide._native import ReconcilePreview as ReconcilePreview +from sqltide._native import ReconcileRejectedTarget as ReconcileRejectedTarget +from sqltide._native import ReconcileResult as ReconcileResult diff --git a/src/sqltide/executor/testing/exceptions.py b/src/sqltide/executor/testing/exceptions.py index 7fea9fdb..9bfec059 100644 --- a/src/sqltide/executor/testing/exceptions.py +++ b/src/sqltide/executor/testing/exceptions.py @@ -1,5 +1,3 @@ """Testing exceptions.""" - -class SqlTestExecutionError(ValueError): - """Raised when testing input or state is invalid.""" +from sqltide._native import SqlTestExecutionError as SqlTestExecutionError diff --git a/src/sqltide/executor/testing/models.py b/src/sqltide/executor/testing/models.py index 2c463c63..05b95a43 100644 --- a/src/sqltide/executor/testing/models.py +++ b/src/sqltide/executor/testing/models.py @@ -1,39 +1,4 @@ """Runtime models for SQL-native test execution.""" -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path - - -@dataclass(frozen=True) -class ComparisonRows: - """Missing and unexpected rows expanded per comparison case.""" - - missing: tuple[tuple[tuple[object, ...], ...], ...] - unexpected: tuple[tuple[tuple[object, ...], ...], ...] - - -@dataclass(frozen=True) -class SqlTestTargetExecutionResult: - """One executed target comparison inside a SQL-native test scenario.""" - - target_model_name: str - passed: bool - column_names: tuple[str, ...] - missing_rows: tuple[tuple[object, ...], ...] - unexpected_rows: tuple[tuple[object, ...], ...] - - -@dataclass(frozen=True) -class SqlTestExecutionResult: - """One executed SQL test scenario and its aggregate diff outcome.""" - - file_path: Path - passed: bool - target_results: tuple[SqlTestTargetExecutionResult, ...] - executed_sql: str - warnings: tuple[str, ...] = () - name: str | None = None - test_index: int = 1 - error_message: str | None = None +from sqltide._native import SqlTestExecutionResult as SqlTestExecutionResult +from sqltide._native import SqlTestTargetExecutionResult as SqlTestTargetExecutionResult From a02c673cade0f8f6227fa052dbfbbe13e4d72616 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 03:12:12 +0000 Subject: [PATCH 2/3] refactor: move CLI, auth, workflow and dev server shapes to the native core CLI, auth, authorization, workflow, sensor error, dev server and diagnostic shapes, enums, exceptions and constants are native classes; the old modules re-export them. Account request bodies stay pydantic models that the native dev server validates, and the dev server types module keeps its protocols and type aliases. Movable Python drops from 3747 to 2685 lines. --- .../sqltide-python/src/shapes/cli_service.rs | 1091 +++++++++++++++++ .../sqltide-python/src/shapes/executor_ops.rs | 12 +- crates/sqltide-python/src/shapes/mod.rs | 24 +- .../src/shapes/planner_execution.rs | 14 +- python-boundary.toml | 12 +- scripts/shape_surface/constants.py | 6 + src/sqltide/_native.pyi | 599 ++++++++- src/sqltide/auth/constants.py | 16 +- src/sqltide/auth/exceptions.py | 39 +- src/sqltide/auth/models.py | 133 +- src/sqltide/auth/types.py | 27 +- src/sqltide/authorization/models.py | 60 +- src/sqltide/authorization/types.py | 15 +- src/sqltide/cli/admin/constants.py | 2 +- src/sqltide/cli/admin/types.py | 14 +- src/sqltide/cli/build/constants.py | 13 +- src/sqltide/cli/build/exceptions.py | 6 +- src/sqltide/cli/build/models.py | 141 +-- src/sqltide/cli/compile/exceptions.py | 4 +- src/sqltide/cli/destruction/models.py | 20 +- src/sqltide/cli/dev/models.py | 26 +- src/sqltide/cli/entry/constants.py | 56 +- src/sqltide/cli/entry/exceptions.py | 8 +- src/sqltide/cli/entry/models.py | 27 +- src/sqltide/cli/entry/types.py | 38 +- src/sqltide/cli/plan/models.py | 21 +- src/sqltide/cli/selection/models.py | 16 +- src/sqltide/dev_server/constants.py | 15 +- src/sqltide/dev_server/exceptions.py | 44 +- src/sqltide/dev_server/types.py | 81 +- src/sqltide/diagnostics/models.py | 45 +- src/sqltide/executor/workflow/constants.py | 21 +- src/sqltide/executor/workflow/models.py | 81 +- src/sqltide/sensors/constants.py | 2 +- src/sqltide/sensors/exceptions.py | 41 +- 35 files changed, 1797 insertions(+), 973 deletions(-) create mode 100644 crates/sqltide-python/src/shapes/cli_service.rs diff --git a/crates/sqltide-python/src/shapes/cli_service.rs b/crates/sqltide-python/src/shapes/cli_service.rs new file mode 100644 index 00000000..df3acc9b --- /dev/null +++ b/crates/sqltide-python/src/shapes/cli_service.rs @@ -0,0 +1,1091 @@ +//! Shapes of the CLI, accounts and authorization, warehouse workflows, sensors, the dev server +//! and diagnostics, plus their value enums, exceptions and constants. + +use pyo3::exceptions::{PyException, PyRuntimeError, PyValueError}; +use pyo3::prelude::*; +use pyo3::types::{IntoPyDict, PyDict, PyFrozenSet, PyTuple}; + +use super::{ + Coerce, Given, add, add_class, base_init, exception, member_set, native_enum, raised, shapes, + str_enum, +}; + +const WORKFLOW_PHASES: [&str; 9] = [ + "PREFLIGHT", + "PREPARATION", + "TEARDOWN", + "REALIZATION", + "STABILIZATION", + "BOUNDARY", + "REPLAY", + "AUDIT", + "FINALIZATION", +]; + +/// `importlib.metadata.version("sqltide")`, read on first use. +pub(super) fn tool_version(py: Python<'_>) -> PyResult> { + py.import("importlib.metadata")? + .getattr("version")? + .call1(("sqltide",)) +} + +/// The workflow phases in execution order, built on first use because `WorkflowPhase` is still +/// defined in Python. +pub(super) fn workflow_phase_order(py: Python<'_>) -> PyResult> { + let phase: Bound<'_, PyAny> = py + .import("sqltide.executor.workflow.types")? + .getattr("WorkflowPhase")?; + let mut members: Vec> = Vec::with_capacity(WORKFLOW_PHASES.len()); + for name in WORKFLOW_PHASES { + members.push(phase.getattr(name)?); + } + Ok(PyTuple::new(py, members)?.into_any()) +} + +/// `WorkflowValidationError(message)`, from the Python workflow exceptions. +fn invalid_workflow(py: Python<'_>, message: String) -> PyResult { + Ok(PyErr::from_value( + py.import("sqltide.executor.workflow.exceptions")? + .getattr("WorkflowValidationError")? + .call1((message,))?, + )) +} + +/// `WarehouseStatement`: a word-character step ID and SQL ending in exactly one semicolon. +fn warehouse_statement(values: &mut [Bound<'_, PyAny>]) -> PyResult<()> { + let [_, step_id, _, _, sql, ..] = values else { + return Err(PyValueError::new_err( + "warehouse statement bound too few values", + )); + }; + let py: Python<'_> = step_id.py(); + if !step_id.is_truthy()? + || !step_id + .call_method1("replace", ("_", ""))? + .call_method0("isalnum")? + .is_truthy()? + { + return Err(invalid_workflow( + py, + "Workflow step IDs may contain only letters, numbers, and underscores".to_owned(), + )?); + } + if !sql.call_method1("endswith", (";",))?.is_truthy()? + || sql.call_method1("endswith", (";;",))?.is_truthy()? + { + return Err(invalid_workflow( + py, + format!( + "Workflow statement {} must end with exactly one semicolon", + step_id.repr()? + ), + )?); + } + Ok(()) +} + +/// `BuildWorkflow`: statement sequences run continuously from one and phases never go back. +fn build_workflow(values: &mut [Bound<'_, PyAny>]) -> PyResult<()> { + let [_, _, statements, ..] = values else { + return Err(PyValueError::new_err("build workflow bound too few values")); + }; + let py: Python<'_> = statements.py(); + let order: Bound<'_, PyAny> = py + .import("sqltide._native")? + .getattr("WORKFLOW_PHASE_ORDER")?; + let mut previous_phase_index: usize = 0; + for (expected_sequence, statement) in (1_i64..).zip(statements.try_iter()?) { + let statement: Bound<'_, PyAny> = statement?; + if statement.getattr("sequence")?.ne(expected_sequence)? { + return Err(invalid_workflow( + py, + "Workflow statement sequences must be continuous from one".to_owned(), + )?); + } + let phase_index: usize = order + .call_method1("index", (statement.getattr("phase")?,))? + .extract()?; + if phase_index < previous_phase_index { + return Err(invalid_workflow( + py, + "Workflow phases must be monotonic".to_owned(), + )?); + } + previous_phase_index = phase_index; + } + Ok(()) +} + +/// `AuthSettings`: a positive session TTL, and trusted-proxy auto-provisioning only as viewer. +fn auth_settings(values: &mut [Bound<'_, PyAny>]) -> PyResult<()> { + let [ + mode, + _, + _, + _, + _, + unknown_user_policy, + default_role, + _, + session_ttl_seconds, + .., + ] = values + else { + return Err(PyValueError::new_err("auth settings bound too few values")); + }; + let py: Python<'_> = mode.py(); + if session_ttl_seconds.le(0)? { + return Err(raised( + py, + "AuthConfigurationError", + "Session TTL must be positive".to_owned(), + )?); + } + if mode.eq(native_enum(py, "AuthenticationMode")?.getattr("TRUSTED_PROXY")?)? + && unknown_user_policy + .eq(native_enum(py, "UnknownUserPolicy")?.getattr("AUTO_PROVISION")?)? + && default_role.ne("viewer")? + { + return Err(raised( + py, + "AuthConfigurationError", + "Trusted-proxy auto-provisioning default role must be 'viewer'".to_owned(), + )?); + } + Ok(()) +} + +shapes! { + module = "sqltide.diagnostics.models"; + + /// One one-based source span. + SourceLocation { + path, + line, + column, + end_line(default = Given::None), + end_column(default = Given::None), + } + + /// One labeled source span related to a primary diagnostic. + RelatedDiagnosticLocation { label, location, message(default = Given::None) } + + /// One structured compiler or runtime diagnostic. + CompilerDiagnostic { + phase(coerce = Coerce::Enum("DiagnosticPhase")), + severity(coerce = Coerce::Enum("DiagnosticSeverity")), + code, + message, + resource_name(default = Given::None), + location(default = Given::None), + related_locations(default = Given::EmptyTuple), + help(default = Given::None), + } +} + +shapes! { + module = "sqltide.auth.models"; + + /// Authenticated request identity, independent of authorization policy. + Principal { user_id, username, display_name, email, authentication_source } + + /// Server-runtime authentication and control-store settings. + AuthSettings @no_repr { + mode, + control_store_url, + username_header(default = Given::Str("X-SQLTide-User")), + display_name_header(default = Given::None), + email_header(default = Given::None), + unknown_user_policy(default = Given::Member("UnknownUserPolicy", "AUTO_PROVISION")), + default_role(default = Given::Str("viewer")), + session_cookie_name(default = Given::Str("sqltide_session")), + session_ttl_seconds(default = Given::Int(43_200)), + session_cookie_secure(default = Given::Bool(true)), + proxy_logout_url(default = Given::None), + } + @post_init(auth_settings) + @impl { + fn __repr__(&self, py: Python<'_>) -> PyResult { + let driver: Bound<'_, PyAny> = self + .control_store_url + .bind(py) + .call_method1("partition", (":",))? + .get_item(0)?; + Ok(format!( + "AuthSettings(mode={}, control_store_url='', username_header={}, \ + unknown_user_policy={}, default_role={})", + self.mode.bind(py).repr()?, + driver.str()?, + self.username_header.bind(py).repr()?, + self.unknown_user_policy.bind(py).repr()?, + self.default_role.bind(py).repr()?, + )) + } + } + + /// Persisted SQLTide account without credential material. + UserAccount { + user_id, + username, + display_name, + email, + is_active, + created_at, + updated_at, + roles(default = Given::EmptyTuple), + authentication_sources(default = Given::EmptyTuple), + } + + /// One newly issued browser session; only the hash is persisted. + SessionCredentials { token, csrf_token, expires_at } + + /// Active browser session and its authenticated principal. + ResolvedSession { principal, roles, csrf_token, expires_at } + + /// Principal plus request-local account/session state. + AuthenticatedRequest { principal, roles, csrf_token(default = Given::None) } + + /// Non-secret account administration audit event. + AccountAuditRecord { operation, actor_user_id, affected_user_id, occurred_at, details } + + /// Audited user membership in one project-authored operational role. + ProjectRoleAssignment { + assignment_id, + user_id, + project_name, + role_name, + target_name, + assigned_by, + assigned_at, + revoked_by(default = Given::None), + revoked_at(default = Given::None), + } +} + +shapes! { + module = "sqltide.authorization.models"; + + /// Authoritative identity, policy, and resolved operation scope. + AuthorizationRequest { + authenticated, + permission, + project_name, + target_name, + grant_scope, + affected_pipelines, + policy, + } + + /// Explainable allow or deny result from centralized evaluation. + AuthorizationDecision { + allowed, + reason, + permission, + project_name, + target_name, + affected_pipelines, + matched_roles(default = Given::EmptyTuple), + missing_pipelines(default = Given::EmptyTuple), + } + + /// Identity and policy inputs for one effective-capability summary. + CapabilityRequest { authenticated, project_name, target_name, pipeline_names, policy } + + /// UI-facing effective operation permissions for one user. + EffectiveCapabilities { + system_admin, + project_name, + target_name, + permissions, + pipeline_permissions, + stale_roles(default = Given::EmptyTuple), + } +} + +shapes! { + module = "sqltide.executor.workflow.models"; + + WarehouseStatement { + sequence, + step_id, + phase, + intent, + sql, + continue_on_error(default = Given::Bool(false)), + display_name(default = Given::None), + replay_offset_progress(default = Given::None), + } + @post_init(warehouse_statement) + + BuildWorkflow { + mode, + plan_json, + statements, + recovery_statements(default = Given::EmptyTuple), + } + @post_init(build_workflow) + + PublishedBuildWorkflow { workflow, artifact_root, workflow_sha256 } + + WorkflowStatementResult { + step_id, + query_result, + mutation_result, + error_message(default = Given::None), + } + + WorkflowExecutionResult { statement_results } +} + +shapes! { + module = "sqltide.cli.build.models"; + + /// Every operator-supplied option for one `sqt build` invocation. + BuildCommandOptions { + pipelines_root, + database, + metadata_database, + selectors, + json_output, + verbose, + auto_approve, + deployment_id(default = Given::None), + full_refresh(default = Given::Bool(false)), + start_time(default = Given::None), + events_output(default = Given::Bool(false)), + confirmations(default = Given::EmptyTuple), + changed(default = Given::Bool(false)), + include_missing_upstream(default = Given::Bool(false)), + } + + /// One protected pipeline touched by the resolved build closure. + BuildProtectionRequirement { pipeline_name, warning, confirmation } + + /// Connected planning options shared by `sqt plan` and `sqt build`. + WorkflowPreparationOptions { + database, + metadata_database, + selectors, + deployment_id, + full_refresh, + start_time, + verbose, + changed(default = Given::Bool(false)), + include_missing_upstream(default = Given::Bool(false)), + } + + /// The plan and resolved databases a build renders before it writes anything. + DirectBuildPreviewContext { + analysis, + plan, + warehouse_snapshot, + database, + metadata_database, + adapter_name, + effective_start_time(default = Given::None), + row_error_policy_changes(default = Given::EmptyTuple), + } + + /// The confirmed virtual deployment plan and its fixed execution identity. + VirtualBuildPreviewContext { + database, + metadata_database, + desired_state, + plan, + replay_lineage_mode, + deployment_id, + created_at, + root_reports, + existing_relation_names, + target_catalog, + metadata_catalog, + full_refresh_keys(default = Given::EmptyFrozenSet), + start_time_keys(default = Given::EmptyFrozenSet), + start_time(default = Given::None), + execution_logical_model_keys(default = Given::EmptyFrozenSet), + run_execution_scope(default = Given::EmptyTuple), + run_context_scope(default = Given::EmptyTuple), + row_error_policy_changes(default = Given::EmptyTuple), + } + + /// One fully assembled direct workflow and its result-decoding request. + DirectWorkflowPreparation { + preview, + request, + workflow, + plan_text, + protection_requirements(default = Given::EmptyTuple), + } + + /// One fully assembled virtual workflow and its result-decoding request. + VirtualWorkflowPreparation { + preview, + request, + workflow, + plan_text, + protection_requirements(default = Given::EmptyTuple), + } + + /// Virtual-first and direct-second workflows prepared for one build. + MixedWorkflowPreparation { + r#virtual, + direct, + plan_text, + plan_json, + protection_requirements(default = Given::EmptyTuple), + } +} + +shapes! { + module = "sqltide.cli.destruction.models"; + + DestructionCommandOptions @no_repr { + operation, + pipelines_root, + project_dir, + selected_target, + database, + selectors, + control_store_url, + include_orphans(default = Given::Bool(false)), + cli_variables(default = Given::EmptyTuple), + environment(default = Given::None), + } +} + +shapes! { + module = "sqltide.cli.dev.models"; + + /// Parsed `sqt dev` invocation options. + DevCommandOptions @no_repr { + pipelines_root, + database, + host, + port, + selected_target(default = Given::None), + cli_variables(default = Given::EmptyTuple), + environment(default = Given::None), + connection_host(default = Given::None), + connection_port(default = Given::None), + connection_username(default = Given::None), + connection_password(default = Given::None), + auth_settings(default = Given::None), + } +} + +shapes! { + module = "sqltide.cli.entry.models"; + + CliEntrypointHandlers { + run_discover, + run_compile, + run_test, + run_audit, + run_plan, + run_build, + run_destruction, + run_deployment_diff, + run_deployment_list, + run_deployment_show, + run_deployment_audit, + run_deployment_promote, + run_deployment_rollback, + run_reconcile, + run_janitor, + run_doctor, + run_repair_active_view, + run_dev, + } +} + +shapes! { + module = "sqltide.cli.plan.models"; + + /// Every operator-supplied option for one `sqt plan` invocation. + PlanCommandOptions { + pipelines_root, + database, + selectors, + deployment_id, + full_refresh, + start_time, + json_output, + verbose, + changed(default = Given::Bool(false)), + include_missing_upstream(default = Given::Bool(false)), + } +} + +shapes! { + module = "sqltide.cli.selection.models"; + + SelectionResolution { + desired_state, + selected_logical_model_keys, + selected_model_keys, + replay_lineage_mode, + execution_logical_model_keys(default = Given::EmptyFrozenSet), + } +} + +/// A native enum: its class, public module, docstring and members. +type EnumSpec = ( + &'static str, + &'static str, + Option<&'static str>, + &'static [(&'static str, &'static str)], +); + +const DEV_SERVER_TYPES: &str = "sqltide.dev_server.types"; + +const ENUMS: [EnumSpec; 14] = [ + ( + "AdminCommand", + "sqltide.cli.admin.types", + Some("Supported operator account-administration commands."), + &[ + ("MIGRATE", "migrate"), + ("CREATE_USER", "create-user"), + ("GRANT_ROLE", "grant-role"), + ("REVOKE_ROLE", "revoke-role"), + ("ENABLE_USER", "enable-user"), + ("DISABLE_USER", "disable-user"), + ("RESET_PASSWORD", "reset-password"), + ], + ), + ( + "CliCommand", + "sqltide.cli.entry.types", + Some("One top-level `sqt` subcommand as argparse reports it."), + &[ + ("ADMIN", "admin"), + ("PLAYGROUND", "playground"), + ("DISCOVER", "discover"), + ("DEV", "dev"), + ("COMPILE", "compile"), + ("LINT", "lint"), + ("FORMAT", "format"), + ("TEST", "test"), + ("PLAN", "plan"), + ("BUILD", "build"), + ("DESTROY", "destroy"), + ("RESET_TARGET", "reset-target"), + ("DEPLOYMENT", "deployment"), + ("AUDIT", "audit"), + ("RECONCILE", "reconcile"), + ("JANITOR", "janitor"), + ("DOCTOR", "doctor"), + ("REPAIR", "repair"), + ], + ), + ( + "CliSubcommand", + "sqltide.cli.entry.types", + Some("One nested `sqt ` selector."), + &[ + ("ACTIVE_VIEW", "active-view"), + ("DIFF", "diff"), + ("LIST", "list"), + ("SHOW", "show"), + ("AUDIT", "audit"), + ("PROMOTE", "promote"), + ("ROLLBACK", "rollback"), + ], + ), + ( + "AuthenticationMode", + "sqltide.auth.types", + Some("Configured server authentication mechanism."), + &[ + ("DISABLED", "disabled"), + ("PASSWORD", "password"), + ("TRUSTED_PROXY", "trusted_proxy"), + ], + ), + ( + "AuthenticationSource", + "sqltide.auth.types", + Some("Mechanism that authenticated one request."), + &[ + ("LOCAL", "local"), + ("PASSWORD", "password"), + ("TRUSTED_PROXY", "trusted_proxy"), + ], + ), + ( + "UnknownUserPolicy", + "sqltide.auth.types", + Some("Trusted-proxy behavior for an identity absent from the control store."), + &[("AUTO_PROVISION", "auto_provision"), ("DENY", "deny")], + ), + ( + "AuthorizationReason", + "sqltide.authorization.types", + Some("Stable explanation category for one operational decision."), + &[ + ("SYSTEM_ADMIN", "system_admin"), + ("GRANTED", "granted"), + ("POLICY_ABSENT", "policy_absent"), + ("NO_MATCHING_ASSIGNMENT", "no_matching_assignment"), + ("STALE_ASSIGNMENT", "stale_assignment"), + ("EMPTY_PIPELINE_SCOPE", "empty_pipeline_scope"), + ("MISSING_PERMISSION", "missing_permission"), + ("MISSING_PIPELINES", "missing_pipelines"), + ], + ), + ( + "ActivityTone", + DEV_SERVER_TYPES, + Some("How an activity line should read: routine, good news, bad news, or caution."), + &[ + ("NEUTRAL", "neutral"), + ("GOOD", "good"), + ("BAD", "bad"), + ("CAUTION", "caution"), + ], + ), + ( + "RunPresentationStatus", + DEV_SERVER_TYPES, + Some("Terminal facts and reversible liveness states shown for a run."), + &[ + ("SUCCEEDED", "succeeded"), + ("FAILED", "failed"), + ("CANCELLED", "cancelled"), + ("CANCELLING", "cancelling"), + ("CANCELLATION_FAILED", "cancellation_failed"), + ("RUNNING", "running"), + ("UNRESPONSIVE", "unresponsive"), + ("PRESUMED_FAILED", "presumed_failed"), + ], + ), + ( + "CompileStateKind", + DEV_SERVER_TYPES, + Some("Whether the held project compile is servable or failing."), + &[("OK", "ok"), ("FAILING", "failing")], + ), + ( + "ReplayAnchorReason", + DEV_SERVER_TYPES, + Some("Why a model is or is not a replay anchor."), + &[ + ("ELIGIBLE", "eligible"), + ("AGGREGATE", "aggregate"), + ("MUTABLE_REF", "mutable_ref"), + ("NEVER", "never"), + ("LINEAGE_LOSS", "lineage_loss"), + ("VIEW", "view"), + ], + ), + ( + "Freshness", + DEV_SERVER_TYPES, + Some("Derived source or model freshness against its authored policy."), + &[ + ("FRESH", "fresh"), + ("LAGGING", "lagging"), + ("STALLED", "stalled"), + ], + ), + ( + "SensorSchedulerState", + DEV_SERVER_TYPES, + Some("Current lifecycle disposition of the hosted sensor dispatcher."), + &[ + ("DISABLED", "disabled"), + ("IDLE", "idle"), + ("RUNNING", "running"), + ("STANDBY", "standby"), + ("BACKING_OFF", "backing_off"), + ], + ), + ( + "AuditScheduleState", + DEV_SERVER_TYPES, + Some("Current scheduler disposition of one compiled audit or target."), + &[ + ("DISABLED", "disabled"), + ("IDLE", "idle"), + ("DUE", "due"), + ("SCHEDULED", "scheduled"), + ("WARMING_UP", "warming_up"), + ("NOT_MATERIALIZED", "not_materialized"), + ("RUNNING", "running"), + ("BLOCKED", "blocked"), + ("BACKING_OFF", "backing_off"), + ], + ), +]; + +fn enums(module: &Bound<'_, PyModule>) -> PyResult<()> { + for (name, public_module, doc, members) in ENUMS { + str_enum(module, name, public_module, name, doc, members)?; + } + Ok(()) +} + +#[pyfunction] +#[pyo3(name = "__init__", signature = (instance, message, *, hint = None))] +fn cli_user_init<'py>( + instance: &Bound<'py, PyAny>, + message: Bound<'py, PyAny>, + hint: Option>, +) -> PyResult<()> { + base_init(instance, [message])?; + instance.setattr("hint", hint) +} + +#[pyfunction] +#[pyo3( + name = "__init__", + signature = ( + instance, + message, + *, + location = None, + related_locations = Python::attach(|py| PyTuple::empty(py).into_any().unbind()), + ) +)] +fn sensor_init<'py>( + py: Python<'py>, + instance: &Bound<'py, PyAny>, + message: Bound<'py, PyAny>, + location: Option>, + related_locations: Py, +) -> PyResult<()> { + base_init(instance, [message.clone()])?; + instance.setattr("location", &location)?; + let keywords: Bound<'py, PyDict> = [ + ( + "phase", + native_enum(py, "DiagnosticPhase")?.getattr("COMPILATION")?, + ), + ( + "severity", + native_enum(py, "DiagnosticSeverity")?.getattr("ERROR")?, + ), + ("code", "SQT-COMPILE-001".into_pyobject(py)?.into_any()), + ("message", message), + ( + "location", + location.unwrap_or_else(|| py.None().into_bound(py)), + ), + ("related_locations", related_locations.into_bound(py)), + ] + .into_py_dict(py)?; + let diagnostic: Bound<'py, PyAny> = py + .get_type::() + .call((), Some(&keywords))?; + instance.setattr("diagnostic", diagnostic) +} + +/// Plain exceptions: public module, name, docstring and the registered parent. +const PLAIN_ERRORS: [(&str, &str, &str, &str); 20] = [ + ( + "sqltide.auth.exceptions", + "AuthenticationError", + "A request could not produce an authenticated principal.", + "Exception", + ), + ( + "sqltide.auth.exceptions", + "AuthorizationError", + "An authenticated principal cannot perform an account operation.", + "Exception", + ), + ( + "sqltide.auth.exceptions", + "ControlStoreError", + "The account control store is unavailable or incompatible.", + "Exception", + ), + ( + "sqltide.auth.exceptions", + "AuthConfigurationError", + "Authentication runtime configuration is invalid.", + "ValueError", + ), + ( + "sqltide.auth.exceptions", + "AccountValidationError", + "Account input violates a stable domain constraint.", + "ValueError", + ), + ( + "sqltide.auth.exceptions", + "PasswordValidationError", + "A password does not meet the configured security policy.", + "ValueError", + ), + ( + "sqltide.auth.exceptions", + "AccountConflictError", + "An account mutation conflicts with existing identity state.", + "ControlStoreError", + ), + ( + "sqltide.auth.exceptions", + "AccountNotFoundError", + "An account or role assignment does not exist.", + "ControlStoreError", + ), + ( + "sqltide.cli.compile.exceptions", + "CompileArtifactError", + "Raised when a compile artifact cannot be generated or published safely.", + "ValueError", + ), + ( + "sqltide.sensors.exceptions", + "SensorStepError", + "Raised when a durable step cannot run or return a memoized result.", + "RuntimeError", + ), + ( + "sqltide.sensors.exceptions", + "SensorEvaluationTimeoutError", + "Raised when one sensor evaluation exceeds its timeout.", + "RuntimeError", + ), + ( + "sqltide.dev_server.exceptions", + "DevServerError", + "Raised when the dev server cannot satisfy a request.", + "RuntimeError", + ), + ( + "sqltide.dev_server.exceptions", + "DevConfigurationError", + "Raised when retained dev server configuration is internally inconsistent.", + "DevServerError", + ), + ( + "sqltide.dev_server.exceptions", + "KafkaCollectorClosedError", + "Raised when Kafka metadata is requested after its retained clients close.", + "DevServerError", + ), + ( + "sqltide.dev_server.exceptions", + "ProjectNotCompiledError", + "Raised when a request needs definitions but the project compile is failing.", + "DevServerError", + ), + ( + "sqltide.dev_server.exceptions", + "BuildInProgressError", + "Raised when a build is requested while another one is still running.", + "DevServerError", + ), + ( + "sqltide.dev_server.exceptions", + "BuildStartError", + "Raised when a spawned build exits or stalls before its run_started event.", + "DevServerError", + ), + ( + "sqltide.dev_server.exceptions", + "AuditSchedulerPersistenceError", + "Raised when a scheduled slot cannot be durably marked as attempted.", + "DevServerError", + ), + ( + "sqltide.dev_server.exceptions", + "MessageQueryValidationError", + "Raised when a message browsing document names an unsupported field or value.", + "DevServerError", + ), + ( + "sqltide.dev_server.exceptions", + "MessageSchemaError", + "Raised when a raw landing table predates the header-array landing schema.", + "DevServerError", + ), +]; + +fn exceptions(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py: Python<'_> = module.py(); + for (public_module, name, doc, parent) in PLAIN_ERRORS { + let parent: Bound<'_, PyAny> = match parent { + "Exception" => py.get_type::().into_any(), + "ValueError" => py.get_type::().into_any(), + "RuntimeError" => py.get_type::().into_any(), + registered => module.getattr(registered)?, + }; + exception(module, public_module, name, doc, &[parent], None, None)?; + } + let cli_user: Bound<'_, PyAny> = exception( + module, + "sqltide.cli.entry.exceptions", + "CliUserError", + "Expected CLI-facing error that should be rendered without a traceback.", + &[py.get_type::().into_any()], + Some(wrap_pyfunction!(cli_user_init, module)?.into_any()), + None, + )?; + exception( + module, + "sqltide.cli.build.exceptions", + "BuildPipelineLimitError", + "The final build closure exceeds its committed target limit.", + &[cli_user], + None, + None, + )?; + exception( + module, + "sqltide.sensors.exceptions", + "SensorError", + "Raised when sensor definitions or project sensor state is invalid.", + &[py.get_type::().into_any()], + Some(wrap_pyfunction!(sensor_init, module)?.into_any()), + None, + )?; + Ok(()) +} + +const TEXT: [(&str, &str); 14] = [ + ("CONTROL_STORE_ENV_VAR", "SQLTIDE_CONTROL_STORE_URL"), + ( + "EXPECTED_BUILD_WRITE_SCOPE_ENV_VAR", + "SQLTIDE_EXPECTED_BUILD_WRITE_SCOPE", + ), + ( + "EXPECTED_BUILD_READ_SCOPE_ENV_VAR", + "SQLTIDE_EXPECTED_BUILD_READ_SCOPE", + ), + ( + "EXPECTED_BUILD_PIPELINE_SCOPE_ENV_VAR", + "SQLTIDE_EXPECTED_BUILD_PIPELINE_SCOPE", + ), + ("DEV_CLI_VARIABLES_ENV_VAR", "SQLTIDE_INTERNAL_CLI_VARS"), + ("METADATA_TABLE_NAME_PREFIX", "sqltide_"), + ("ADMIN_ROLE", "admin"), + ("VIEWER_ROLE", "viewer"), + ("ACCOUNT_SCHEMA_COMPONENT", "accounts"), + ("SQLITE_MEMORY_PATH", ":memory:"), + ("RECOVER_MODEL_VIEW_STEP_PREFIX", "recover_view_"), + ("RECOVER_SOURCE_STEP_PREFIX", "recover_source_"), + ("RECOVER_DEAD_LETTERS_STEP_PREFIX", "recover_dead_letters_"), + ("DEFAULT_DEV_SERVER_HOST", "127.0.0.1"), +]; + +fn constants(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py: Python<'_> = module.py(); + for (name, value) in TEXT { + add(module, name, value)?; + } + add( + module, + "COMMANDS_REQUIRING_PIPELINES_ROOT", + member_set( + py, + "CliCommand", + &[ + "DEV", + "DISCOVER", + "COMPILE", + "LINT", + "FORMAT", + "TEST", + "PLAN", + "BUILD", + "DESTROY", + "RESET_TARGET", + "DEPLOYMENT", + "RECONCILE", + "AUDIT", + ], + )?, + )?; + add( + module, + "COMMANDS_WITHOUT_ADAPTER_CONNECTION", + member_set(py, "CliCommand", &["DISCOVER", "COMPILE", "LINT", "FORMAT"])?, + )?; + add( + module, + "VIRTUAL_ENVIRONMENT_ONLY_COMMANDS", + member_set( + py, + "CliCommand", + &["DEPLOYMENT", "RECONCILE", "JANITOR", "DOCTOR", "REPAIR"], + )?, + )?; + let command: Bound<'_, PyAny> = native_enum(py, "CliCommand")?; + let display_names: Bound<'_, PyDict> = PyDict::new(py); + for (member, display) in [ + ("AUDIT", "audit"), + ("DESTROY", "destroy"), + ("REPAIR", "repair active-view"), + ("RESET_TARGET", "reset-target"), + ] { + display_names.set_item(command.getattr(member)?, display)?; + } + add(module, "DISPLAY_NAME_BY_COMMAND", display_names)?; + add( + module, + "AFFIRMATIVE_RESPONSES", + PyFrozenSet::new(py, ["y", "yes"])?, + )?; + add( + module, + "TRUE_ENV_VALUES", + PyFrozenSet::new(py, ["1", "true", "yes"])?, + )?; + add(module, "PASSWORD_MIN_LENGTH", 12)?; + add( + module, + "LOCAL_USER_ID", + py.import("uuid")? + .getattr("UUID")? + .call1(("a25809d6-cf14-592c-b7b6-c94ef048e0cc",))?, + )?; + add(module, "DEFAULT_SENSOR_TIMEOUT_SECONDS", 60.0)?; + add(module, "DEFAULT_DEV_SERVER_PORT", 8000)?; + add(module, "KAFKA_REUSE_LOG_FIRST_READ_COUNT", 2)?; + add(module, "KAFKA_REUSE_LOG_INTERVAL", 100)?; + add(module, "CANCEL_GRACE_SECONDS", 15.0)?; + add(module, "TERMINATE_GRACE_SECONDS", 3.0)?; + add(module, "ACTIVITY_WINDOW_SECONDS", 120) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + enums(module)?; + exceptions(module)?; + constants(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module) +} diff --git a/crates/sqltide-python/src/shapes/executor_ops.rs b/crates/sqltide-python/src/shapes/executor_ops.rs index 2b2700d5..701ece1f 100644 --- a/crates/sqltide-python/src/shapes/executor_ops.rs +++ b/crates/sqltide-python/src/shapes/executor_ops.rs @@ -7,21 +7,13 @@ use pyo3::prelude::*; use pyo3::types::{IntoPyDict, PyFrozenSet, PySet, PyTuple}; use super::{ - Coerce, Given, add, add_class, base_init, exception, factory, native_enum, shapes, str_enum, + Coerce, Given, add, add_class, base_init, exception, factory, native_enum, raised, shapes, + str_enum, }; const DESTRUCTION_EXCEPTIONS: &str = "sqltide.executor.destruction.exceptions"; const REPLAY_LINEAGE_MODE: Coerce = Coerce::Maybe(&Coerce::Enum("ReplayLineageMode")); -/// The native exception `name` raised with `message`. -fn raised(py: Python<'_>, name: &str, message: String) -> PyResult { - Ok(PyErr::from_value( - py.import("sqltide._native")? - .getattr(name)? - .call1((message,))?, - )) -} - /// `DestructionValidationError(message)`. fn invalid_destruction(py: Python<'_>, message: &str) -> PyResult { raised(py, "DestructionValidationError", message.to_owned()) diff --git a/crates/sqltide-python/src/shapes/mod.rs b/crates/sqltide-python/src/shapes/mod.rs index 1d4b936e..45a49d98 100644 --- a/crates/sqltide-python/src/shapes/mod.rs +++ b/crates/sqltide-python/src/shapes/mod.rs @@ -15,6 +15,7 @@ use pyo3::types::{ IntoPyDict, PyCFunction, PyDict, PyFloat, PyFrozenSet, PyString, PyTuple, PyType, }; +mod cli_service; mod compiler_analysis; mod compiler_project; mod executor_ops; @@ -790,6 +791,15 @@ pub(crate) fn add_class(module: &Bound<'_, PyModule>) -> Py add(module, name.to_str()?, class) } +/// The native exception `name` raised with `message`. +pub(crate) fn raised(py: Python<'_>, name: &str, message: String) -> PyResult { + Ok(PyErr::from_value( + py.import("sqltide._native")? + .getattr(name)? + .call1((message,))?, + )) +} + /// Constants built on first read, by name. static LAZY_VALUES: PyOnceLock> = PyOnceLock::new(); @@ -856,5 +866,17 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { compiler_analysis::register(module)?; compiler_project::register(module)?; planner_execution::register(module)?; - executor_ops::register(module) + executor_ops::register(module)?; + cli_service::register(module)?; + lazy_constants( + module, + &[ + ( + "CLICKHOUSE_REPLAY_BOUNDARY_MODES", + planner_execution::clickhouse_replay_boundary_modes, + ), + ("SQLTIDE_TOOL_VERSION", cli_service::tool_version), + ("WORKFLOW_PHASE_ORDER", cli_service::workflow_phase_order), + ], + ) } diff --git a/crates/sqltide-python/src/shapes/planner_execution.rs b/crates/sqltide-python/src/shapes/planner_execution.rs index 56ab5453..bbefb7b9 100644 --- a/crates/sqltide-python/src/shapes/planner_execution.rs +++ b/crates/sqltide-python/src/shapes/planner_execution.rs @@ -8,8 +8,7 @@ use pyo3::sync::PyOnceLock; use pyo3::types::{IntoPyDict, PyFrozenSet, PyString, PyTuple}; use super::{ - Coerce, Given, add, add_class, base_init, exception, lazy_constants, native_enum, path, shapes, - str_enum, + Coerce, Given, add, add_class, base_init, exception, native_enum, path, shapes, str_enum, }; const PLANNER_TYPES: &str = "sqltide.compiler.planner.types"; @@ -978,7 +977,7 @@ fn exceptions(module: &Bound<'_, PyModule>) -> PyResult<()> { /// `frozenset(AdapterReplayBoundaryMode)`, built on first read because the enum is still /// defined in Python. -fn clickhouse_replay_boundary_modes(py: Python<'_>) -> PyResult> { +pub(super) fn clickhouse_replay_boundary_modes(py: Python<'_>) -> PyResult> { let modes: Bound<'_, PyAny> = py .import("sqltide.adapter.types")? .getattr("AdapterReplayBoundaryMode")?; @@ -1223,12 +1222,5 @@ pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { add_class::(module)?; add_class::(module)?; add_class::(module)?; - add_class::(module)?; - lazy_constants( - module, - &[( - "CLICKHOUSE_REPLAY_BOUNDARY_MODES", - clickhouse_replay_boundary_modes, - )], - ) + add_class::(module) } diff --git a/python-boundary.toml b/python-boundary.toml index 63748a66..ee8358da 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -8,7 +8,7 @@ # still carries planning, validation, rendering or other movable logic counts as movable until # that logic moves to Rust. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 3747 +remaining_lines = 2685 [[stays]] path = "adapter/classes/adapter.py" @@ -220,3 +220,13 @@ path = "sensors/classes/polling_sensor_context.py" rule = 3 note = "Rule 3: PollingSensorContext that user polling sensors receive, re-exported by sqltide.sensors; read-only properties." + +[[stays]] +path = "auth/models.py" +rule = 4 +note = "Rule 4: pydantic request bodies LoginRequest, CreateUserRequest, UpdateUserRequest, PasswordResetRequest, RoleRequest and ProjectRoleRequest that the native dev server validates with pydantic for the account routes, beside native re-exports of the account shapes; no logic." + +[[stays]] +path = "dev_server/types.py" +rule = 1 +note = "Rule 1: Logic-free protocols CompileAuthorizationGuard (the compile authorization check the dev server runtime calls) and DevServerReporter (the terminal narrator the CLI implements) and the data-only type aliases KafkaClientKey and KafkaLagOffsets the dev server runtime consumes, beside native re-exports of the dev server value enums; no logic." diff --git a/scripts/shape_surface/constants.py b/scripts/shape_surface/constants.py index 4a9b81de..3f98e35d 100644 --- a/scripts/shape_surface/constants.py +++ b/scripts/shape_surface/constants.py @@ -40,6 +40,11 @@ "sqltide.adapter.constants:REPLAY_VALUE_KIND_TIMESTAMP": _NATIVE_CONSTANT, "sqltide.adapter.constants:TARGET_MUTATION_LOCK_TABLE_NAME": _NATIVE_CONSTANT, "sqltide.adapters.clickhouse.constants:CLICKHOUSE_DEFAULT_INDEX_GRANULARITY": _NATIVE_CONSTANT, + "sqltide.auth.constants:CONTROL_STORE_TABLE_PREFIX": _NATIVE_CONSTANT, + "sqltide.auth.constants:STREAMBUILD_CONTROL_STORE_TABLE_PREFIX": _NATIVE_CONSTANT, + "sqltide.cli.build.constants:BUILD_CANCELLED_MESSAGE": _NATIVE_CONSTANT, + "sqltide.cli.entry.constants:ACTIVE_VIEW_SUBCOMMAND": _NATIVE_CONSTANT, + "sqltide.cli.entry.constants:DIRECT_ONLY_COMMANDS": _NATIVE_CONSTANT, "sqltide.cli.entry.models:CliConnectionOptions": _CLI_INTERNAL, "sqltide.cli.entry.models:ResolvedCliInvocation": _CLI_INTERNAL, "sqltide.cli.entry.models:ResolvedCliProjectConfig": _CLI_INTERNAL, @@ -90,6 +95,7 @@ "sqltide.compiler.discovery.types:PipelineFacts": _UNUSED_SHAPE, "sqltide.compiler.discovery.types:ProjectPolicyFacts": _UNUSED_SHAPE, "sqltide.compiler.test_discovery.models:SqlTestHeader": _UNUSED_SHAPE, + "sqltide.dev_server.constants:REDACTED_SECRET_PLACEHOLDER": _NATIVE_CONSTANT, "sqltide.executor.auditing.types:AuditResultStatus": _UNUSED_SHAPE, "sqltide.executor.deployment.types:NativeDeploymentSummary": _NATIVE_INTERCHANGE, "sqltide.executor.deployment.types:NativeInventoryRecord": _NATIVE_INTERCHANGE, diff --git a/src/sqltide/_native.pyi b/src/sqltide/_native.pyi index f7c8f897..8c46433a 100644 --- a/src/sqltide/_native.pyi +++ b/src/sqltide/_native.pyi @@ -77,36 +77,6 @@ from sqltide.adapter.types import ( ) from sqltide.adapters.clickhouse.classes.clickhouse_connection import ClickHouseConnection from sqltide.adapters.clickhouse.types import RawClickHouseClient -from sqltide.auth.models import ( - AccountAuditRecord, - AuthSettings, - ProjectRoleAssignment, - ResolvedSession, - SessionCredentials, - UserAccount, -) -from sqltide.auth.types import AuthenticationSource -from sqltide.authorization.models import ( - AuthorizationDecision, - AuthorizationRequest, - CapabilityRequest, - EffectiveCapabilities, -) -from sqltide.cli.build.models import ( - BuildCommandOptions, - BuildProtectionRequirement, - DirectBuildPreviewContext, - DirectWorkflowPreparation, - MixedWorkflowPreparation, - VirtualBuildPreviewContext, - VirtualWorkflowPreparation, - WorkflowPreparationOptions, -) -from sqltide.cli.destruction.models import DestructionCommandOptions -from sqltide.cli.dev.models import DevCommandOptions -from sqltide.cli.entry.models import CliEntrypointHandlers -from sqltide.cli.plan.models import PlanCommandOptions -from sqltide.cli.selection.models import SelectionResolution from sqltide.compiler.discovery.models import ( AllowedCrossPipelineReference, AuditDefaults, @@ -150,13 +120,7 @@ from sqltide.dev_server.models import ( MessagesQueryRequest, ReplayOffsetProgress, ) -from sqltide.dev_server.types import ( - ActivityTone, - CompileAuthorizationGuard, - DevServerReporter, - RunPresentationStatus, -) -from sqltide.diagnostics.models import CompilerDiagnostic, RelatedDiagnosticLocation, SourceLocation +from sqltide.dev_server.types import CompileAuthorizationGuard, DevServerReporter from sqltide.events.models import ( AuditCompleted, InvocationObservation, @@ -170,13 +134,13 @@ from sqltide.executor.destruction.types import ( ) from sqltide.executor.kafka_admin.models import ConsumerGroupOffsetReset from sqltide.executor.kafka_admin.types import KafkaAdminClientFactory -from sqltide.executor.workflow.models import ( - BuildWorkflow, - PublishedBuildWorkflow, - WarehouseStatement, - WorkflowExecutionResult, +from sqltide.executor.workflow.types import ( + NativeStatement, + StatementIntent, + WorkflowEventEmitter, + WorkflowMode, + WorkflowPhase, ) -from sqltide.executor.workflow.types import NativeStatement, WorkflowEventEmitter, WorkflowMode from sqltide.provider.models import DiscoveredProvider from sqltide.sensors.models import ( CompiledSensors, @@ -2519,6 +2483,555 @@ RUN_INVOCATION_ID_ENV_VAR: str PROJECT_REVISION_ENV_VAR: str RECONCILE_DEPLOYMENT_ID_PREFIX: str +# CLI, account, authorization, workflow, sensor, dev server and diagnostic shapes. + +class AdminCommand(StrEnum): + MIGRATE = "migrate" + CREATE_USER = "create-user" + GRANT_ROLE = "grant-role" + REVOKE_ROLE = "revoke-role" + ENABLE_USER = "enable-user" + DISABLE_USER = "disable-user" + RESET_PASSWORD = "reset-password" + +class CliCommand(StrEnum): + ADMIN = "admin" + PLAYGROUND = "playground" + DISCOVER = "discover" + DEV = "dev" + COMPILE = "compile" + LINT = "lint" + FORMAT = "format" + TEST = "test" + PLAN = "plan" + BUILD = "build" + DESTROY = "destroy" + RESET_TARGET = "reset-target" + DEPLOYMENT = "deployment" + AUDIT = "audit" + RECONCILE = "reconcile" + JANITOR = "janitor" + DOCTOR = "doctor" + REPAIR = "repair" + +class CliSubcommand(StrEnum): + ACTIVE_VIEW = "active-view" + DIFF = "diff" + LIST = "list" + SHOW = "show" + AUDIT = "audit" + PROMOTE = "promote" + ROLLBACK = "rollback" + +class AuthenticationMode(StrEnum): + DISABLED = "disabled" + PASSWORD = "password" + TRUSTED_PROXY = "trusted_proxy" + +class AuthenticationSource(StrEnum): + LOCAL = "local" + PASSWORD = "password" + TRUSTED_PROXY = "trusted_proxy" + +class UnknownUserPolicy(StrEnum): + AUTO_PROVISION = "auto_provision" + DENY = "deny" + +class AuthorizationReason(StrEnum): + SYSTEM_ADMIN = "system_admin" + GRANTED = "granted" + POLICY_ABSENT = "policy_absent" + NO_MATCHING_ASSIGNMENT = "no_matching_assignment" + STALE_ASSIGNMENT = "stale_assignment" + EMPTY_PIPELINE_SCOPE = "empty_pipeline_scope" + MISSING_PERMISSION = "missing_permission" + MISSING_PIPELINES = "missing_pipelines" + +class ActivityTone(StrEnum): + NEUTRAL = "neutral" + GOOD = "good" + BAD = "bad" + CAUTION = "caution" + +class RunPresentationStatus(StrEnum): + SUCCEEDED = "succeeded" + FAILED = "failed" + CANCELLED = "cancelled" + CANCELLING = "cancelling" + CANCELLATION_FAILED = "cancellation_failed" + RUNNING = "running" + UNRESPONSIVE = "unresponsive" + PRESUMED_FAILED = "presumed_failed" + +class CompileStateKind(StrEnum): + OK = "ok" + FAILING = "failing" + +class ReplayAnchorReason(StrEnum): + ELIGIBLE = "eligible" + AGGREGATE = "aggregate" + MUTABLE_REF = "mutable_ref" + NEVER = "never" + LINEAGE_LOSS = "lineage_loss" + VIEW = "view" + +class Freshness(StrEnum): + FRESH = "fresh" + LAGGING = "lagging" + STALLED = "stalled" + +class SensorSchedulerState(StrEnum): + DISABLED = "disabled" + IDLE = "idle" + RUNNING = "running" + STANDBY = "standby" + BACKING_OFF = "backing_off" + +class AuditScheduleState(StrEnum): + DISABLED = "disabled" + IDLE = "idle" + DUE = "due" + SCHEDULED = "scheduled" + WARMING_UP = "warming_up" + NOT_MATERIALIZED = "not_materialized" + RUNNING = "running" + BLOCKED = "blocked" + BACKING_OFF = "backing_off" + +@final +@dataclass(frozen=True) +class SourceLocation: + path: Path + line: int + column: int + end_line: int | None = None + end_column: int | None = None + +@final +@dataclass(frozen=True) +class RelatedDiagnosticLocation: + label: str + location: SourceLocation + message: str | None = None + +@final +@dataclass(frozen=True) +class CompilerDiagnostic: + phase: DiagnosticPhase | str + severity: DiagnosticSeverity | str + code: str + message: str + resource_name: str | None = None + location: SourceLocation | None = None + related_locations: tuple[RelatedDiagnosticLocation, ...] = () + help: str | None = None + +@final +@dataclass(frozen=True) +class Principal: + user_id: UUID + username: str + display_name: str | None + email: str | None + authentication_source: AuthenticationSource + +@final +@dataclass(frozen=True, repr=False) +class AuthSettings: + mode: AuthenticationMode + control_store_url: str + username_header: str = "X-SQLTide-User" + display_name_header: str | None = None + email_header: str | None = None + unknown_user_policy: UnknownUserPolicy = ... + default_role: str = "viewer" + session_cookie_name: str = "sqltide_session" + session_ttl_seconds: int = ... + session_cookie_secure: bool = True + proxy_logout_url: str | None = None + def __repr__(self) -> str: ... + +@final +@dataclass(frozen=True) +class UserAccount: + user_id: UUID + username: str + display_name: str | None + email: str | None + is_active: bool + created_at: datetime + updated_at: datetime + roles: tuple[str, ...] = () + authentication_sources: tuple[AuthenticationSource, ...] = () + +@final +@dataclass(frozen=True) +class SessionCredentials: + token: str + csrf_token: str + expires_at: datetime + +@final +@dataclass(frozen=True) +class ResolvedSession: + principal: Principal + roles: tuple[str, ...] + csrf_token: str + expires_at: datetime + +@final +@dataclass(frozen=True) +class AuthenticatedRequest: + principal: Principal + roles: tuple[str, ...] + csrf_token: str | None = None + +@final +@dataclass(frozen=True) +class AccountAuditRecord: + operation: str + actor_user_id: UUID | None + affected_user_id: UUID | None + occurred_at: datetime + details: dict[str, object] + +@final +@dataclass(frozen=True) +class ProjectRoleAssignment: + assignment_id: UUID + user_id: UUID + project_name: str + role_name: str + target_name: str | None + assigned_by: UUID | None + assigned_at: datetime + revoked_by: UUID | None = None + revoked_at: datetime | None = None + +@final +@dataclass(frozen=True) +class AuthorizationRequest: + authenticated: AuthenticatedRequest + permission: Permission + project_name: str + target_name: str | None + grant_scope: GrantScope | None + affected_pipelines: tuple[str, ...] + policy: CompiledAccessPolicy | None + +@final +@dataclass(frozen=True) +class AuthorizationDecision: + allowed: bool + reason: AuthorizationReason + permission: Permission + project_name: str + target_name: str | None + affected_pipelines: tuple[str, ...] + matched_roles: tuple[str, ...] = () + missing_pipelines: tuple[str, ...] = () + +@final +@dataclass(frozen=True) +class CapabilityRequest: + authenticated: AuthenticatedRequest + project_name: str + target_name: str | None + pipeline_names: tuple[str, ...] + policy: CompiledAccessPolicy | None + +@final +@dataclass(frozen=True) +class EffectiveCapabilities: + system_admin: bool + project_name: str + target_name: str | None + permissions: tuple[Permission, ...] + pipeline_permissions: tuple[tuple[Permission, tuple[str, ...]], ...] + stale_roles: tuple[str, ...] = () + +@final +@dataclass(frozen=True) +class WarehouseStatement: + sequence: int + step_id: str + phase: WorkflowPhase + intent: StatementIntent + sql: str + continue_on_error: bool = False + display_name: str | None = None + replay_offset_progress: AdapterReplayOffsetProgressRequest | None = None + +@dataclass(frozen=True) +class BuildWorkflow: + mode: WorkflowMode + plan_json: str + statements: tuple[WarehouseStatement, ...] + recovery_statements: tuple[WarehouseStatement, ...] = () + +@final +@dataclass(frozen=True) +class PublishedBuildWorkflow: + workflow: BuildWorkflow + artifact_root: Path + workflow_sha256: str + +@final +@dataclass(frozen=True) +class WorkflowStatementResult: + step_id: str + query_result: AdapterQueryResult | None + mutation_result: AdapterMutationResult | None + error_message: str | None = None + +@final +@dataclass(frozen=True) +class WorkflowExecutionResult: + statement_results: tuple[WorkflowStatementResult, ...] + +@final +@dataclass(frozen=True) +class BuildCommandOptions: + pipelines_root: Path + database: str | None + metadata_database: str | None + selectors: tuple[str, ...] + json_output: bool + verbose: bool + auto_approve: bool + deployment_id: str | None = None + full_refresh: bool = False + start_time: str | None = None + events_output: bool = False + confirmations: tuple[str, ...] = () + changed: bool = False + include_missing_upstream: bool = False + +@final +@dataclass(frozen=True) +class BuildProtectionRequirement: + pipeline_name: str + warning: str + confirmation: str + +@final +@dataclass(frozen=True) +class WorkflowPreparationOptions: + database: str | None + metadata_database: str | None + selectors: tuple[str, ...] + deployment_id: str | None + full_refresh: bool + start_time: str | None + verbose: bool + changed: bool = False + include_missing_upstream: bool = False + +@dataclass(frozen=True) +class DirectBuildPreviewContext: + analysis: CompileAnalysis + plan: DirectPlan + warehouse_snapshot: DirectWarehouseSnapshot + database: str + metadata_database: str + adapter_name: str + effective_start_time: str | None = None + row_error_policy_changes: tuple[DirectRowErrorPolicyChange, ...] = () + +@dataclass(frozen=True) +class VirtualBuildPreviewContext: + database: str + metadata_database: str + desired_state: DesiredState + plan: DeploymentPlan + replay_lineage_mode: ReplayLineageMode + deployment_id: str + created_at: str + root_reports: tuple[RootBackfillReport, ...] + existing_relation_names: frozenset[str] + target_catalog: CatalogSnapshot + metadata_catalog: CatalogSnapshot + full_refresh_keys: frozenset[ObjectKey] = frozenset() + start_time_keys: frozenset[ObjectKey] = frozenset() + start_time: str | None = None + execution_logical_model_keys: frozenset[LogicalResourceKey] = frozenset() + run_execution_scope: tuple[LogicalResourceKey, ...] = () + run_context_scope: tuple[LogicalResourceKey, ...] = () + row_error_policy_changes: tuple[DirectRowErrorPolicyChange, ...] = () + +@final +@dataclass(frozen=True) +class DirectWorkflowPreparation: + preview: DirectBuildPreviewContext + request: DirectBuildRequest + workflow: DirectBuildWorkflow + plan_text: str + protection_requirements: tuple[BuildProtectionRequirement, ...] = () + +@final +@dataclass(frozen=True) +class VirtualWorkflowPreparation: + preview: VirtualBuildPreviewContext + request: BackfillBootstrapRequest + workflow: BuildWorkflow + plan_text: str + protection_requirements: tuple[BuildProtectionRequirement, ...] = () + +@final +@dataclass(frozen=True) +class MixedWorkflowPreparation: + virtual: VirtualWorkflowPreparation + direct: DirectWorkflowPreparation + plan_text: str + plan_json: str + protection_requirements: tuple[BuildProtectionRequirement, ...] = () + +@final +@dataclass(frozen=True, repr=False) +class DestructionCommandOptions: + operation: DestructionOperation + pipelines_root: Path + project_dir: Path + selected_target: str + database: str | None + selectors: tuple[str, ...] + control_store_url: str + include_orphans: bool = False + cli_variables: tuple[tuple[str, object], ...] = () + environment: Mapping[str, str] | None = None + +@final +@dataclass(frozen=True, repr=False) +class DevCommandOptions: + pipelines_root: Path + database: str | None + host: str + port: int + selected_target: str | None = None + cli_variables: tuple[tuple[str, object], ...] = () + environment: Mapping[str, str] | None = None + connection_host: str | None = None + connection_port: int | None = None + connection_username: str | None = None + connection_password: str | None = None + auth_settings: AuthSettings | None = None + +@final +@dataclass(frozen=True) +class CliEntrypointHandlers: + run_discover: Callable[..., int] + run_compile: Callable[..., int] + run_test: Callable[..., int] + run_audit: Callable[..., int] + run_plan: Callable[..., int] + run_build: Callable[..., int] + run_destruction: Callable[..., int] + run_deployment_diff: Callable[..., int] + run_deployment_list: Callable[..., int] + run_deployment_show: Callable[..., int] + run_deployment_audit: Callable[..., int] + run_deployment_promote: Callable[..., int] + run_deployment_rollback: Callable[..., int] + run_reconcile: Callable[..., int] + run_janitor: Callable[..., int] + run_doctor: Callable[..., int] + run_repair_active_view: Callable[..., int] + run_dev: Callable[..., int] + +@final +@dataclass(frozen=True) +class PlanCommandOptions: + pipelines_root: Path + database: str | None + selectors: tuple[str, ...] + deployment_id: str | None + full_refresh: bool + start_time: str | None + json_output: bool + verbose: bool + changed: bool = False + include_missing_upstream: bool = False + +@final +@dataclass(frozen=True) +class SelectionResolution: + desired_state: DesiredState + selected_logical_model_keys: frozenset[LogicalResourceKey] + selected_model_keys: frozenset[ObjectKey] + replay_lineage_mode: ReplayLineageMode | None + execution_logical_model_keys: frozenset[LogicalResourceKey] = frozenset() + +class AuthenticationError(Exception): ... +class AuthorizationError(Exception): ... +class ControlStoreError(Exception): ... +class AuthConfigurationError(ValueError): ... +class AccountValidationError(ValueError): ... +class PasswordValidationError(ValueError): ... +class AccountConflictError(ControlStoreError): ... +class AccountNotFoundError(ControlStoreError): ... +class CompileArtifactError(ValueError): ... + +class CliUserError(Exception): + hint: str | None + def __init__(self, message: str, *, hint: str | None = None) -> None: ... + +class BuildPipelineLimitError(CliUserError): ... + +class SensorError(ValueError): + location: SourceLocation | None + diagnostic: CompilerDiagnostic + def __init__( + self, + message: str, + *, + location: SourceLocation | None = None, + related_locations: tuple[RelatedDiagnosticLocation, ...] = (), + ) -> None: ... + +class SensorStepError(RuntimeError): ... +class SensorEvaluationTimeoutError(RuntimeError): ... +class DevServerError(RuntimeError): ... +class DevConfigurationError(DevServerError): ... +class KafkaCollectorClosedError(DevServerError): ... +class ProjectNotCompiledError(DevServerError): ... +class BuildInProgressError(DevServerError): ... +class BuildStartError(DevServerError): ... +class AuditSchedulerPersistenceError(DevServerError): ... +class MessageQueryValidationError(DevServerError): ... +class MessageSchemaError(DevServerError): ... + +CONTROL_STORE_ENV_VAR: str +SQLTIDE_TOOL_VERSION: str +EXPECTED_BUILD_WRITE_SCOPE_ENV_VAR: str +EXPECTED_BUILD_READ_SCOPE_ENV_VAR: str +EXPECTED_BUILD_PIPELINE_SCOPE_ENV_VAR: str +DEV_CLI_VARIABLES_ENV_VAR: str +COMMANDS_REQUIRING_PIPELINES_ROOT: frozenset[CliCommand] +COMMANDS_WITHOUT_ADAPTER_CONNECTION: frozenset[CliCommand] +VIRTUAL_ENVIRONMENT_ONLY_COMMANDS: frozenset[CliCommand] +DISPLAY_NAME_BY_COMMAND: Mapping[CliCommand, str] +METADATA_TABLE_NAME_PREFIX: str +AFFIRMATIVE_RESPONSES: frozenset[str] +TRUE_ENV_VALUES: frozenset[str] +ADMIN_ROLE: str +VIEWER_ROLE: str +ACCOUNT_SCHEMA_COMPONENT: str +PASSWORD_MIN_LENGTH: int +SQLITE_MEMORY_PATH: str +LOCAL_USER_ID: UUID +WORKFLOW_PHASE_ORDER: tuple[WorkflowPhase, ...] +RECOVER_MODEL_VIEW_STEP_PREFIX: str +RECOVER_SOURCE_STEP_PREFIX: str +RECOVER_DEAD_LETTERS_STEP_PREFIX: str +DEFAULT_SENSOR_TIMEOUT_SECONDS: float +DEFAULT_DEV_SERVER_HOST: str +DEFAULT_DEV_SERVER_PORT: int +KAFKA_REUSE_LOG_FIRST_READ_COUNT: int +KAFKA_REUSE_LOG_INTERVAL: int +CANCEL_GRACE_SECONDS: float +TERMINATE_GRACE_SECONDS: float +ACTIVITY_WINDOW_SECONDS: int + type _Span = tuple[int, int, int, int, int, int] class SqlScanError(ValueError): diff --git a/src/sqltide/auth/constants.py b/src/sqltide/auth/constants.py index c14c350a..c92f243f 100644 --- a/src/sqltide/auth/constants.py +++ b/src/sqltide/auth/constants.py @@ -1,12 +1,8 @@ """Stable authentication and account-control constants.""" -from uuid import UUID - -ADMIN_ROLE: str = "admin" -VIEWER_ROLE: str = "viewer" -ACCOUNT_SCHEMA_COMPONENT: str = "accounts" -PASSWORD_MIN_LENGTH: int = 12 -SQLITE_MEMORY_PATH: str = ":memory:" -CONTROL_STORE_TABLE_PREFIX: str = "sqltide_" -STREAMBUILD_CONTROL_STORE_TABLE_PREFIX: str = "streambuild_" -LOCAL_USER_ID: UUID = UUID("a25809d6-cf14-592c-b7b6-c94ef048e0cc") +from sqltide._native import ACCOUNT_SCHEMA_COMPONENT as ACCOUNT_SCHEMA_COMPONENT +from sqltide._native import ADMIN_ROLE as ADMIN_ROLE +from sqltide._native import LOCAL_USER_ID as LOCAL_USER_ID +from sqltide._native import PASSWORD_MIN_LENGTH as PASSWORD_MIN_LENGTH +from sqltide._native import SQLITE_MEMORY_PATH as SQLITE_MEMORY_PATH +from sqltide._native import VIEWER_ROLE as VIEWER_ROLE diff --git a/src/sqltide/auth/exceptions.py b/src/sqltide/auth/exceptions.py index 6863d1ef..6ef46c43 100644 --- a/src/sqltide/auth/exceptions.py +++ b/src/sqltide/auth/exceptions.py @@ -1,33 +1,10 @@ """Authentication and account-domain failures.""" - -class AuthenticationError(Exception): - """A request could not produce an authenticated principal.""" - - -class AuthorizationError(Exception): - """An authenticated principal cannot perform an account operation.""" - - -class ControlStoreError(Exception): - """The account control store is unavailable or incompatible.""" - - -class AuthConfigurationError(ValueError): - """Authentication runtime configuration is invalid.""" - - -class AccountValidationError(ValueError): - """Account input violates a stable domain constraint.""" - - -class PasswordValidationError(ValueError): - """A password does not meet the configured security policy.""" - - -class AccountConflictError(ControlStoreError): - """An account mutation conflicts with existing identity state.""" - - -class AccountNotFoundError(ControlStoreError): - """An account or role assignment does not exist.""" +from sqltide._native import AccountConflictError as AccountConflictError +from sqltide._native import AccountNotFoundError as AccountNotFoundError +from sqltide._native import AccountValidationError as AccountValidationError +from sqltide._native import AuthConfigurationError as AuthConfigurationError +from sqltide._native import AuthenticationError as AuthenticationError +from sqltide._native import AuthorizationError as AuthorizationError +from sqltide._native import ControlStoreError as ControlStoreError +from sqltide._native import PasswordValidationError as PasswordValidationError diff --git a/src/sqltide/auth/models.py b/src/sqltide/auth/models.py index 915e517f..1c736698 100644 --- a/src/sqltide/auth/models.py +++ b/src/sqltide/auth/models.py @@ -2,135 +2,18 @@ from __future__ import annotations -from dataclasses import dataclass -from datetime import datetime from typing import Literal -from uuid import UUID from pydantic import BaseModel, Field -from sqltide.auth.constants import VIEWER_ROLE -from sqltide.auth.exceptions import AuthConfigurationError -from sqltide.auth.types import AuthenticationMode, AuthenticationSource, UnknownUserPolicy - - -@dataclass(frozen=True) -class Principal: - """Authenticated request identity, independent of authorization policy.""" - - user_id: UUID - username: str - display_name: str | None - email: str | None - authentication_source: AuthenticationSource - - -@dataclass(frozen=True, repr=False) -class AuthSettings: - """Server-runtime authentication and control-store settings.""" - - mode: AuthenticationMode - control_store_url: str - username_header: str = "X-SQLTide-User" - display_name_header: str | None = None - email_header: str | None = None - unknown_user_policy: UnknownUserPolicy = UnknownUserPolicy.AUTO_PROVISION - default_role: str = "viewer" - session_cookie_name: str = "sqltide_session" - session_ttl_seconds: int = 12 * 60 * 60 - session_cookie_secure: bool = True - proxy_logout_url: str | None = None - - def __post_init__(self) -> None: - if self.session_ttl_seconds <= 0: - raise AuthConfigurationError("Session TTL must be positive") - if ( - self.mode == AuthenticationMode.TRUSTED_PROXY - and self.unknown_user_policy == UnknownUserPolicy.AUTO_PROVISION - and self.default_role != VIEWER_ROLE - ): - raise AuthConfigurationError( - "Trusted-proxy auto-provisioning default role must be 'viewer'" - ) - - def __repr__(self) -> str: - driver: str = self.control_store_url.partition(":")[0] - return ( - "AuthSettings(" - f"mode={self.mode!r}, control_store_url='', " - f"username_header={self.username_header!r}, " - f"unknown_user_policy={self.unknown_user_policy!r}, " - f"default_role={self.default_role!r})" - ) - - -@dataclass(frozen=True) -class UserAccount: - """Persisted SQLTide account without credential material.""" - - user_id: UUID - username: str - display_name: str | None - email: str | None - is_active: bool - created_at: datetime - updated_at: datetime - roles: tuple[str, ...] = () - authentication_sources: tuple[AuthenticationSource, ...] = () - - -@dataclass(frozen=True) -class SessionCredentials: - """One newly issued browser session; only the hash is persisted.""" - - token: str - csrf_token: str - expires_at: datetime - - -@dataclass(frozen=True) -class ResolvedSession: - """Active browser session and its authenticated principal.""" - - principal: Principal - roles: tuple[str, ...] - csrf_token: str - expires_at: datetime - - -@dataclass(frozen=True) -class AuthenticatedRequest: - """Principal plus request-local account/session state.""" - - principal: Principal - roles: tuple[str, ...] - csrf_token: str | None = None - - -@dataclass(frozen=True) -class AccountAuditRecord: - """Non-secret account administration audit event.""" - - operation: str - actor_user_id: UUID | None - affected_user_id: UUID | None - occurred_at: datetime - details: dict[str, object] - - -@dataclass(frozen=True) -class ProjectRoleAssignment: - """Audited user membership in one project-authored operational role.""" - - assignment_id: UUID - user_id: UUID - project_name: str - role_name: str - target_name: str | None - assigned_by: UUID | None - assigned_at: datetime - revoked_by: UUID | None = None - revoked_at: datetime | None = None +from sqltide._native import AccountAuditRecord as AccountAuditRecord +from sqltide._native import AuthenticatedRequest as AuthenticatedRequest +from sqltide._native import AuthSettings as AuthSettings +from sqltide._native import Principal as Principal +from sqltide._native import ProjectRoleAssignment as ProjectRoleAssignment +from sqltide._native import ResolvedSession as ResolvedSession +from sqltide._native import SessionCredentials as SessionCredentials +from sqltide._native import UserAccount as UserAccount class LoginRequest(BaseModel): diff --git a/src/sqltide/auth/types.py b/src/sqltide/auth/types.py index c22cecc5..dd0b4d30 100644 --- a/src/sqltide/auth/types.py +++ b/src/sqltide/auth/types.py @@ -1,26 +1,5 @@ """Authentication enums and public type declarations.""" -from enum import StrEnum - - -class AuthenticationMode(StrEnum): - """Configured server authentication mechanism.""" - - DISABLED = "disabled" - PASSWORD = "password" - TRUSTED_PROXY = "trusted_proxy" - - -class AuthenticationSource(StrEnum): - """Mechanism that authenticated one request.""" - - LOCAL = "local" - PASSWORD = "password" - TRUSTED_PROXY = "trusted_proxy" - - -class UnknownUserPolicy(StrEnum): - """Trusted-proxy behavior for an identity absent from the control store.""" - - AUTO_PROVISION = "auto_provision" - DENY = "deny" +from sqltide._native import AuthenticationMode as AuthenticationMode +from sqltide._native import AuthenticationSource as AuthenticationSource +from sqltide._native import UnknownUserPolicy as UnknownUserPolicy diff --git a/src/sqltide/authorization/models.py b/src/sqltide/authorization/models.py index c9a8ea40..e85fb92a 100644 --- a/src/sqltide/authorization/models.py +++ b/src/sqltide/authorization/models.py @@ -1,58 +1,6 @@ """Immutable operational authorization inputs and decisions.""" -from dataclasses import dataclass - -from sqltide.auth.models import AuthenticatedRequest -from sqltide.authorization.types import AuthorizationReason -from sqltide.compiler.access.models import CompiledAccessPolicy -from sqltide.compiler.access.types import GrantScope, Permission - - -@dataclass(frozen=True) -class AuthorizationRequest: - """Authoritative identity, policy, and resolved operation scope.""" - - authenticated: AuthenticatedRequest - permission: Permission - project_name: str - target_name: str | None - grant_scope: GrantScope | None - affected_pipelines: tuple[str, ...] - policy: CompiledAccessPolicy | None - - -@dataclass(frozen=True) -class AuthorizationDecision: - """Explainable allow or deny result from centralized evaluation.""" - - allowed: bool - reason: AuthorizationReason - permission: Permission - project_name: str - target_name: str | None - affected_pipelines: tuple[str, ...] - matched_roles: tuple[str, ...] = () - missing_pipelines: tuple[str, ...] = () - - -@dataclass(frozen=True) -class CapabilityRequest: - """Identity and policy inputs for one effective-capability summary.""" - - authenticated: AuthenticatedRequest - project_name: str - target_name: str | None - pipeline_names: tuple[str, ...] - policy: CompiledAccessPolicy | None - - -@dataclass(frozen=True) -class EffectiveCapabilities: - """UI-facing effective operation permissions for one user.""" - - system_admin: bool - project_name: str - target_name: str | None - permissions: tuple[Permission, ...] - pipeline_permissions: tuple[tuple[Permission, tuple[str, ...]], ...] - stale_roles: tuple[str, ...] = () +from sqltide._native import AuthorizationDecision as AuthorizationDecision +from sqltide._native import AuthorizationRequest as AuthorizationRequest +from sqltide._native import CapabilityRequest as CapabilityRequest +from sqltide._native import EffectiveCapabilities as EffectiveCapabilities diff --git a/src/sqltide/authorization/types.py b/src/sqltide/authorization/types.py index 911e3723..50adc8e5 100644 --- a/src/sqltide/authorization/types.py +++ b/src/sqltide/authorization/types.py @@ -1,16 +1,3 @@ """Closed authorization decision reasons.""" -from enum import StrEnum - - -class AuthorizationReason(StrEnum): - """Stable explanation category for one operational decision.""" - - SYSTEM_ADMIN = "system_admin" - GRANTED = "granted" - POLICY_ABSENT = "policy_absent" - NO_MATCHING_ASSIGNMENT = "no_matching_assignment" - STALE_ASSIGNMENT = "stale_assignment" - EMPTY_PIPELINE_SCOPE = "empty_pipeline_scope" - MISSING_PERMISSION = "missing_permission" - MISSING_PIPELINES = "missing_pipelines" +from sqltide._native import AuthorizationReason as AuthorizationReason diff --git a/src/sqltide/cli/admin/constants.py b/src/sqltide/cli/admin/constants.py index 5e3e3985..c3e442d8 100644 --- a/src/sqltide/cli/admin/constants.py +++ b/src/sqltide/cli/admin/constants.py @@ -1,3 +1,3 @@ """Stable account-administration CLI constants.""" -CONTROL_STORE_ENV_VAR: str = "SQLTIDE_CONTROL_STORE_URL" +from sqltide._native import CONTROL_STORE_ENV_VAR as CONTROL_STORE_ENV_VAR diff --git a/src/sqltide/cli/admin/types.py b/src/sqltide/cli/admin/types.py index 85b52e23..54c26f1b 100644 --- a/src/sqltide/cli/admin/types.py +++ b/src/sqltide/cli/admin/types.py @@ -1,15 +1,3 @@ """Account-administration CLI command types.""" -from enum import StrEnum - - -class AdminCommand(StrEnum): - """Supported operator account-administration commands.""" - - MIGRATE = "migrate" - CREATE_USER = "create-user" - GRANT_ROLE = "grant-role" - REVOKE_ROLE = "revoke-role" - ENABLE_USER = "enable-user" - DISABLE_USER = "disable-user" - RESET_PASSWORD = "reset-password" +from sqltide._native import AdminCommand as AdminCommand diff --git a/src/sqltide/cli/build/constants.py b/src/sqltide/cli/build/constants.py index 7068237d..a7c1c146 100644 --- a/src/sqltide/cli/build/constants.py +++ b/src/sqltide/cli/build/constants.py @@ -1,9 +1,8 @@ """Direct build command constants.""" -from importlib.metadata import version - -SQLTIDE_TOOL_VERSION: str = version("sqltide") -BUILD_CANCELLED_MESSAGE: str = "Build cancelled." -EXPECTED_BUILD_WRITE_SCOPE_ENV_VAR: str = "SQLTIDE_EXPECTED_BUILD_WRITE_SCOPE" -EXPECTED_BUILD_READ_SCOPE_ENV_VAR: str = "SQLTIDE_EXPECTED_BUILD_READ_SCOPE" -EXPECTED_BUILD_PIPELINE_SCOPE_ENV_VAR: str = "SQLTIDE_EXPECTED_BUILD_PIPELINE_SCOPE" +from sqltide._native import ( + EXPECTED_BUILD_PIPELINE_SCOPE_ENV_VAR as EXPECTED_BUILD_PIPELINE_SCOPE_ENV_VAR, +) +from sqltide._native import EXPECTED_BUILD_READ_SCOPE_ENV_VAR as EXPECTED_BUILD_READ_SCOPE_ENV_VAR +from sqltide._native import EXPECTED_BUILD_WRITE_SCOPE_ENV_VAR as EXPECTED_BUILD_WRITE_SCOPE_ENV_VAR +from sqltide._native import SQLTIDE_TOOL_VERSION as SQLTIDE_TOOL_VERSION diff --git a/src/sqltide/cli/build/exceptions.py b/src/sqltide/cli/build/exceptions.py index 0fabd344..a8045d40 100644 --- a/src/sqltide/cli/build/exceptions.py +++ b/src/sqltide/cli/build/exceptions.py @@ -1,7 +1,3 @@ """Build command policy exceptions.""" -from sqltide.cli.entry.exceptions import CliUserError - - -class BuildPipelineLimitError(CliUserError): - """The final build closure exceeds its committed target limit.""" +from sqltide._native import BuildPipelineLimitError as BuildPipelineLimitError diff --git a/src/sqltide/cli/build/models.py b/src/sqltide/cli/build/models.py index 68bf2abb..76430876 100644 --- a/src/sqltide/cli/build/models.py +++ b/src/sqltide/cli/build/models.py @@ -1,135 +1,10 @@ """Options and prepared contexts for the mode-aware build command.""" -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path - -from sqltide.adapter.models import CatalogSnapshot -from sqltide.compiler.compile.models import DesiredState, LogicalResourceKey, ObjectKey -from sqltide.compiler.discovery.types import ReplayLineageMode -from sqltide.compiler.pipeline.models import CompileAnalysis -from sqltide.compiler.planner.models import ( - DeploymentPlan, - DirectPlan, - DirectRowErrorPolicyChange, - DirectWarehouseSnapshot, -) -from sqltide.executor.backfill.models import BackfillBootstrapRequest, RootBackfillReport -from sqltide.executor.direct.models import DirectBuildRequest, DirectBuildWorkflow -from sqltide.executor.workflow.models import BuildWorkflow - - -@dataclass(frozen=True) -class BuildCommandOptions: - """Every operator-supplied option for one `sqt build` invocation.""" - - pipelines_root: Path - database: str | None - metadata_database: str | None - selectors: tuple[str, ...] - json_output: bool - verbose: bool - auto_approve: bool - deployment_id: str | None = None - full_refresh: bool = False - start_time: str | None = None - events_output: bool = False - confirmations: tuple[str, ...] = () - changed: bool = False - include_missing_upstream: bool = False - - -@dataclass(frozen=True) -class BuildProtectionRequirement: - """One protected pipeline touched by the resolved build closure.""" - - pipeline_name: str - warning: str - confirmation: str - - -@dataclass(frozen=True) -class WorkflowPreparationOptions: - """Connected planning options shared by `sqt plan` and `sqt build`.""" - - database: str | None - metadata_database: str | None - selectors: tuple[str, ...] - deployment_id: str | None - full_refresh: bool - start_time: str | None - verbose: bool - changed: bool = False - include_missing_upstream: bool = False - - -@dataclass(frozen=True) -class DirectBuildPreviewContext: - """The plan and resolved databases a build renders before it writes anything.""" - - analysis: CompileAnalysis - plan: DirectPlan - warehouse_snapshot: DirectWarehouseSnapshot - database: str - metadata_database: str - adapter_name: str - effective_start_time: str | None = None - row_error_policy_changes: tuple[DirectRowErrorPolicyChange, ...] = () - - -@dataclass(frozen=True) -class VirtualBuildPreviewContext: - """The confirmed virtual deployment plan and its fixed execution identity.""" - - database: str - metadata_database: str - desired_state: DesiredState - plan: DeploymentPlan - replay_lineage_mode: ReplayLineageMode - deployment_id: str - created_at: str - root_reports: tuple[RootBackfillReport, ...] - existing_relation_names: frozenset[str] - target_catalog: CatalogSnapshot - metadata_catalog: CatalogSnapshot - full_refresh_keys: frozenset[ObjectKey] = frozenset() - start_time_keys: frozenset[ObjectKey] = frozenset() - start_time: str | None = None - execution_logical_model_keys: frozenset[LogicalResourceKey] = frozenset() - run_execution_scope: tuple[LogicalResourceKey, ...] = () - run_context_scope: tuple[LogicalResourceKey, ...] = () - row_error_policy_changes: tuple[DirectRowErrorPolicyChange, ...] = () - - -@dataclass(frozen=True) -class DirectWorkflowPreparation: - """One fully assembled direct workflow and its result-decoding request.""" - - preview: DirectBuildPreviewContext - request: DirectBuildRequest - workflow: DirectBuildWorkflow - plan_text: str - protection_requirements: tuple[BuildProtectionRequirement, ...] = () - - -@dataclass(frozen=True) -class VirtualWorkflowPreparation: - """One fully assembled virtual workflow and its result-decoding request.""" - - preview: VirtualBuildPreviewContext - request: BackfillBootstrapRequest - workflow: BuildWorkflow - plan_text: str - protection_requirements: tuple[BuildProtectionRequirement, ...] = () - - -@dataclass(frozen=True) -class MixedWorkflowPreparation: - """Virtual-first and direct-second workflows prepared for one build.""" - - virtual: VirtualWorkflowPreparation - direct: DirectWorkflowPreparation - plan_text: str - plan_json: str - protection_requirements: tuple[BuildProtectionRequirement, ...] = () +from sqltide._native import BuildCommandOptions as BuildCommandOptions +from sqltide._native import BuildProtectionRequirement as BuildProtectionRequirement +from sqltide._native import DirectBuildPreviewContext as DirectBuildPreviewContext +from sqltide._native import DirectWorkflowPreparation as DirectWorkflowPreparation +from sqltide._native import MixedWorkflowPreparation as MixedWorkflowPreparation +from sqltide._native import VirtualBuildPreviewContext as VirtualBuildPreviewContext +from sqltide._native import VirtualWorkflowPreparation as VirtualWorkflowPreparation +from sqltide._native import WorkflowPreparationOptions as WorkflowPreparationOptions diff --git a/src/sqltide/cli/compile/exceptions.py b/src/sqltide/cli/compile/exceptions.py index 9bd7c2bc..216fee33 100644 --- a/src/sqltide/cli/compile/exceptions.py +++ b/src/sqltide/cli/compile/exceptions.py @@ -1,5 +1,3 @@ """Compile artifact exceptions.""" - -class CompileArtifactError(ValueError): - """Raised when a compile artifact cannot be generated or published safely.""" +from sqltide._native import CompileArtifactError as CompileArtifactError diff --git a/src/sqltide/cli/destruction/models.py b/src/sqltide/cli/destruction/models.py index 62f06c74..dd183347 100644 --- a/src/sqltide/cli/destruction/models.py +++ b/src/sqltide/cli/destruction/models.py @@ -1,21 +1,3 @@ """CLI destruction command inputs.""" -from collections.abc import Mapping -from dataclasses import dataclass -from pathlib import Path - -from sqltide.executor.destruction.types import DestructionOperation - - -@dataclass(frozen=True, repr=False) -class DestructionCommandOptions: - operation: DestructionOperation - pipelines_root: Path - project_dir: Path - selected_target: str - database: str | None - selectors: tuple[str, ...] - control_store_url: str - include_orphans: bool = False - cli_variables: tuple[tuple[str, object], ...] = () - environment: Mapping[str, str] | None = None +from sqltide._native import DestructionCommandOptions as DestructionCommandOptions diff --git a/src/sqltide/cli/dev/models.py b/src/sqltide/cli/dev/models.py index bbbe43da..69ae80ec 100644 --- a/src/sqltide/cli/dev/models.py +++ b/src/sqltide/cli/dev/models.py @@ -1,27 +1,3 @@ """Dev command option models.""" -from __future__ import annotations - -from collections.abc import Mapping -from dataclasses import dataclass -from pathlib import Path - -from sqltide.auth.models import AuthSettings - - -@dataclass(frozen=True, repr=False) -class DevCommandOptions: - """Parsed `sqt dev` invocation options.""" - - pipelines_root: Path - database: str | None - host: str - port: int - selected_target: str | None = None - cli_variables: tuple[tuple[str, object], ...] = () - environment: Mapping[str, str] | None = None - connection_host: str | None = None - connection_port: int | None = None - connection_username: str | None = None - connection_password: str | None = None - auth_settings: AuthSettings | None = None +from sqltide._native import DevCommandOptions as DevCommandOptions diff --git a/src/sqltide/cli/entry/constants.py b/src/sqltide/cli/entry/constants.py index 6e6bb1df..fb2c8100 100644 --- a/src/sqltide/cli/entry/constants.py +++ b/src/sqltide/cli/entry/constants.py @@ -1,50 +1,12 @@ """CLI entry constants.""" -from collections.abc import Mapping - -from sqltide.cli.entry.types import CliCommand, CliSubcommand - -ACTIVE_VIEW_SUBCOMMAND: str = CliSubcommand.ACTIVE_VIEW -DEV_CLI_VARIABLES_ENV_VAR: str = "SQLTIDE_INTERNAL_CLI_VARS" - -COMMANDS_REQUIRING_PIPELINES_ROOT: frozenset[CliCommand] = frozenset( - { - CliCommand.DEV, - CliCommand.DISCOVER, - CliCommand.COMPILE, - CliCommand.LINT, - CliCommand.FORMAT, - CliCommand.TEST, - CliCommand.PLAN, - CliCommand.BUILD, - CliCommand.DESTROY, - CliCommand.RESET_TARGET, - CliCommand.DEPLOYMENT, - CliCommand.RECONCILE, - CliCommand.AUDIT, - } -) -COMMANDS_WITHOUT_ADAPTER_CONNECTION: frozenset[CliCommand] = frozenset( - {CliCommand.DISCOVER, CliCommand.COMPILE, CliCommand.LINT, CliCommand.FORMAT} +from sqltide._native import AFFIRMATIVE_RESPONSES as AFFIRMATIVE_RESPONSES +from sqltide._native import COMMANDS_REQUIRING_PIPELINES_ROOT as COMMANDS_REQUIRING_PIPELINES_ROOT +from sqltide._native import ( + COMMANDS_WITHOUT_ADAPTER_CONNECTION as COMMANDS_WITHOUT_ADAPTER_CONNECTION, ) -DIRECT_ONLY_COMMANDS: frozenset[CliCommand] = frozenset() -VIRTUAL_ENVIRONMENT_ONLY_COMMANDS: frozenset[CliCommand] = frozenset( - { - CliCommand.DEPLOYMENT, - CliCommand.RECONCILE, - CliCommand.JANITOR, - CliCommand.DOCTOR, - CliCommand.REPAIR, - } -) - -DISPLAY_NAME_BY_COMMAND: Mapping[CliCommand, str] = { - CliCommand.AUDIT: "audit", - CliCommand.DESTROY: "destroy", - CliCommand.REPAIR: "repair active-view", - CliCommand.RESET_TARGET: "reset-target", -} - -METADATA_TABLE_NAME_PREFIX: str = "sqltide_" -AFFIRMATIVE_RESPONSES: frozenset[str] = frozenset({"y", "yes"}) -TRUE_ENV_VALUES: frozenset[str] = frozenset({"1", "true", "yes"}) +from sqltide._native import DEV_CLI_VARIABLES_ENV_VAR as DEV_CLI_VARIABLES_ENV_VAR +from sqltide._native import DISPLAY_NAME_BY_COMMAND as DISPLAY_NAME_BY_COMMAND +from sqltide._native import METADATA_TABLE_NAME_PREFIX as METADATA_TABLE_NAME_PREFIX +from sqltide._native import TRUE_ENV_VALUES as TRUE_ENV_VALUES +from sqltide._native import VIRTUAL_ENVIRONMENT_ONLY_COMMANDS as VIRTUAL_ENVIRONMENT_ONLY_COMMANDS diff --git a/src/sqltide/cli/entry/exceptions.py b/src/sqltide/cli/entry/exceptions.py index 5b480e5a..61a56e36 100644 --- a/src/sqltide/cli/entry/exceptions.py +++ b/src/sqltide/cli/entry/exceptions.py @@ -1,9 +1,3 @@ """CLI-specific expected exception types.""" - -class CliUserError(Exception): - """Expected CLI-facing error that should be rendered without a traceback.""" - - def __init__(self, message: str, *, hint: str | None = None) -> None: - super().__init__(message) - self.hint: str | None = hint +from sqltide._native import CliUserError as CliUserError diff --git a/src/sqltide/cli/entry/models.py b/src/sqltide/cli/entry/models.py index fdf5d1a8..808cb465 100644 --- a/src/sqltide/cli/entry/models.py +++ b/src/sqltide/cli/entry/models.py @@ -1,26 +1 @@ -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass - - -@dataclass(frozen=True) -class CliEntrypointHandlers: - run_discover: Callable[..., int] - run_compile: Callable[..., int] - run_test: Callable[..., int] - run_audit: Callable[..., int] - run_plan: Callable[..., int] - run_build: Callable[..., int] - run_destruction: Callable[..., int] - run_deployment_diff: Callable[..., int] - run_deployment_list: Callable[..., int] - run_deployment_show: Callable[..., int] - run_deployment_audit: Callable[..., int] - run_deployment_promote: Callable[..., int] - run_deployment_rollback: Callable[..., int] - run_reconcile: Callable[..., int] - run_janitor: Callable[..., int] - run_doctor: Callable[..., int] - run_repair_active_view: Callable[..., int] - run_dev: Callable[..., int] +from sqltide._native import CliEntrypointHandlers as CliEntrypointHandlers diff --git a/src/sqltide/cli/entry/types.py b/src/sqltide/cli/entry/types.py index 69480632..24aa1a3e 100644 --- a/src/sqltide/cli/entry/types.py +++ b/src/sqltide/cli/entry/types.py @@ -1,38 +1,4 @@ """CLI entry runtime domain types.""" -from enum import StrEnum - - -class CliCommand(StrEnum): - """One top-level `sqt` subcommand as argparse reports it.""" - - ADMIN = "admin" - PLAYGROUND = "playground" - DISCOVER = "discover" - DEV = "dev" - COMPILE = "compile" - LINT = "lint" - FORMAT = "format" - TEST = "test" - PLAN = "plan" - BUILD = "build" - DESTROY = "destroy" - RESET_TARGET = "reset-target" - DEPLOYMENT = "deployment" - AUDIT = "audit" - RECONCILE = "reconcile" - JANITOR = "janitor" - DOCTOR = "doctor" - REPAIR = "repair" - - -class CliSubcommand(StrEnum): - """One nested `sqt ` selector.""" - - ACTIVE_VIEW = "active-view" - DIFF = "diff" - LIST = "list" - SHOW = "show" - AUDIT = "audit" - PROMOTE = "promote" - ROLLBACK = "rollback" +from sqltide._native import CliCommand as CliCommand +from sqltide._native import CliSubcommand as CliSubcommand diff --git a/src/sqltide/cli/plan/models.py b/src/sqltide/cli/plan/models.py index c641e3b2..ecb93253 100644 --- a/src/sqltide/cli/plan/models.py +++ b/src/sqltide/cli/plan/models.py @@ -1,22 +1,3 @@ """CLI plan models.""" -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path - - -@dataclass(frozen=True) -class PlanCommandOptions: - """Every operator-supplied option for one `sqt plan` invocation.""" - - pipelines_root: Path - database: str | None - selectors: tuple[str, ...] - deployment_id: str | None - full_refresh: bool - start_time: str | None - json_output: bool - verbose: bool - changed: bool = False - include_missing_upstream: bool = False +from sqltide._native import PlanCommandOptions as PlanCommandOptions diff --git a/src/sqltide/cli/selection/models.py b/src/sqltide/cli/selection/models.py index 6b6e280c..ef2478eb 100644 --- a/src/sqltide/cli/selection/models.py +++ b/src/sqltide/cli/selection/models.py @@ -1,17 +1,3 @@ """CLI selection models.""" -from __future__ import annotations - -from dataclasses import dataclass - -from sqltide.compiler.compile.models import DesiredState, LogicalResourceKey, ObjectKey -from sqltide.compiler.discovery.types import ReplayLineageMode - - -@dataclass(frozen=True) -class SelectionResolution: - desired_state: DesiredState - selected_logical_model_keys: frozenset[LogicalResourceKey] - selected_model_keys: frozenset[ObjectKey] - replay_lineage_mode: ReplayLineageMode | None - execution_logical_model_keys: frozenset[LogicalResourceKey] = frozenset() +from sqltide._native import SelectionResolution as SelectionResolution diff --git a/src/sqltide/dev_server/constants.py b/src/sqltide/dev_server/constants.py index 78bf1ebd..e5fd950b 100644 --- a/src/sqltide/dev_server/constants.py +++ b/src/sqltide/dev_server/constants.py @@ -1,10 +1,9 @@ """Dev server constants.""" -DEFAULT_DEV_SERVER_HOST: str = "127.0.0.1" -DEFAULT_DEV_SERVER_PORT: int = 8000 -REDACTED_SECRET_PLACEHOLDER: str = "" -KAFKA_REUSE_LOG_FIRST_READ_COUNT: int = 2 -KAFKA_REUSE_LOG_INTERVAL: int = 100 -CANCEL_GRACE_SECONDS: float = 15.0 -TERMINATE_GRACE_SECONDS: float = 3.0 -ACTIVITY_WINDOW_SECONDS: int = 120 +from sqltide._native import ACTIVITY_WINDOW_SECONDS as ACTIVITY_WINDOW_SECONDS +from sqltide._native import CANCEL_GRACE_SECONDS as CANCEL_GRACE_SECONDS +from sqltide._native import DEFAULT_DEV_SERVER_HOST as DEFAULT_DEV_SERVER_HOST +from sqltide._native import DEFAULT_DEV_SERVER_PORT as DEFAULT_DEV_SERVER_PORT +from sqltide._native import KAFKA_REUSE_LOG_FIRST_READ_COUNT as KAFKA_REUSE_LOG_FIRST_READ_COUNT +from sqltide._native import KAFKA_REUSE_LOG_INTERVAL as KAFKA_REUSE_LOG_INTERVAL +from sqltide._native import TERMINATE_GRACE_SECONDS as TERMINATE_GRACE_SECONDS diff --git a/src/sqltide/dev_server/exceptions.py b/src/sqltide/dev_server/exceptions.py index 6d10844d..d0271af7 100644 --- a/src/sqltide/dev_server/exceptions.py +++ b/src/sqltide/dev_server/exceptions.py @@ -1,37 +1,11 @@ """Dev server exceptions.""" - -class DevServerError(RuntimeError): - """Raised when the dev server cannot satisfy a request.""" - - -class DevConfigurationError(DevServerError): - """Raised when retained dev server configuration is internally inconsistent.""" - - -class KafkaCollectorClosedError(DevServerError): - """Raised when Kafka metadata is requested after its retained clients close.""" - - -class ProjectNotCompiledError(DevServerError): - """Raised when a request needs definitions but the project compile is failing.""" - - -class BuildInProgressError(DevServerError): - """Raised when a build is requested while another one is still running.""" - - -class BuildStartError(DevServerError): - """Raised when a spawned build exits or stalls before its run_started event.""" - - -class AuditSchedulerPersistenceError(DevServerError): - """Raised when a scheduled slot cannot be durably marked as attempted.""" - - -class MessageQueryValidationError(DevServerError): - """Raised when a message browsing document names an unsupported field or value.""" - - -class MessageSchemaError(DevServerError): - """Raised when a raw landing table predates the header-array landing schema.""" +from sqltide._native import AuditSchedulerPersistenceError as AuditSchedulerPersistenceError +from sqltide._native import BuildInProgressError as BuildInProgressError +from sqltide._native import BuildStartError as BuildStartError +from sqltide._native import DevConfigurationError as DevConfigurationError +from sqltide._native import DevServerError as DevServerError +from sqltide._native import KafkaCollectorClosedError as KafkaCollectorClosedError +from sqltide._native import MessageQueryValidationError as MessageQueryValidationError +from sqltide._native import MessageSchemaError as MessageSchemaError +from sqltide._native import ProjectNotCompiledError as ProjectNotCompiledError diff --git a/src/sqltide/dev_server/types.py b/src/sqltide/dev_server/types.py index 95a875b2..f2783257 100644 --- a/src/sqltide/dev_server/types.py +++ b/src/sqltide/dev_server/types.py @@ -2,9 +2,16 @@ from __future__ import annotations -from enum import StrEnum from typing import TYPE_CHECKING, Protocol +from sqltide._native import ActivityTone as ActivityTone +from sqltide._native import AuditScheduleState as AuditScheduleState +from sqltide._native import CompileStateKind as CompileStateKind +from sqltide._native import Freshness as Freshness +from sqltide._native import ReplayAnchorReason as ReplayAnchorReason +from sqltide._native import RunPresentationStatus as RunPresentationStatus +from sqltide._native import SensorSchedulerState as SensorSchedulerState + if TYPE_CHECKING: from pathlib import Path @@ -17,28 +24,6 @@ ] -class ActivityTone(StrEnum): - """How an activity line should read: routine, good news, bad news, or caution.""" - - NEUTRAL = "neutral" - GOOD = "good" - BAD = "bad" - CAUTION = "caution" - - -class RunPresentationStatus(StrEnum): - """Terminal facts and reversible liveness states shown for a run.""" - - SUCCEEDED = "succeeded" - FAILED = "failed" - CANCELLED = "cancelled" - CANCELLING = "cancelling" - CANCELLATION_FAILED = "cancellation_failed" - RUNNING = "running" - UNRESPONSIVE = "unresponsive" - PRESUMED_FAILED = "presumed_failed" - - class CompileAuthorizationGuard(Protocol): """Authorize a reload against the held successful analysis.""" @@ -71,53 +56,3 @@ def report_activity( def report_shutdown(self) -> None: """Print the shutdown line after the server stops.""" - - -class CompileStateKind(StrEnum): - """Whether the held project compile is servable or failing.""" - - OK = "ok" - FAILING = "failing" - - -class ReplayAnchorReason(StrEnum): - """Why a model is or is not a replay anchor.""" - - ELIGIBLE = "eligible" - AGGREGATE = "aggregate" - MUTABLE_REF = "mutable_ref" - NEVER = "never" - LINEAGE_LOSS = "lineage_loss" - VIEW = "view" - - -class Freshness(StrEnum): - """Derived source or model freshness against its authored policy.""" - - FRESH = "fresh" - LAGGING = "lagging" - STALLED = "stalled" - - -class SensorSchedulerState(StrEnum): - """Current lifecycle disposition of the hosted sensor dispatcher.""" - - DISABLED = "disabled" - IDLE = "idle" - RUNNING = "running" - STANDBY = "standby" - BACKING_OFF = "backing_off" - - -class AuditScheduleState(StrEnum): - """Current scheduler disposition of one compiled audit or target.""" - - DISABLED = "disabled" - IDLE = "idle" - DUE = "due" - SCHEDULED = "scheduled" - WARMING_UP = "warming_up" - NOT_MATERIALIZED = "not_materialized" - RUNNING = "running" - BLOCKED = "blocked" - BACKING_OFF = "backing_off" diff --git a/src/sqltide/diagnostics/models.py b/src/sqltide/diagnostics/models.py index a3cf064c..a3dcfb0d 100644 --- a/src/sqltide/diagnostics/models.py +++ b/src/sqltide/diagnostics/models.py @@ -1,44 +1,5 @@ """Immutable structured diagnostic models.""" -from dataclasses import dataclass -from pathlib import Path - -from sqltide.diagnostics.types import DiagnosticPhase, DiagnosticSeverity - - -@dataclass(frozen=True) -class SourceLocation: - """One one-based source span.""" - - path: Path - line: int - column: int - end_line: int | None = None - end_column: int | None = None - - -@dataclass(frozen=True) -class RelatedDiagnosticLocation: - """One labeled source span related to a primary diagnostic.""" - - label: str - location: SourceLocation - message: str | None = None - - -@dataclass(frozen=True) -class CompilerDiagnostic: - """One structured compiler or runtime diagnostic.""" - - phase: DiagnosticPhase | str - severity: DiagnosticSeverity | str - code: str - message: str - resource_name: str | None = None - location: SourceLocation | None = None - related_locations: tuple[RelatedDiagnosticLocation, ...] = () - help: str | None = None - - def __post_init__(self) -> None: - object.__setattr__(self, "phase", DiagnosticPhase(self.phase)) - object.__setattr__(self, "severity", DiagnosticSeverity(self.severity)) +from sqltide._native import CompilerDiagnostic as CompilerDiagnostic +from sqltide._native import RelatedDiagnosticLocation as RelatedDiagnosticLocation +from sqltide._native import SourceLocation as SourceLocation diff --git a/src/sqltide/executor/workflow/constants.py b/src/sqltide/executor/workflow/constants.py index 61975fc4..3e115f0f 100644 --- a/src/sqltide/executor/workflow/constants.py +++ b/src/sqltide/executor/workflow/constants.py @@ -1,19 +1,6 @@ """Stable workflow validation constants.""" -from sqltide.executor.workflow.types import WorkflowPhase - -WORKFLOW_PHASE_ORDER: tuple[WorkflowPhase, ...] = ( - WorkflowPhase.PREFLIGHT, - WorkflowPhase.PREPARATION, - WorkflowPhase.TEARDOWN, - WorkflowPhase.REALIZATION, - WorkflowPhase.STABILIZATION, - WorkflowPhase.BOUNDARY, - WorkflowPhase.REPLAY, - WorkflowPhase.AUDIT, - WorkflowPhase.FINALIZATION, -) - -RECOVER_MODEL_VIEW_STEP_PREFIX: str = "recover_view_" -RECOVER_SOURCE_STEP_PREFIX: str = "recover_source_" -RECOVER_DEAD_LETTERS_STEP_PREFIX: str = "recover_dead_letters_" +from sqltide._native import RECOVER_DEAD_LETTERS_STEP_PREFIX as RECOVER_DEAD_LETTERS_STEP_PREFIX +from sqltide._native import RECOVER_MODEL_VIEW_STEP_PREFIX as RECOVER_MODEL_VIEW_STEP_PREFIX +from sqltide._native import RECOVER_SOURCE_STEP_PREFIX as RECOVER_SOURCE_STEP_PREFIX +from sqltide._native import WORKFLOW_PHASE_ORDER as WORKFLOW_PHASE_ORDER diff --git a/src/sqltide/executor/workflow/models.py b/src/sqltide/executor/workflow/models.py index 20571a8d..8f0b992f 100644 --- a/src/sqltide/executor/workflow/models.py +++ b/src/sqltide/executor/workflow/models.py @@ -1,78 +1,7 @@ """Immutable workflow values and execution results.""" -from dataclasses import dataclass -from pathlib import Path - -from sqltide.adapter.models import ( - AdapterMutationResult, - AdapterQueryResult, - AdapterReplayOffsetProgressRequest, -) -from sqltide.executor.workflow.constants import WORKFLOW_PHASE_ORDER -from sqltide.executor.workflow.exceptions import WorkflowValidationError -from sqltide.executor.workflow.types import StatementIntent, WorkflowMode, WorkflowPhase - - -@dataclass(frozen=True) -class WarehouseStatement: - sequence: int - step_id: str - phase: WorkflowPhase - intent: StatementIntent - sql: str - continue_on_error: bool = False - display_name: str | None = None - replay_offset_progress: AdapterReplayOffsetProgressRequest | None = None - - def __post_init__(self) -> None: - if not self.step_id or not self.step_id.replace("_", "").isalnum(): - raise WorkflowValidationError( - "Workflow step IDs may contain only letters, numbers, and underscores" - ) - if not self.sql.endswith(";") or self.sql.endswith(";;"): - raise WorkflowValidationError( - f"Workflow statement {self.step_id!r} must end with exactly one semicolon" - ) - - -@dataclass(frozen=True) -class BuildWorkflow: - mode: WorkflowMode - plan_json: str - statements: tuple[WarehouseStatement, ...] - recovery_statements: tuple[WarehouseStatement, ...] = () - - def __post_init__(self) -> None: - expected_sequence: int = 1 - previous_phase_index: int = 0 - statement: WarehouseStatement - for statement in self.statements: - if statement.sequence != expected_sequence: - raise WorkflowValidationError( - "Workflow statement sequences must be continuous from one" - ) - phase_index: int = WORKFLOW_PHASE_ORDER.index(statement.phase) - if phase_index < previous_phase_index: - raise WorkflowValidationError("Workflow phases must be monotonic") - expected_sequence += 1 - previous_phase_index = phase_index - - -@dataclass(frozen=True) -class PublishedBuildWorkflow: - workflow: BuildWorkflow - artifact_root: Path - workflow_sha256: str - - -@dataclass(frozen=True) -class WorkflowStatementResult: - step_id: str - query_result: AdapterQueryResult | None - mutation_result: AdapterMutationResult | None - error_message: str | None = None - - -@dataclass(frozen=True) -class WorkflowExecutionResult: - statement_results: tuple[WorkflowStatementResult, ...] +from sqltide._native import BuildWorkflow as BuildWorkflow +from sqltide._native import PublishedBuildWorkflow as PublishedBuildWorkflow +from sqltide._native import WarehouseStatement as WarehouseStatement +from sqltide._native import WorkflowExecutionResult as WorkflowExecutionResult +from sqltide._native import WorkflowStatementResult as WorkflowStatementResult diff --git a/src/sqltide/sensors/constants.py b/src/sqltide/sensors/constants.py index e4c4dc03..e936f9f7 100644 --- a/src/sqltide/sensors/constants.py +++ b/src/sqltide/sensors/constants.py @@ -1,3 +1,3 @@ """Sensor domain constants.""" -DEFAULT_SENSOR_TIMEOUT_SECONDS: float = 60.0 +from sqltide._native import DEFAULT_SENSOR_TIMEOUT_SECONDS as DEFAULT_SENSOR_TIMEOUT_SECONDS diff --git a/src/sqltide/sensors/exceptions.py b/src/sqltide/sensors/exceptions.py index 1dc80d70..85df1c66 100644 --- a/src/sqltide/sensors/exceptions.py +++ b/src/sqltide/sensors/exceptions.py @@ -1,40 +1,5 @@ """Sensor exceptions.""" -from __future__ import annotations - -from sqltide.diagnostics.models import ( - CompilerDiagnostic, - RelatedDiagnosticLocation, - SourceLocation, -) -from sqltide.diagnostics.types import DiagnosticPhase, DiagnosticSeverity - - -class SensorError(ValueError): - """Raised when sensor definitions or project sensor state is invalid.""" - - def __init__( - self, - message: str, - *, - location: SourceLocation | None = None, - related_locations: tuple[RelatedDiagnosticLocation, ...] = (), - ) -> None: - super().__init__(message) - self.location: SourceLocation | None = location - self.diagnostic: CompilerDiagnostic = CompilerDiagnostic( - phase=DiagnosticPhase.COMPILATION, - severity=DiagnosticSeverity.ERROR, - code="SQT-COMPILE-001", - message=message, - location=location, - related_locations=related_locations, - ) - - -class SensorStepError(RuntimeError): - """Raised when a durable step cannot run or return a memoized result.""" - - -class SensorEvaluationTimeoutError(RuntimeError): - """Raised when one sensor evaluation exceeds its timeout.""" +from sqltide._native import SensorError as SensorError +from sqltide._native import SensorEvaluationTimeoutError as SensorEvaluationTimeoutError +from sqltide._native import SensorStepError as SensorStepError From cd38bc0f68299809275b1728aad2ca054c990ab8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 03:25:36 +0000 Subject: [PATCH 3/3] docs: name the consumers and contents of the new boundary stays --- python-boundary.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python-boundary.toml b/python-boundary.toml index ee8358da..795fc1a4 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -224,9 +224,9 @@ note = "Rule 3: PollingSensorContext that user polling sensors receive, re-expor [[stays]] path = "auth/models.py" rule = 4 -note = "Rule 4: pydantic request bodies LoginRequest, CreateUserRequest, UpdateUserRequest, PasswordResetRequest, RoleRequest and ProjectRoleRequest that the native dev server validates with pydantic for the account routes, beside native re-exports of the account shapes; no logic." +note = "Rule 4: pydantic request bodies LoginRequest, CreateUserRequest, UpdateUserRequest, PasswordResetRequest, RoleRequest and ProjectRoleRequest, which the native dev server validates with pydantic for the account routes. They hold only field declarations with Field constraints and defaults and no custom validators or model config; the other names here are native re-exports of the account shapes." [[stays]] path = "dev_server/types.py" rule = 1 -note = "Rule 1: Logic-free protocols CompileAuthorizationGuard (the compile authorization check the dev server runtime calls) and DevServerReporter (the terminal narrator the CLI implements) and the data-only type aliases KafkaClientKey and KafkaLagOffsets the dev server runtime consumes, beside native re-exports of the dev server value enums; no logic." +note = "Rule 1: Logic-free protocols CompileAuthorizationGuard, the reload guard native DevServerState.reload_guarded calls, and DevServerReporter, the terminal narrator native create_dev_app, run_dev_server and BuildProcessManager call; and the data-only type aliases KafkaClientKey, used by the rule-4 Kafka topic and lag collectors, and KafkaLagOffsets, used by the lag collector. The other names here are native re-exports of the dev server value enums; no logic."