diff --git a/crates/sqltide-python/src/shapes/adapter_models.rs b/crates/sqltide-python/src/shapes/adapter_models.rs new file mode 100644 index 00000000..2fe68840 --- /dev/null +++ b/crates/sqltide-python/src/shapes/adapter_models.rs @@ -0,0 +1,896 @@ +//! Neutral adapter identity, connection, catalog, metadata and result shapes. + +use pyo3::prelude::*; +use pyo3::types::{IntoPyDict, PyDict, PyFrozenSet, PyFrozenSetBuilder, PyString, PyTuple}; + +use super::{Coerce, Given, add_class, shapes}; + +const ADAPTER_TYPES: &str = "sqltide.adapter.types"; +const ADAPTER_EXCEPTIONS: &str = "sqltide.adapter.exceptions"; +const REPLAY_BOUNDARY_MODE: Coerce = Coerce::With(replay_boundary_mode); +const OPTIONAL_STATE_STATUS: Coerce = Coerce::With(optional_state_status); +const HEALTH_AVAILABILITY: Coerce = Coerce::With(health_availability); +const HEALTH_STATUS: Coerce = Coerce::With(health_status); + +/// `value` as a member of the adapter enum `name`, which stays defined in Python and is read +/// when first needed so the native module never imports it while it loads. +fn adapter_enum<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult> { + value + .py() + .import(ADAPTER_TYPES)? + .getattr(name)? + .call1((value,)) +} + +fn replay_boundary_mode<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + adapter_enum(value, "AdapterReplayBoundaryMode") +} + +fn optional_state_status<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + adapter_enum(value, "AdapterOptionalStateStatus") +} + +fn health_availability<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + adapter_enum(value, "AdapterWarehouseHealthAvailability") +} + +fn health_status<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + adapter_enum(value, "AdapterWarehouseHealthStatus") +} + +shapes! { + module = "sqltide.adapter.models"; + + /// The registered name of one adapter implementation. + AdapterIdentity { name } + + /// Capabilities implemented by one adapter. + AdapterCapabilities { + virtual_environments, + managed_source_kinds, + replay_boundary_modes, + history_prefix_seed, + stable_logical_bindings, + per_relation_atomic_replace, + graph_atomic_publish, + set_difference_comparison, + direct_rebuild, + } + + /// One retained lineage range required to reproduce a direct target. + AdapterReplayCoverageRange { + driving_input_relation_name, + replay_boundary_mode(coerce = REPLAY_BOUNDARY_MODE), + boundary_key, + source_partition_column_name, + source_position_column_name, + source_timestamp_column_name, + lower_value, + upper_value, + } + + /// One neutral actual/expected bag comparison requested from an adapter. + AdapterSetDifferenceTarget { name, column_names, ctes, actual_query, expected_query } + + /// One executable statement containing every SQL-test comparison target. + AdapterSetDifferenceComparisonRequest { targets } + + /// One column in a neutral adapter resource request. + AdapterColumn { name, r#type, default_expression(default = Given::None) } + + /// A managed streaming source to realize in a warehouse. + AdapterManagedSource { + source_kind, + name, + columns, + broker_list, + topic, + consumer_group, + format, + settings(default = Given::EmptyTuple), + naming_macro_fingerprint(default = Given::None), + } + + /// A managed table to realize in a warehouse. + AdapterTable { + name, + columns, + engine, + order_by, + partition_by(default = Given::None), + ttl(default = Given::None), + settings(default = Given::EmptyTuple), + } + + /// A materialized view to realize between warehouse relations. + AdapterMaterializedView { + name, + source_relation_name, + target_relation_name, + query, + database_template, + refresh(default = Given::None), + append(default = Given::Bool(true)), + dead_letters(default = Given::Bool(false)), + } + + /// An ordinary query view to realize in a warehouse. + AdapterView { name, query, database_template } + + /// Logical managed-source fields needed for adapter realization. + AdapterManagedSourceRealizationRequest { + logical_name, + source_kind, + broker_list, + topic, + consumer_group, + format, + project_name(default = Given::None), + target_name(default = Given::None), + ttl(default = Given::None), + settings(default = Given::EmptyTuple), + naming_macro_fingerprint(default = Given::None), + } + + /// Logical adopted-source fields needed for adapter realization. + AdapterAdoptedSourceRealizationRequest { logical_name, relation_name } + + /// One logical source mapped to its adapter relation and resources. + AdapterSourceRealization { relation_name, resources } + + /// The warehouse-reported refresh state of one scheduled relation. + AdapterRefreshState { + view_name, + status, + last_refresh_at, + last_success_at, + next_refresh_at, + exception, + } + + /// One semantically compiled logical model ready for adapter realization. + AdapterModelRealizationRequest { + logical_name, + target_relation_name, + source_relation_name, + resolved_query, + resolved_database_template, + columns, + engine, + order_by, + partition_by(default = Given::None), + ttl(default = Given::None), + settings(default = Given::EmptyTuple), + refresh(default = Given::None), + append(default = Given::Bool(true)), + dead_letter_query(default = Given::None), + dead_letter_database_template(default = Given::None), + } + + /// One compiled query-only view ready for adapter realization. + AdapterViewRealizationRequest { + logical_name, + target_relation_name, + resolved_query, + resolved_database_template, + } + + /// One logical model mapped to its adapter relation and resources. + AdapterModelRealization { relation_name, resources } + + /// A stable logical view to bind to one physical relation. + AdapterStableView { name, target_relation_name } + + /// One logical relation bound to one physical relation. + AdapterStableBinding { database, logical_name, physical_name } + + /// One obsolete framework-owned stable binding to remove. + AdapterStableBindingRemoval { database, logical_name } + + /// A set of stable bindings to replace during one publish operation. + AdapterBindingReplacementRequest { bindings, removals(default = Given::EmptyTuple) } + + /// Physical relations to remove from one database. + AdapterRelationCleanupRequest { database, relation_names } + + /// Exclusive warehouse-backed ownership of one target mutation slot. + AdapterTargetMutationLock { database, owner_id } + + /// One staged root whose live and candidate relations must be compared. + AdapterReadinessRootRequest { database, logical_name, staged_relation_name, active_exists } + + /// A mode-neutral readiness comparison request. + AdapterReadinessRequest { roots } + + /// Adapter-observed partition and lag metrics for offset replay. + AdapterReadinessOffsetSummary { + active_partition_count, + staged_partition_count, + partitions_compared, + missing_staged_partition_count, + missing_freshness_partition_count, + lagging_partition_count, + max_offset_gap, + average_offset_gap, + lag_boundary_column, + max_lag_seconds, + average_lag_seconds, + } + + /// Adapter-observed range and lag metrics for scalar replay. + AdapterReadinessScalarSummary { + active_min_value, + active_max_value, + staged_min_value, + staged_max_value, + lag_seconds, + } + + /// Warehouse observations for one requested staged root. + AdapterReadinessRootObservation { + root, + staged_exists, + active_row_count, + staged_row_count, + replay_source_name, + replay_source_row_count, + replay_boundary_mode, + offset_summary, + scalar_summary, + } + + /// A logical relation mapped to its replay-time physical relation. + AdapterPhysicalRelationMapping { logical_name, physical_name } + + /// A compiler-analyzed query ready for adapter replay realization. + AdapterReplayQuery { query, physical_relation_mappings, aggregate_semantics } + + /// One inclusive replay cutoff, optionally scoped to a partition. + AdapterReplayBoundary { + boundary_key, + cutoff_value, + cutoff_inclusive, + partition_value(default = Given::None), + } + + /// Physical boundary columns used to read an anchor relation. + AdapterReplayColumns { partition, offset, timestamp, landed_at, cursor } + + /// The lower-bound policy for one replay root. + AdapterReplayWindow { + lower_bound_mode, + lower_bound_inclusive, + boundary_time, + forced_start_time, + lookback_seconds, + } + + /// Logical and physical relations participating in one replay root. + AdapterReplayRelations { root, source, anchor, target } + + /// A mode-neutral request to seed and replay one rebuild root. + AdapterReplayRequest { + mode, + database, + relations, + replay_query, + boundaries, + columns, + window, + seed_mode, + target_column_names, + settings(default = Given::EmptyTuple), + filter_boundaries_at_source(default = Given::Bool(false)), + dedupe_offsets(default = Given::Bool(false)), + } + + /// A request for the retained physical ranges selected by one replay window. + AdapterReplayCoverageRequest { + replay, + boundary_column_type, + count_anchor_rows(default = Given::Bool(false)), + } + + /// One captured replay lower bound in adapter-neutral string form. + AdapterReplayLowerBound { value, partition_value(default = Given::None) } + + /// A replay rendered from boundaries captured by the current process. + AdapterCapturedReplayRequest { replay, boundary_column_type, lower_bounds } + + /// One logical SQL baseline recorded after direct materialization. + AdapterDirectFingerprintRecord { + fingerprint_id, + logical_model_identity, + definition_sql, + definition_hash, + identity_metadata, + workflow_id, + tool_version, + applied_at(default = Given::None), + } + + /// Explicit availability and latest logical direct SQL baselines. + AdapterDirectFingerprintSnapshot { + status(coerce = OPTIONAL_STATE_STATUS), + baselines, + warning(default = Given::None), + } + + /// One physical resource recorded in a published project manifest. + AdapterManifestResource { + pipeline_name, + logical_type, + logical_name, + resource_role, + resource_database, + resource_name, + resource_kind, + } + + /// One complete, immutable publication manifest. + AdapterManifest { + manifest_id, + invocation_id, + project_identity, + target_name, + target_database, + is_production, + project_revision, + manifest_fingerprint, + manifest_version, + pipelines, + resources, + tool_version, + published_at, + } + + /// Availability and complete publication history for one project target. + AdapterManifestSnapshot { + status(coerce = OPTIONAL_STATE_STATUS), + manifests, + warning(default = Given::None), + } + + /// A replay whose dynamic boundaries live in deployment watermark metadata. + AdapterDeploymentReplayRequest { + replay, + metadata_database, + deployment_id, + boundary_column_type, + active_relation_name, + active_column_names, + anchor_column_names, + } + + /// A logical object identity stored in framework metadata. + AdapterMetadataObjectKey { database, object_type, name } + + /// A persisted logical-to-physical object mapping. + AdapterPreparedObjectMapping { logical_key, physical_name, logical_model_name } + + /// One adapter-neutral object-state metadata record. + AdapterObjectStateRecord { + deployment_id, + key, + normalized_fingerprint, + normalized_query, + recorded_at, + observation_id(default = Given::Str("")), + state_kind(default = Given::Str("deployment")), + physical_database_name(default = Given::None), + physical_relation_name(default = Given::None), + logical_model_database(default = Given::None), + logical_model_name(default = Given::None), + is_selected_root(default = Given::Bool(false)), + } + + /// One adapter-neutral staged deployment record. + AdapterDeploymentRecord { + deployment_id, + created_at, + status, + replay_lineage_mode, + selected_root_keys, + warning_codes, + prepared_object_mappings, + workflow_fingerprint(default = Given::Str("")), + boundary_time(default = Given::None), + tool_version(default = Given::Str("")), + } + + /// One adapter-neutral replay watermark record. + AdapterDeploymentWatermarkRecord { + deployment_id, + root_key, + anchor_key, + boundary_key, + cutoff_value, + lower_value(default = Given::None), + cutoff_inclusive(default = Given::Bool(true)), + captured_at(default = Given::Str("1970-01-01 00:00:00.000")), + } + + /// One adapter-neutral publish-history record. + AdapterPublishEventRecord { + deployment_id, + published_at, + logical_view_names, + publication_id(default = Given::Str("")), + bindings(default = Given::EmptyTuple), + operation(default = Given::Str("promote")), + previous_deployment_id(default = Given::None), + } + + /// One immutable terminal CLI invocation observation. + AdapterInvocationRecord { + invocation_id, + project_identity, + target_identity, + command, + mode, + outcome, + exit_code, + materialized_outcome, + deployment_id, + workflow_id, + selected_node_count, + started_at, + completed_at, + duration_ms, + error_message, + summary_json, + tool_version, + artifact_project_dir(default = Given::None), + } + + /// One immutable terminal audit or test result observation. + AdapterNodeResultRecord { + result_id, + invocation_id, + node_kind, + node_name, + binding_key, + definition_fingerprint, + execution_fingerprint, + target_identity, + trigger, + scheduled_for, + cadence_seconds, + warmup_seconds, + status, + severity, + failure_count, + completed_at, + payload_json, + error_message, + } + + /// One step-granular workflow event streamed while a run executes. + AdapterRunEventRecord { invocation_id, sequence, event_kind, step_id, phase, payload_json } + + /// One display-safe warehouse statement durably associated with a run. + AdapterRunStatementRecord { + invocation_id, + statement_sequence, + step_id, + phase, + intent, + sql, + sql_sha256, + workflow_sha256, + workflow_revision, + } + + /// One append-only latest-wins sensor stream position. + AdapterSensorCheckpointRecord { sensor_name, source, position_completed_at, position_result_id } + + /// One persisted sensor evaluation attempt. + AdapterSensorTickRecord { + tick_id, + sensor_name, + definition_fingerprint, + kind, + event_id, + event_kind, + attempt, + status, + started_at, + completed_at, + error_message, + skip_reason, + cursor, + } + + /// One durable step marker keyed by sensor, event, and step key. + AdapterSensorStepRecord { + sensor_name, + event_id, + step_key, + policy, + status, + attempt, + result_json, + error_message, + } + + /// One append-only operator override of a sensor's declared status. + AdapterSensorOverrideRecord { override_id, sensor_name, status, actor } + + /// One warehouse-time dispatch lease claim. + AdapterSensorLeaseRecord { lease_name, owner_id, acquired_at, expires_at } + + /// Sensor state records persisted in one batch. + AdapterSensorState { + checkpoints(default = Given::EmptyTuple), + ticks(default = Given::EmptyTuple), + steps(default = Given::EmptyTuple), + overrides(default = Given::EmptyTuple), + leases(default = Given::EmptyTuple), + } + + /// One current manifest node joined to persisted terminal history. + AdapterCurrentQualityNode { + node_kind, + node_name, + binding_key, + definition_fingerprint, + execution_fingerprint, + cadence_seconds(default = Given::None), + warmup_seconds(default = Given::Int(0)), + } + + /// Persisted deployments and publish events used for lifecycle cleanup. + AdapterDeploymentInventory { deployments, publish_events } + + /// A batch of framework metadata records for adapter persistence. + AdapterMetadataState { + object_states, + deployments, + deployment_watermarks, + publish_events, + invocations(default = Given::EmptyTuple), + node_results(default = Given::EmptyTuple), + } + + /// Stable identity of one adapter/database catalog observation. + CatalogIdentity { adapter, database } + + /// One warehouse relation column observed by an adapter. + CatalogColumn { name, r#type, default_expression(default = Given::None) } + + /// One immutable warehouse relation observation. + CatalogRelation { + name, + engine, + columns, + full_engine(default = Given::None), + order_by(default = Given::EmptyTuple), + partition_by(default = Given::None), + ttl(default = Given::None), + settings(default = Given::EmptyTuple), + definition_sql(default = Given::None), + query_sql(default = Given::None), + source_relation_names(default = Given::EmptyTuple), + target_relation_name(default = Given::None), + stable_binding_name(default = Given::None), + ownership_generation(default = Given::None), + } + @impl { + /// Return the engine including storage arguments when available. + #[getter] + fn storage_engine(&self, py: Python<'_>) -> PyResult> { + let full_engine: &Bound<'_, PyAny> = self.full_engine.bind(py); + if full_engine.is_truthy()? { + return Ok(full_engine.clone().unbind()); + } + Ok(self.engine.clone_ref(py)) + } + + /// Return the first physical source when one exists. + #[getter] + fn source_relation_name(&self, py: Python<'_>) -> PyResult> { + let names: &Bound<'_, PyAny> = self.source_relation_names.bind(py); + if names.is_truthy()? { + return Ok(names.get_item(0)?.unbind()); + } + Ok(py.None()) + } + } + + /// One immutable point-in-time catalog for an adapter and database. + CatalogSnapshot { identity, warehouse_timezone, relations } + @impl { + /// Return one relation by unqualified name when it exists. + fn relation(&self, py: Python<'_>, name: &Bound<'_, PyAny>) -> PyResult> { + for relation in self.relations.bind(py).try_iter()? { + let relation: Bound<'_, PyAny> = relation?; + if relation.getattr("name")?.eq(name)? { + return Ok(relation.unbind()); + } + } + Ok(py.None()) + } + + /// Return every observed unqualified relation name. + fn relation_names<'py>(&self, py: Python<'py>) -> PyResult> { + let mut names: PyFrozenSetBuilder<'py> = PyFrozenSetBuilder::new(py)?; + for relation in self.relations.bind(py).try_iter()? { + names.add(relation?.getattr("name")?)?; + } + Ok(names.finalize()) + } + } + + /// A stable logical view pointing at an active physical table. + InspectedActiveTableBinding { database, logical_name, physical_name } + + /// A deployment-suffixed physical model candidate for a logical root. + InspectedPhysicalTableCandidate { + database, + logical_name, + physical_name, + object_type(default = Given::Str("table")), + } + + /// Managed table state used for active-deployment resolution. + InspectedManagedTableState { active_bindings, physical_candidates } + + /// Resolved, format-neutral connection settings for one adapter. + AdapterConnectionConfig @no_repr { + host, + port, + username, + password, + database(default = Given::None), + settings(default = Given::EmptyTuple), + } + @impl { + /// Render the configuration without exposing the password. + fn __repr__(slf: &Bound<'_, Self>) -> PyResult { + let py: Python<'_> = slf.py(); + let config: &Self = slf.get(); + let class: Bound<'_, PyString> = slf.get_type().name()?; + let host: Bound<'_, PyString> = config.host.bind(py).repr()?; + let port: Bound<'_, PyString> = config.port.bind(py).repr()?; + let username: Bound<'_, PyString> = config.username.bind(py).repr()?; + let password: Bound<'_, PyString> = py + .import("sqltide._native")? + .getattr("REDACTED_SECRET_PLACEHOLDER")? + .repr()?; + let database: Bound<'_, PyString> = config.database.bind(py).repr()?; + let settings: Bound<'_, PyString> = config.settings.bind(py).repr()?; + Ok(format!( + "{class}(host={host}, port={port}, username={username}, password={password}, \ + database={database}, settings={settings})" + )) + } + } + + /// A normalized query result returned by the adapter boundary. + AdapterQueryResult { rows, column_names(default = Given::EmptyTuple) } + @impl { + /// Return rows keyed by the query's column names. + fn named_rows<'py>(&self, py: Python<'py>) -> PyResult> { + let column_names: &Bound<'py, PyAny> = self.column_names.bind(py); + let rows: &Bound<'py, PyAny> = self.rows.bind(py); + if !column_names.is_truthy()? { + if !rows.is_truthy()? { + return Ok(PyTuple::empty(py)); + } + return Err(PyErr::from_value( + py.import(ADAPTER_EXCEPTIONS)? + .getattr("AdapterResultError")? + .call1(("Query result does not include column names",))?, + )); + } + let zip: Bound<'py, PyAny> = py.import("builtins")?.getattr("zip")?; + let strict: Bound<'py, PyDict> = [("strict", true)].into_py_dict(py)?; + let mut named: Vec> = Vec::new(); + for row in rows.try_iter()? { + let pairs: Bound<'py, PyAny> = zip.call((column_names, row?), Some(&strict))?; + named.push(py.get_type::().call1((pairs,))?); + } + PyTuple::new(py, named) + } + } + + /// Capacity and severity for one configured warehouse disk. + AdapterWarehouseDisk { + name, + path, + disk_type, + total_bytes, + free_bytes, + unreserved_bytes, + keep_free_bytes, + status(coerce = HEALTH_STATUS), + } + + /// Truthfully labelled warehouse process and execution-boundary memory. + AdapterWarehouseMemory { + resident_bytes, + host_total_bytes, + cgroup_used_bytes, + cgroup_limit_bytes, + basis, + pressure_fraction, + } + + /// Current bounded warehouse work counts. + AdapterWarehouseActivity { active_queries, active_merges, incomplete_mutations } + + /// A Kafka table that keeps failing without committing an offset. + AdapterWarehouseStuckKafkaConsumer { table, stuck_seconds, recent_errors, latest_error } + + /// Managed Kafka consumer health observed inside the warehouse. + AdapterWarehouseKafkaConsumers { + configured_tables, + materialized_tables, + materialized_table_names, + polling_tables, + exception_tables, + stuck_consumers(default = Given::EmptyTuple), + } + + /// Quarantined rows of one model over the last 24 hours and its latest row. + AdapterWarehouseDeadLetterModel { + model_name, + row_count, + latest_failed_columns, + latest_payload, + latest_quarantined_at, + } + + /// Active physical footprint for one project-database table. + AdapterWarehouseTable { name, rows, bytes_on_disk, active_parts } + + /// Adapter-neutral point-in-time warehouse diagnostic snapshot. + AdapterWarehouseHealth { + availability(coerce = HEALTH_AVAILABILITY), + status(coerce = HEALTH_STATUS), + version, + uptime_seconds, + disks, + inode_total, + inode_free, + inode_status(coerce = HEALTH_STATUS), + memory, + activity, + tables, + collection_duration_ms, + kafka_consumers(default = Given::None), + dead_letters(default = Given::None), + capacity_warning_fraction(default = Given::None), + capacity_critical_fraction(default = Given::None), + warnings(default = Given::EmptyTuple), + stale(default = Given::Bool(false)), + measured_at(default = Given::None), + } + + /// Warehouse-reported evidence for one executed mutation statement. + AdapterMutationResult { written_rows(default = Given::None) } + + /// Ephemeral warehouse telemetry for one currently executing statement. + AdapterStatementProgress { + elapsed_seconds, + read_rows, + read_bytes, + total_rows_approx, + memory_usage_bytes, + settings(default = Given::EmptyTuple), + } + + /// Adapter-confirmed outcome for one exactly identified workflow query. + AdapterQueryCancellation { + query_id, + supported, + query_found, + termination_confirmed, + detail(default = Given::None), + } + + /// One captured partition span used only for approximate replay progress. + AdapterReplayOffsetRange { partition, lower_offset, upper_offset } + + /// Target relation and immutable captured ranges for frontier observation. + AdapterReplayOffsetProgressRequest { + database, + relation, + partition_column, + offset_column, + ranges, + } + + /// Greatest committed target offset observed for one replay partition. + AdapterReplayOffsetFrontier { partition, completed_offset } +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + add_class::(module)?; + Ok(()) +} diff --git a/crates/sqltide-python/src/shapes/mod.rs b/crates/sqltide-python/src/shapes/mod.rs index 5c0f09d9..c1e622de 100644 --- a/crates/sqltide-python/src/shapes/mod.rs +++ b/crates/sqltide-python/src/shapes/mod.rs @@ -15,6 +15,7 @@ use pyo3::types::{ IntoPyDict, PyCFunction, PyDict, PyFloat, PyFrozenSet, PyString, PyTuple, PyType, }; +mod adapter_models; mod cli_service; mod compiler_analysis; mod compiler_project; @@ -186,7 +187,10 @@ macro_rules! shapes { kwargs: Option<&pyo3::Bound<'_, pyo3::types::PyDict>>, ) -> pyo3::PyResult> { let values: Vec> = - $crate::shapes::bind(&Self::SPEC, args, kwargs)?; + match $crate::shapes::copied_values(args, kwargs)? { + Some(values) => values, + None => $crate::shapes::bind(&Self::SPEC, args, kwargs)?, + }; Ok($crate::shapes::shapes!(@init values $($base)?)) } @@ -249,8 +253,27 @@ macro_rules! shapes { $crate::shapes::missing_attribute(slf.as_any(), name) } - /// Rebuilds the shape from its field values, so `copy`, `deepcopy` and `pickle` - /// work as they did on the dataclass. + /// A new instance of the same class sharing every field value, as `copy.copy` of + /// the dataclass gave. + fn __copy__<'py>( + slf: &pyo3::Bound<'py, Self>, + ) -> pyo3::PyResult> { + $crate::shapes::shallow_copy(slf.as_any(), slf.get().values(slf.py())) + } + + /// `copy.replace` on Python 3.13 and later, through `dataclasses.replace` as the + /// dataclass's own `__replace__` did. + #[pyo3(signature = (**changes))] + fn __replace__<'py>( + slf: &pyo3::Bound<'py, Self>, + changes: Option<&pyo3::Bound<'py, pyo3::types::PyDict>>, + ) -> pyo3::PyResult> { + use pyo3::types::PyAnyMethods; + slf.py().import("dataclasses")?.getattr("replace")?.call((slf,), changes) + } + + /// Rebuilds the shape from its field values, so `deepcopy` and `pickle` work as they + /// did on the dataclass. fn __reduce__<'py>( slf: &pyo3::Bound<'py, Self>, ) -> pyo3::PyResult<( @@ -283,6 +306,52 @@ pub(crate) fn next<'py>( .ok_or_else(|| PyValueError::new_err("shape constructor bound too few values")) } +/// The private first constructor argument with which `copy.copy` hands a shape class the field +/// values to keep as they are, so a copy of any subclass is built without rebinding them. +static COPY_MARKER: PyOnceLock> = PyOnceLock::new(); + +fn copy_marker(py: Python<'_>) -> PyResult<&Py> { + COPY_MARKER.get_or_try_init(py, || { + Ok(py.import("builtins")?.getattr("object")?.call0()?.unbind()) + }) +} + +/// The field values `shallow_copy` passed to the constructor, or `None` for any other call. +pub(crate) fn copied_values<'py>( + args: &Bound<'py, PyTuple>, + kwargs: Option<&Bound<'py, PyDict>>, +) -> PyResult>>> { + if args.len() != 2 || kwargs.is_some_and(|kwargs| !kwargs.is_empty()) { + return Ok(None); + } + if !args.get_item(0)?.is(copy_marker(args.py())?) { + return Ok(None); + } + let values: Bound<'py, PyTuple> = args.get_item(1)?.cast_into()?; + Ok(Some(values.iter().collect())) +} + +/// `copy.copy` as it worked on the dataclass: a new instance of the instance's own class that +/// shares every field value, without rerunning normalization, post-init checks or a subclass +/// `__init__`, and with a copy of a Python subclass instance's `__dict__`. +pub(crate) fn shallow_copy<'py>( + instance: &Bound<'py, PyAny>, + values: Vec>, +) -> PyResult> { + let py: Python<'py> = instance.py(); + let class: Bound<'py, PyType> = instance.get_type(); + let copied: Bound<'py, PyAny> = + class + .getattr("__new__")? + .call1((&class, copy_marker(py)?, PyTuple::new(py, values)?))?; + if class.getattr("__dictoffset__")?.extract::()? != 0 { + copied + .getattr("__dict__")? + .call_method1("update", (instance.getattr("__dict__")?,))?; + } + Ok(copied) +} + /// Native enums by their `sqltide._native` attribute, for defaults and coercions. static ENUMS: PyOnceLock> = PyOnceLock::new(); @@ -877,6 +946,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { cli_service::register(module)?; sensors_dev::register(module)?; discovery_models::register(module)?; + adapter_models::register(module)?; lazy_constants( module, &[ diff --git a/python-boundary.toml b/python-boundary.toml index 47217848..207da462 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -8,7 +8,7 @@ # still carries planning, validation, rendering or other movable logic counts as movable until # that logic moves to Rust. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 1080 +remaining_lines = 0 [[stays]] path = "adapter/classes/adapter.py" diff --git a/src/sqltide/_native.pyi b/src/sqltide/_native.pyi index c6ee8913..79910231 100644 --- a/src/sqltide/_native.pyi +++ b/src/sqltide/_native.pyi @@ -12,68 +12,18 @@ from uuid import UUID from sqltide.adapter.classes.adapter import Adapter from sqltide.adapter.classes.adapter_connection import AdapterConnection from sqltide.adapter.exceptions import AdapterWarehouseError -from sqltide.adapter.models import ( - AdapterAdoptedSourceRealizationRequest, - AdapterBindingReplacementRequest, - AdapterCapabilities, - AdapterCapturedReplayRequest, - AdapterConnectionConfig, - AdapterCurrentQualityNode, - AdapterDeploymentInventory, - AdapterDeploymentReplayRequest, - AdapterDirectFingerprintRecord, - AdapterDirectFingerprintSnapshot, - AdapterIdentity, - AdapterInvocationRecord, - AdapterManagedSource, - AdapterManagedSourceRealizationRequest, - AdapterManifest, - AdapterManifestSnapshot, - AdapterMaterializedView, - AdapterMetadataState, - AdapterModelRealization, - AdapterModelRealizationRequest, - AdapterMutationResult, - AdapterNodeResultRecord, - AdapterPhysicalRelationMapping, - AdapterQueryCancellation, - AdapterQueryResult, - AdapterReadinessRequest, - AdapterReadinessRootObservation, - AdapterRefreshState, - AdapterRelationCleanupRequest, - AdapterReplayColumns, - AdapterReplayCoverageRequest, - AdapterReplayOffsetFrontier, - AdapterReplayOffsetProgressRequest, - AdapterReplayQuery, - AdapterReplayRequest, - AdapterRunEventRecord, - AdapterRunStatementRecord, - AdapterSensorState, - AdapterSensorStepRecord, - AdapterSensorTickRecord, - AdapterSetDifferenceComparisonRequest, - AdapterSourceRealization, - AdapterStableView, - AdapterStatementProgress, - AdapterTable, - AdapterTargetMutationLock, - AdapterView, - AdapterViewRealizationRequest, - AdapterWarehouseHealth, - CatalogRelation, - CatalogSnapshot, - InspectedActiveTableBinding, - InspectedManagedTableState, -) from sqltide.adapter.types import ( AdapterModelRealizer, AdapterModelRelationNamer, + AdapterOptionalStateStatus, AdapterReplayBoundaryMode, + AdapterReplayLowerBoundMode, + AdapterReplaySeedMode, AdapterResourceRenderer, AdapterSetDifferenceComparisonRenderer, AdapterSourceRealizer, + AdapterWarehouseHealthAvailability, + AdapterWarehouseHealthStatus, ) from sqltide.adapters.clickhouse.classes.clickhouse_connection import ClickHouseConnection from sqltide.adapters.clickhouse.types import RawClickHouseClient @@ -3693,6 +3643,846 @@ class DiscoveredProjectInputs: constant_files: tuple[DiscoveredProjectFile, ...] = () enum_files: tuple[DiscoveredProjectFile, ...] = () +# Neutral adapter identity, connection, catalog, metadata and result shapes. + +@final +@dataclass(frozen=True) +class AdapterIdentity: + name: str + +@final +@dataclass(frozen=True) +class AdapterCapabilities: + virtual_environments: bool + managed_source_kinds: frozenset[str] + replay_boundary_modes: frozenset[AdapterReplayBoundaryMode] + history_prefix_seed: bool + stable_logical_bindings: bool + per_relation_atomic_replace: bool + graph_atomic_publish: bool + set_difference_comparison: bool + direct_rebuild: bool + +@final +@dataclass(frozen=True) +class AdapterReplayCoverageRange: + driving_input_relation_name: str + replay_boundary_mode: AdapterReplayBoundaryMode | str + boundary_key: str + source_partition_column_name: str | None + source_position_column_name: str + source_timestamp_column_name: str | None + lower_value: str + upper_value: str + +@final +@dataclass(frozen=True) +class AdapterSetDifferenceTarget: + name: str + column_names: tuple[str, ...] + ctes: tuple[tuple[str, str], ...] + actual_query: str + expected_query: str | None + +@final +@dataclass(frozen=True) +class AdapterSetDifferenceComparisonRequest: + targets: tuple[AdapterSetDifferenceTarget, ...] + +@final +@dataclass(frozen=True) +class AdapterColumn: + name: str + type: str + default_expression: str | None = None + +@final +@dataclass(frozen=True) +class AdapterManagedSource: + source_kind: str + name: str + columns: tuple[AdapterColumn, ...] + broker_list: str + topic: str + consumer_group: str + format: str + settings: tuple[tuple[str, str], ...] = () + naming_macro_fingerprint: str | None = None + +@final +@dataclass(frozen=True) +class AdapterTable: + name: str + columns: tuple[AdapterColumn, ...] + engine: str + order_by: tuple[str, ...] + partition_by: str | None = None + ttl: str | None = None + settings: tuple[tuple[str, str], ...] = () + +@final +@dataclass(frozen=True) +class AdapterMaterializedView: + name: str + source_relation_name: str + target_relation_name: str + query: str + database_template: str + refresh: str | None = None + append: bool = True + dead_letters: bool = False + +@final +@dataclass(frozen=True) +class AdapterView: + name: str + query: str + database_template: str + +@final +@dataclass(frozen=True) +class AdapterManagedSourceRealizationRequest: + logical_name: str + source_kind: str + broker_list: str + topic: str + consumer_group: str | None + format: str + project_name: str | None = None + target_name: str | None = None + ttl: str | None = None + settings: tuple[tuple[str, str], ...] = () + naming_macro_fingerprint: str | None = None + +@final +@dataclass(frozen=True) +class AdapterAdoptedSourceRealizationRequest: + logical_name: str + relation_name: str + +@final +@dataclass(frozen=True) +class AdapterSourceRealization: + relation_name: str + resources: tuple[AdapterManagedSource | AdapterTable | AdapterMaterializedView, ...] + +@final +@dataclass(frozen=True) +class AdapterRefreshState: + view_name: str + status: str + last_refresh_at: str | None + last_success_at: str | None + next_refresh_at: str | None + exception: str | None + +@final +@dataclass(frozen=True) +class AdapterModelRealizationRequest: + logical_name: str + target_relation_name: str + source_relation_name: str + resolved_query: str + resolved_database_template: str + columns: tuple[AdapterColumn, ...] + engine: str + order_by: tuple[str, ...] + partition_by: str | None = None + ttl: str | None = None + settings: tuple[tuple[str, str], ...] = () + refresh: str | None = None + append: bool = True + dead_letter_query: str | None = None + dead_letter_database_template: str | None = None + +@final +@dataclass(frozen=True) +class AdapterViewRealizationRequest: + logical_name: str + target_relation_name: str + resolved_query: str + resolved_database_template: str + +@final +@dataclass(frozen=True) +class AdapterModelRealization: + relation_name: str + resources: tuple[AdapterTable | AdapterMaterializedView | AdapterView, ...] + +@final +@dataclass(frozen=True) +class AdapterStableView: + name: str + target_relation_name: str + +@final +@dataclass(frozen=True) +class AdapterStableBinding: + database: str + logical_name: str + physical_name: str + +@final +@dataclass(frozen=True) +class AdapterStableBindingRemoval: + database: str + logical_name: str + +@final +@dataclass(frozen=True) +class AdapterBindingReplacementRequest: + bindings: tuple[AdapterStableBinding, ...] + removals: tuple[AdapterStableBindingRemoval, ...] = () + +@final +@dataclass(frozen=True) +class AdapterRelationCleanupRequest: + database: str + relation_names: tuple[str, ...] + +@final +@dataclass(frozen=True) +class AdapterTargetMutationLock: + database: str + owner_id: str + +@final +@dataclass(frozen=True) +class AdapterReadinessRootRequest: + database: str + logical_name: str + staged_relation_name: str + active_exists: bool + +@final +@dataclass(frozen=True) +class AdapterReadinessRequest: + roots: tuple[AdapterReadinessRootRequest, ...] + +@final +@dataclass(frozen=True) +class AdapterReadinessOffsetSummary: + active_partition_count: int + staged_partition_count: int + partitions_compared: int + missing_staged_partition_count: int + missing_freshness_partition_count: int + lagging_partition_count: int + max_offset_gap: int + average_offset_gap: float + lag_boundary_column: str | None + max_lag_seconds: float | None + average_lag_seconds: float | None + +@final +@dataclass(frozen=True) +class AdapterReadinessScalarSummary: + active_min_value: str | None + active_max_value: str | None + staged_min_value: str | None + staged_max_value: str | None + lag_seconds: float | None + +@final +@dataclass(frozen=True) +class AdapterReadinessRootObservation: + root: AdapterReadinessRootRequest + staged_exists: bool + active_row_count: int | None + staged_row_count: int | None + replay_source_name: str | None + replay_source_row_count: int | None + replay_boundary_mode: AdapterReplayBoundaryMode | None + offset_summary: AdapterReadinessOffsetSummary | None + scalar_summary: AdapterReadinessScalarSummary | None + +@final +@dataclass(frozen=True) +class AdapterPhysicalRelationMapping: + logical_name: str + physical_name: str + +@final +@dataclass(frozen=True) +class AdapterReplayQuery: + query: str + physical_relation_mappings: tuple[AdapterPhysicalRelationMapping, ...] + aggregate_semantics: bool + +@final +@dataclass(frozen=True) +class AdapterReplayBoundary: + boundary_key: str + cutoff_value: str + cutoff_inclusive: bool + partition_value: str | None = None + +@final +@dataclass(frozen=True) +class AdapterReplayColumns: + partition: str + offset: str + timestamp: str + landed_at: str + cursor: str + +@final +@dataclass(frozen=True) +class AdapterReplayWindow: + lower_bound_mode: AdapterReplayLowerBoundMode + lower_bound_inclusive: bool + boundary_time: str + forced_start_time: str | None + lookback_seconds: int | None + +@final +@dataclass(frozen=True) +class AdapterReplayRelations: + root: str + source: str + anchor: str + target: str + +@final +@dataclass(frozen=True) +class AdapterReplayRequest: + mode: AdapterReplayBoundaryMode + database: str + relations: AdapterReplayRelations + replay_query: AdapterReplayQuery + boundaries: tuple[AdapterReplayBoundary, ...] + columns: AdapterReplayColumns + window: AdapterReplayWindow + seed_mode: AdapterReplaySeedMode + target_column_names: tuple[str, ...] + settings: tuple[tuple[str, str], ...] = () + filter_boundaries_at_source: bool = False + dedupe_offsets: bool = False + +@final +@dataclass(frozen=True) +class AdapterReplayCoverageRequest: + replay: AdapterReplayRequest + boundary_column_type: str | None + count_anchor_rows: bool = False + +@final +@dataclass(frozen=True) +class AdapterReplayLowerBound: + value: str + partition_value: str | None = None + +@final +@dataclass(frozen=True) +class AdapterCapturedReplayRequest: + replay: AdapterReplayRequest + boundary_column_type: str | None + lower_bounds: tuple[AdapterReplayLowerBound, ...] + +@final +@dataclass(frozen=True) +class AdapterDirectFingerprintRecord: + fingerprint_id: str + logical_model_identity: str + definition_sql: str + definition_hash: str + identity_metadata: str + workflow_id: str + tool_version: str + applied_at: str | None = None + +@final +@dataclass(frozen=True) +class AdapterDirectFingerprintSnapshot: + status: AdapterOptionalStateStatus | str + baselines: tuple[AdapterDirectFingerprintRecord, ...] + warning: str | None = None + +@final +@dataclass(frozen=True) +class AdapterManifestResource: + pipeline_name: str + logical_type: str + logical_name: str + resource_role: str + resource_database: str + resource_name: str + resource_kind: str + +@final +@dataclass(frozen=True) +class AdapterManifest: + manifest_id: str + invocation_id: str + project_identity: str + target_name: str + target_database: str + is_production: bool + project_revision: str | None + manifest_fingerprint: str + manifest_version: int + pipelines: tuple[str, ...] + resources: tuple[AdapterManifestResource, ...] + tool_version: str + published_at: str + +@final +@dataclass(frozen=True) +class AdapterManifestSnapshot: + status: AdapterOptionalStateStatus | str + manifests: tuple[AdapterManifest, ...] + warning: str | None = None + +@final +@dataclass(frozen=True) +class AdapterDeploymentReplayRequest: + replay: AdapterReplayRequest + metadata_database: str + deployment_id: str + boundary_column_type: str | None + active_relation_name: str + active_column_names: tuple[str, ...] + anchor_column_names: tuple[str, ...] + +@final +@dataclass(frozen=True) +class AdapterMetadataObjectKey: + database: str | None + object_type: str + name: str + +@final +@dataclass(frozen=True) +class AdapterPreparedObjectMapping: + logical_key: AdapterMetadataObjectKey + physical_name: str + logical_model_name: str + +@final +@dataclass(frozen=True) +class AdapterObjectStateRecord: + deployment_id: str + key: AdapterMetadataObjectKey + normalized_fingerprint: str + normalized_query: str | None + recorded_at: str + observation_id: str = "" + state_kind: str = "deployment" + physical_database_name: str | None = None + physical_relation_name: str | None = None + logical_model_database: str | None = None + logical_model_name: str | None = None + is_selected_root: bool = False + +@final +@dataclass(frozen=True) +class AdapterDeploymentRecord: + deployment_id: str + created_at: str + status: str + replay_lineage_mode: str + selected_root_keys: tuple[AdapterMetadataObjectKey, ...] + warning_codes: tuple[str, ...] + prepared_object_mappings: tuple[AdapterPreparedObjectMapping, ...] + workflow_fingerprint: str = "" + boundary_time: str | None = None + tool_version: str = "" + +@final +@dataclass(frozen=True) +class AdapterDeploymentWatermarkRecord: + deployment_id: str + root_key: AdapterMetadataObjectKey + anchor_key: AdapterMetadataObjectKey + boundary_key: str + cutoff_value: str + lower_value: str | None = None + cutoff_inclusive: bool = True + captured_at: str = "1970-01-01 00:00:00.000" + +@final +@dataclass(frozen=True) +class AdapterPublishEventRecord: + deployment_id: str + published_at: str + logical_view_names: tuple[str, ...] + publication_id: str = "" + bindings: tuple[AdapterStableBinding, ...] = () + operation: str = "promote" + previous_deployment_id: str | None = None + +@final +@dataclass(frozen=True) +class AdapterInvocationRecord: + invocation_id: str + project_identity: str + target_identity: str + command: str + mode: str | None + outcome: str + exit_code: int + materialized_outcome: str | None + deployment_id: str | None + workflow_id: str | None + selected_node_count: int + started_at: str + completed_at: str + duration_ms: int + error_message: str | None + summary_json: str + tool_version: str + artifact_project_dir: str | None = None + +@final +@dataclass(frozen=True) +class AdapterNodeResultRecord: + result_id: str + invocation_id: str + node_kind: str + node_name: str + binding_key: str + definition_fingerprint: str + execution_fingerprint: str + target_identity: str + trigger: str + scheduled_for: str | None + cadence_seconds: int | None + warmup_seconds: int + status: str + severity: str | None + failure_count: int + completed_at: str + payload_json: str + error_message: str | None + +@final +@dataclass(frozen=True) +class AdapterRunEventRecord: + invocation_id: str + sequence: int + event_kind: str + step_id: str | None + phase: str | None + payload_json: str + +@final +@dataclass(frozen=True) +class AdapterRunStatementRecord: + invocation_id: str + statement_sequence: int + step_id: str + phase: str + intent: str + sql: str + sql_sha256: str + workflow_sha256: str + workflow_revision: int + +@final +@dataclass(frozen=True) +class AdapterSensorCheckpointRecord: + sensor_name: str + source: str + position_completed_at: str + position_result_id: str + +@final +@dataclass(frozen=True) +class AdapterSensorTickRecord: + tick_id: str + sensor_name: str + definition_fingerprint: str + kind: str + event_id: str | None + event_kind: str | None + attempt: int + status: str + started_at: str + completed_at: str | None + error_message: str | None + skip_reason: str | None + cursor: str | None + +@final +@dataclass(frozen=True) +class AdapterSensorStepRecord: + sensor_name: str + event_id: str + step_key: str + policy: str + status: str + attempt: int + result_json: str | None + error_message: str | None + +@final +@dataclass(frozen=True) +class AdapterSensorOverrideRecord: + override_id: str + sensor_name: str + status: str + actor: str | None + +@final +@dataclass(frozen=True) +class AdapterSensorLeaseRecord: + lease_name: str + owner_id: str + acquired_at: str + expires_at: str + +@final +@dataclass(frozen=True) +class AdapterSensorState: + checkpoints: tuple[AdapterSensorCheckpointRecord, ...] = () + ticks: tuple[AdapterSensorTickRecord, ...] = () + steps: tuple[AdapterSensorStepRecord, ...] = () + overrides: tuple[AdapterSensorOverrideRecord, ...] = () + leases: tuple[AdapterSensorLeaseRecord, ...] = () + +@final +@dataclass(frozen=True) +class AdapterCurrentQualityNode: + node_kind: str + node_name: str + binding_key: str + definition_fingerprint: str + execution_fingerprint: str + cadence_seconds: int | None = None + warmup_seconds: int = 0 + +@final +@dataclass(frozen=True) +class AdapterDeploymentInventory: + deployments: tuple[AdapterDeploymentRecord, ...] + publish_events: tuple[AdapterPublishEventRecord, ...] + +@final +@dataclass(frozen=True) +class AdapterMetadataState: + object_states: tuple[AdapterObjectStateRecord, ...] + deployments: tuple[AdapterDeploymentRecord, ...] + deployment_watermarks: tuple[AdapterDeploymentWatermarkRecord, ...] + publish_events: tuple[AdapterPublishEventRecord, ...] + invocations: tuple[AdapterInvocationRecord, ...] = () + node_results: tuple[AdapterNodeResultRecord, ...] = () + +@final +@dataclass(frozen=True) +class CatalogIdentity: + adapter: AdapterIdentity + database: str + +@final +@dataclass(frozen=True) +class CatalogColumn: + name: str + type: str + default_expression: str | None = None + +@final +@dataclass(frozen=True) +class CatalogRelation: + name: str + engine: str + columns: tuple[CatalogColumn, ...] + full_engine: str | None = None + order_by: tuple[str, ...] = () + partition_by: str | None = None + ttl: str | None = None + settings: tuple[tuple[str, str], ...] = () + definition_sql: str | None = None + query_sql: str | None = None + source_relation_names: tuple[str, ...] = () + target_relation_name: str | None = None + stable_binding_name: str | None = None + ownership_generation: str | None = None + @property + def storage_engine(self) -> str: ... + @property + def source_relation_name(self) -> str | None: ... + +@final +@dataclass(frozen=True) +class CatalogSnapshot: + identity: CatalogIdentity + warehouse_timezone: str + relations: tuple[CatalogRelation, ...] + def relation(self, name: str) -> CatalogRelation | None: ... + def relation_names(self) -> frozenset[str]: ... + +@final +@dataclass(frozen=True) +class InspectedActiveTableBinding: + database: str + logical_name: str + physical_name: str + +@final +@dataclass(frozen=True) +class InspectedPhysicalTableCandidate: + database: str + logical_name: str + physical_name: str + object_type: str = "table" + +@final +@dataclass(frozen=True) +class InspectedManagedTableState: + active_bindings: tuple[InspectedActiveTableBinding, ...] + physical_candidates: tuple[InspectedPhysicalTableCandidate, ...] + +@final +@dataclass(frozen=True, repr=False) +class AdapterConnectionConfig: + host: str + port: int + username: str + password: str + database: str | None = None + settings: tuple[tuple[str, str], ...] = () + def __repr__(self) -> str: ... + +@final +@dataclass(frozen=True) +class AdapterQueryResult: + rows: tuple[tuple[object, ...], ...] + column_names: tuple[str, ...] = () + def named_rows(self) -> tuple[Mapping[str, object], ...]: ... + +@final +@dataclass(frozen=True) +class AdapterWarehouseDisk: + name: str + path: str | None + disk_type: str | None + total_bytes: int | None + free_bytes: int | None + unreserved_bytes: int | None + keep_free_bytes: int | None + status: AdapterWarehouseHealthStatus | str + +@final +@dataclass(frozen=True) +class AdapterWarehouseMemory: + resident_bytes: int | None + host_total_bytes: int | None + cgroup_used_bytes: int | None + cgroup_limit_bytes: int | None + basis: str + pressure_fraction: float | None + +@final +@dataclass(frozen=True) +class AdapterWarehouseActivity: + active_queries: int | None + active_merges: int | None + incomplete_mutations: int | None + +@final +@dataclass(frozen=True) +class AdapterWarehouseStuckKafkaConsumer: + table: str + stuck_seconds: int | None + recent_errors: int + latest_error: str + +@final +@dataclass(frozen=True) +class AdapterWarehouseKafkaConsumers: + configured_tables: int + materialized_tables: int + materialized_table_names: tuple[str, ...] + polling_tables: int + exception_tables: int + stuck_consumers: tuple[AdapterWarehouseStuckKafkaConsumer, ...] = () + +@final +@dataclass(frozen=True) +class AdapterWarehouseDeadLetterModel: + model_name: str + row_count: int + latest_failed_columns: tuple[str, ...] + latest_payload: str + latest_quarantined_at: str + +@final +@dataclass(frozen=True) +class AdapterWarehouseTable: + name: str + rows: int | None + bytes_on_disk: int | None + active_parts: int | None + +@final +@dataclass(frozen=True) +class AdapterWarehouseHealth: + availability: AdapterWarehouseHealthAvailability | str + status: AdapterWarehouseHealthStatus | str + version: str | None + uptime_seconds: int | None + disks: tuple[AdapterWarehouseDisk, ...] + inode_total: int | None + inode_free: int | None + inode_status: AdapterWarehouseHealthStatus | str + memory: AdapterWarehouseMemory | None + activity: AdapterWarehouseActivity | None + tables: tuple[AdapterWarehouseTable, ...] | None + collection_duration_ms: int + kafka_consumers: AdapterWarehouseKafkaConsumers | None = None + dead_letters: tuple[AdapterWarehouseDeadLetterModel, ...] | None = None + capacity_warning_fraction: float | None = None + capacity_critical_fraction: float | None = None + warnings: tuple[str, ...] = () + stale: bool = False + measured_at: str | None = None + +@final +@dataclass(frozen=True) +class AdapterMutationResult: + written_rows: int | None = None + +@final +@dataclass(frozen=True) +class AdapterStatementProgress: + elapsed_seconds: float + read_rows: int + read_bytes: int + total_rows_approx: int + memory_usage_bytes: int + settings: tuple[tuple[str, str], ...] = () + +@final +@dataclass(frozen=True) +class AdapterQueryCancellation: + query_id: str + supported: bool + query_found: bool + termination_confirmed: bool + detail: str | None = None + +@final +@dataclass(frozen=True) +class AdapterReplayOffsetRange: + partition: int + lower_offset: int + upper_offset: int + +@final +@dataclass(frozen=True) +class AdapterReplayOffsetProgressRequest: + database: str + relation: str + partition_column: str + offset_column: str + ranges: tuple[AdapterReplayOffsetRange, ...] + +@final +@dataclass(frozen=True) +class AdapterReplayOffsetFrontier: + partition: int + completed_offset: int + type _Span = tuple[int, int, int, int, int, int] class SqlScanError(ValueError): diff --git a/src/sqltide/adapter/models.py b/src/sqltide/adapter/models.py index b3e61348..af14023c 100644 --- a/src/sqltide/adapter/models.py +++ b/src/sqltide/adapter/models.py @@ -1,1080 +1,94 @@ """Neutral adapter identity, connection, and result models.""" -from __future__ import annotations - -from collections.abc import Mapping -from dataclasses import dataclass - -from sqltide.adapter.constants import REDACTED_SECRET_PLACEHOLDER -from sqltide.adapter.exceptions import AdapterResultError -from sqltide.adapter.types import ( - AdapterOptionalStateStatus, - AdapterReplayBoundaryMode, - AdapterReplayLowerBoundMode, - AdapterReplaySeedMode, - AdapterWarehouseHealthAvailability, - AdapterWarehouseHealthStatus, +from sqltide._native import ( + AdapterAdoptedSourceRealizationRequest as AdapterAdoptedSourceRealizationRequest, ) - - -@dataclass(frozen=True) -class AdapterIdentity: - """The registered name of one adapter implementation.""" - - name: str - - -@dataclass(frozen=True) -class AdapterCapabilities: - """Capabilities implemented by one adapter.""" - - virtual_environments: bool - managed_source_kinds: frozenset[str] - replay_boundary_modes: frozenset[AdapterReplayBoundaryMode] - history_prefix_seed: bool - stable_logical_bindings: bool - per_relation_atomic_replace: bool - graph_atomic_publish: bool - set_difference_comparison: bool - direct_rebuild: bool - - -@dataclass(frozen=True) -class AdapterReplayCoverageRange: - """One retained lineage range required to reproduce a direct target.""" - - driving_input_relation_name: str - replay_boundary_mode: AdapterReplayBoundaryMode | str - boundary_key: str - source_partition_column_name: str | None - source_position_column_name: str - source_timestamp_column_name: str | None - lower_value: str - upper_value: str - - def __post_init__(self) -> None: - object.__setattr__( - self, "replay_boundary_mode", AdapterReplayBoundaryMode(self.replay_boundary_mode) - ) - - -@dataclass(frozen=True) -class AdapterSetDifferenceTarget: - """One neutral actual/expected bag comparison requested from an adapter.""" - - name: str - column_names: tuple[str, ...] - ctes: tuple[tuple[str, str], ...] - actual_query: str - expected_query: str | None - - -@dataclass(frozen=True) -class AdapterSetDifferenceComparisonRequest: - """One executable statement containing every SQL-test comparison target.""" - - targets: tuple[AdapterSetDifferenceTarget, ...] - - -@dataclass(frozen=True) -class AdapterColumn: - """One column in a neutral adapter resource request.""" - - name: str - type: str - default_expression: str | None = None - - -@dataclass(frozen=True) -class AdapterManagedSource: - """A managed streaming source to realize in a warehouse.""" - - source_kind: str - name: str - columns: tuple[AdapterColumn, ...] - broker_list: str - topic: str - consumer_group: str - format: str - settings: tuple[tuple[str, str], ...] = () - naming_macro_fingerprint: str | None = None - - -@dataclass(frozen=True) -class AdapterTable: - """A managed table to realize in a warehouse.""" - - name: str - columns: tuple[AdapterColumn, ...] - engine: str - order_by: tuple[str, ...] - partition_by: str | None = None - ttl: str | None = None - settings: tuple[tuple[str, str], ...] = () - - -@dataclass(frozen=True) -class AdapterMaterializedView: - """A materialized view to realize between warehouse relations.""" - - name: str - source_relation_name: str - target_relation_name: str - query: str - database_template: str - refresh: str | None = None - append: bool = True - dead_letters: bool = False - - -@dataclass(frozen=True) -class AdapterView: - """An ordinary query view to realize in a warehouse.""" - - name: str - query: str - database_template: str - - -@dataclass(frozen=True) -class AdapterManagedSourceRealizationRequest: - """Logical managed-source fields needed for adapter realization.""" - - logical_name: str - source_kind: str - broker_list: str - topic: str - consumer_group: str | None - format: str - project_name: str | None = None - target_name: str | None = None - ttl: str | None = None - settings: tuple[tuple[str, str], ...] = () - naming_macro_fingerprint: str | None = None - - -@dataclass(frozen=True) -class AdapterAdoptedSourceRealizationRequest: - """Logical adopted-source fields needed for adapter realization.""" - - logical_name: str - relation_name: str - - -@dataclass(frozen=True) -class AdapterSourceRealization: - """One logical source mapped to its adapter relation and resources.""" - - relation_name: str - resources: tuple[AdapterManagedSource | AdapterTable | AdapterMaterializedView, ...] - - -@dataclass(frozen=True) -class AdapterRefreshState: - """The warehouse-reported refresh state of one scheduled relation.""" - - view_name: str - status: str - last_refresh_at: str | None - last_success_at: str | None - next_refresh_at: str | None - exception: str | None - - -@dataclass(frozen=True) -class AdapterModelRealizationRequest: - """One semantically compiled logical model ready for adapter realization.""" - - logical_name: str - target_relation_name: str - source_relation_name: str - resolved_query: str - resolved_database_template: str - columns: tuple[AdapterColumn, ...] - engine: str - order_by: tuple[str, ...] - partition_by: str | None = None - ttl: str | None = None - settings: tuple[tuple[str, str], ...] = () - refresh: str | None = None - append: bool = True - dead_letter_query: str | None = None - dead_letter_database_template: str | None = None - - -@dataclass(frozen=True) -class AdapterViewRealizationRequest: - """One compiled query-only view ready for adapter realization.""" - - logical_name: str - target_relation_name: str - resolved_query: str - resolved_database_template: str - - -@dataclass(frozen=True) -class AdapterModelRealization: - """One logical model mapped to its adapter relation and resources.""" - - relation_name: str - resources: tuple[AdapterTable | AdapterMaterializedView | AdapterView, ...] - - -@dataclass(frozen=True) -class AdapterStableView: - """A stable logical view to bind to one physical relation.""" - - name: str - target_relation_name: str - - -@dataclass(frozen=True) -class AdapterStableBinding: - """One logical relation bound to one physical relation.""" - - database: str - logical_name: str - physical_name: str - - -@dataclass(frozen=True) -class AdapterStableBindingRemoval: - """One obsolete framework-owned stable binding to remove.""" - - database: str - logical_name: str - - -@dataclass(frozen=True) -class AdapterBindingReplacementRequest: - """A set of stable bindings to replace during one publish operation.""" - - bindings: tuple[AdapterStableBinding, ...] - removals: tuple[AdapterStableBindingRemoval, ...] = () - - -@dataclass(frozen=True) -class AdapterRelationCleanupRequest: - """Physical relations to remove from one database.""" - - database: str - relation_names: tuple[str, ...] - - -@dataclass(frozen=True) -class AdapterTargetMutationLock: - """Exclusive warehouse-backed ownership of one target mutation slot.""" - - database: str - owner_id: str - - -@dataclass(frozen=True) -class AdapterReadinessRootRequest: - """One staged root whose live and candidate relations must be compared.""" - - database: str - logical_name: str - staged_relation_name: str - active_exists: bool - - -@dataclass(frozen=True) -class AdapterReadinessRequest: - """A mode-neutral readiness comparison request.""" - - roots: tuple[AdapterReadinessRootRequest, ...] - - -@dataclass(frozen=True) -class AdapterReadinessOffsetSummary: - """Adapter-observed partition and lag metrics for offset replay.""" - - active_partition_count: int - staged_partition_count: int - partitions_compared: int - missing_staged_partition_count: int - missing_freshness_partition_count: int - lagging_partition_count: int - max_offset_gap: int - average_offset_gap: float - lag_boundary_column: str | None - max_lag_seconds: float | None - average_lag_seconds: float | None - - -@dataclass(frozen=True) -class AdapterReadinessScalarSummary: - """Adapter-observed range and lag metrics for scalar replay.""" - - active_min_value: str | None - active_max_value: str | None - staged_min_value: str | None - staged_max_value: str | None - lag_seconds: float | None - - -@dataclass(frozen=True) -class AdapterReadinessRootObservation: - """Warehouse observations for one requested staged root.""" - - root: AdapterReadinessRootRequest - staged_exists: bool - active_row_count: int | None - staged_row_count: int | None - replay_source_name: str | None - replay_source_row_count: int | None - replay_boundary_mode: AdapterReplayBoundaryMode | None - offset_summary: AdapterReadinessOffsetSummary | None - scalar_summary: AdapterReadinessScalarSummary | None - - -@dataclass(frozen=True) -class AdapterPhysicalRelationMapping: - """A logical relation mapped to its replay-time physical relation.""" - - logical_name: str - physical_name: str - - -@dataclass(frozen=True) -class AdapterReplayQuery: - """A compiler-analyzed query ready for adapter replay realization.""" - - query: str - physical_relation_mappings: tuple[AdapterPhysicalRelationMapping, ...] - aggregate_semantics: bool - - -@dataclass(frozen=True) -class AdapterReplayBoundary: - """One inclusive replay cutoff, optionally scoped to a partition.""" - - boundary_key: str - cutoff_value: str - cutoff_inclusive: bool - partition_value: str | None = None - - -@dataclass(frozen=True) -class AdapterReplayColumns: - """Physical boundary columns used to read an anchor relation.""" - - partition: str - offset: str - timestamp: str - landed_at: str - cursor: str - - -@dataclass(frozen=True) -class AdapterReplayWindow: - """The lower-bound policy for one replay root.""" - - lower_bound_mode: AdapterReplayLowerBoundMode - lower_bound_inclusive: bool - boundary_time: str - forced_start_time: str | None - lookback_seconds: int | None - - -@dataclass(frozen=True) -class AdapterReplayRelations: - """Logical and physical relations participating in one replay root.""" - - root: str - source: str - anchor: str - target: str - - -@dataclass(frozen=True) -class AdapterReplayRequest: - """A mode-neutral request to seed and replay one rebuild root.""" - - mode: AdapterReplayBoundaryMode - database: str - relations: AdapterReplayRelations - replay_query: AdapterReplayQuery - boundaries: tuple[AdapterReplayBoundary, ...] - columns: AdapterReplayColumns - window: AdapterReplayWindow - seed_mode: AdapterReplaySeedMode - target_column_names: tuple[str, ...] - settings: tuple[tuple[str, str], ...] = () - filter_boundaries_at_source: bool = False - dedupe_offsets: bool = False - - -@dataclass(frozen=True) -class AdapterReplayCoverageRequest: - """A request for the retained physical ranges selected by one replay window.""" - - replay: AdapterReplayRequest - boundary_column_type: str | None - count_anchor_rows: bool = False - - -@dataclass(frozen=True) -class AdapterReplayLowerBound: - """One captured replay lower bound in adapter-neutral string form.""" - - value: str - partition_value: str | None = None - - -@dataclass(frozen=True) -class AdapterCapturedReplayRequest: - """A replay rendered from boundaries captured by the current process.""" - - replay: AdapterReplayRequest - boundary_column_type: str | None - lower_bounds: tuple[AdapterReplayLowerBound, ...] - - -@dataclass(frozen=True) -class AdapterDirectFingerprintRecord: - """One logical SQL baseline recorded after direct materialization.""" - - fingerprint_id: str - logical_model_identity: str - definition_sql: str - definition_hash: str - identity_metadata: str - workflow_id: str - tool_version: str - applied_at: str | None = None - - -@dataclass(frozen=True) -class AdapterDirectFingerprintSnapshot: - """Explicit availability and latest logical direct SQL baselines.""" - - status: AdapterOptionalStateStatus | str - baselines: tuple[AdapterDirectFingerprintRecord, ...] - warning: str | None = None - - def __post_init__(self) -> None: - object.__setattr__(self, "status", AdapterOptionalStateStatus(self.status)) - - -@dataclass(frozen=True) -class AdapterManifestResource: - """One physical resource recorded in a published project manifest.""" - - pipeline_name: str - logical_type: str - logical_name: str - resource_role: str - resource_database: str - resource_name: str - resource_kind: str - - -@dataclass(frozen=True) -class AdapterManifest: - """One complete, immutable publication manifest.""" - - manifest_id: str - invocation_id: str - project_identity: str - target_name: str - target_database: str - is_production: bool - project_revision: str | None - manifest_fingerprint: str - manifest_version: int - pipelines: tuple[str, ...] - resources: tuple[AdapterManifestResource, ...] - tool_version: str - published_at: str - - -@dataclass(frozen=True) -class AdapterManifestSnapshot: - """Availability and complete publication history for one project target.""" - - status: AdapterOptionalStateStatus | str - manifests: tuple[AdapterManifest, ...] - warning: str | None = None - - def __post_init__(self) -> None: - object.__setattr__(self, "status", AdapterOptionalStateStatus(self.status)) - - -@dataclass(frozen=True) -class AdapterDeploymentReplayRequest: - """A replay whose dynamic boundaries live in deployment watermark metadata.""" - - replay: AdapterReplayRequest - metadata_database: str - deployment_id: str - boundary_column_type: str | None - active_relation_name: str - active_column_names: tuple[str, ...] - anchor_column_names: tuple[str, ...] - - -@dataclass(frozen=True) -class AdapterMetadataObjectKey: - """A logical object identity stored in framework metadata.""" - - database: str | None - object_type: str - name: str - - -@dataclass(frozen=True) -class AdapterPreparedObjectMapping: - """A persisted logical-to-physical object mapping.""" - - logical_key: AdapterMetadataObjectKey - physical_name: str - logical_model_name: str - - -@dataclass(frozen=True) -class AdapterObjectStateRecord: - """One adapter-neutral object-state metadata record.""" - - deployment_id: str - key: AdapterMetadataObjectKey - normalized_fingerprint: str - normalized_query: str | None - recorded_at: str - observation_id: str = "" - state_kind: str = "deployment" - physical_database_name: str | None = None - physical_relation_name: str | None = None - logical_model_database: str | None = None - logical_model_name: str | None = None - is_selected_root: bool = False - - -@dataclass(frozen=True) -class AdapterDeploymentRecord: - """One adapter-neutral staged deployment record.""" - - deployment_id: str - created_at: str - status: str - replay_lineage_mode: str - selected_root_keys: tuple[AdapterMetadataObjectKey, ...] - warning_codes: tuple[str, ...] - prepared_object_mappings: tuple[AdapterPreparedObjectMapping, ...] - workflow_fingerprint: str = "" - boundary_time: str | None = None - tool_version: str = "" - - -@dataclass(frozen=True) -class AdapterDeploymentWatermarkRecord: - """One adapter-neutral replay watermark record.""" - - deployment_id: str - root_key: AdapterMetadataObjectKey - anchor_key: AdapterMetadataObjectKey - boundary_key: str - cutoff_value: str - lower_value: str | None = None - cutoff_inclusive: bool = True - captured_at: str = "1970-01-01 00:00:00.000" - - -@dataclass(frozen=True) -class AdapterPublishEventRecord: - """One adapter-neutral publish-history record.""" - - deployment_id: str - published_at: str - logical_view_names: tuple[str, ...] - publication_id: str = "" - bindings: tuple[AdapterStableBinding, ...] = () - operation: str = "promote" - previous_deployment_id: str | None = None - - -@dataclass(frozen=True) -class AdapterInvocationRecord: - """One immutable terminal CLI invocation observation.""" - - invocation_id: str - project_identity: str - target_identity: str - command: str - mode: str | None - outcome: str - exit_code: int - materialized_outcome: str | None - deployment_id: str | None - workflow_id: str | None - selected_node_count: int - started_at: str - completed_at: str - duration_ms: int - error_message: str | None - summary_json: str - tool_version: str - artifact_project_dir: str | None = None - - -@dataclass(frozen=True) -class AdapterNodeResultRecord: - """One immutable terminal audit or test result observation.""" - - result_id: str - invocation_id: str - node_kind: str - node_name: str - binding_key: str - definition_fingerprint: str - execution_fingerprint: str - target_identity: str - trigger: str - scheduled_for: str | None - cadence_seconds: int | None - warmup_seconds: int - status: str - severity: str | None - failure_count: int - completed_at: str - payload_json: str - error_message: str | None - - -@dataclass(frozen=True) -class AdapterRunEventRecord: - """One step-granular workflow event streamed while a run executes.""" - - invocation_id: str - sequence: int - event_kind: str - step_id: str | None - phase: str | None - payload_json: str - - -@dataclass(frozen=True) -class AdapterRunStatementRecord: - """One display-safe warehouse statement durably associated with a run.""" - - invocation_id: str - statement_sequence: int - step_id: str - phase: str - intent: str - sql: str - sql_sha256: str - workflow_sha256: str - workflow_revision: int - - -@dataclass(frozen=True) -class AdapterSensorCheckpointRecord: - """One append-only latest-wins sensor stream position.""" - - sensor_name: str - source: str - position_completed_at: str - position_result_id: str - - -@dataclass(frozen=True) -class AdapterSensorTickRecord: - """One persisted sensor evaluation attempt.""" - - tick_id: str - sensor_name: str - definition_fingerprint: str - kind: str - event_id: str | None - event_kind: str | None - attempt: int - status: str - started_at: str - completed_at: str | None - error_message: str | None - skip_reason: str | None - cursor: str | None - - -@dataclass(frozen=True) -class AdapterSensorStepRecord: - """One durable step marker keyed by sensor, event, and step key.""" - - sensor_name: str - event_id: str - step_key: str - policy: str - status: str - attempt: int - result_json: str | None - error_message: str | None - - -@dataclass(frozen=True) -class AdapterSensorOverrideRecord: - """One append-only operator override of a sensor's declared status.""" - - override_id: str - sensor_name: str - status: str - actor: str | None - - -@dataclass(frozen=True) -class AdapterSensorLeaseRecord: - """One warehouse-time dispatch lease claim.""" - - lease_name: str - owner_id: str - acquired_at: str - expires_at: str - - -@dataclass(frozen=True) -class AdapterSensorState: - """Sensor state records persisted in one batch.""" - - checkpoints: tuple[AdapterSensorCheckpointRecord, ...] = () - ticks: tuple[AdapterSensorTickRecord, ...] = () - steps: tuple[AdapterSensorStepRecord, ...] = () - overrides: tuple[AdapterSensorOverrideRecord, ...] = () - leases: tuple[AdapterSensorLeaseRecord, ...] = () - - -@dataclass(frozen=True) -class AdapterCurrentQualityNode: - """One current manifest node joined to persisted terminal history.""" - - node_kind: str - node_name: str - binding_key: str - definition_fingerprint: str - execution_fingerprint: str - cadence_seconds: int | None = None - warmup_seconds: int = 0 - - -@dataclass(frozen=True) -class AdapterDeploymentInventory: - """Persisted deployments and publish events used for lifecycle cleanup.""" - - deployments: tuple[AdapterDeploymentRecord, ...] - publish_events: tuple[AdapterPublishEventRecord, ...] - - -@dataclass(frozen=True) -class AdapterMetadataState: - """A batch of framework metadata records for adapter persistence.""" - - object_states: tuple[AdapterObjectStateRecord, ...] - deployments: tuple[AdapterDeploymentRecord, ...] - deployment_watermarks: tuple[AdapterDeploymentWatermarkRecord, ...] - publish_events: tuple[AdapterPublishEventRecord, ...] - invocations: tuple[AdapterInvocationRecord, ...] = () - node_results: tuple[AdapterNodeResultRecord, ...] = () - - -@dataclass(frozen=True) -class CatalogIdentity: - """Stable identity of one adapter/database catalog observation.""" - - adapter: AdapterIdentity - database: str - - -@dataclass(frozen=True) -class CatalogColumn: - """One warehouse relation column observed by an adapter.""" - - name: str - type: str - default_expression: str | None = None - - -@dataclass(frozen=True) -class CatalogRelation: - """One immutable warehouse relation observation.""" - - name: str - engine: str - columns: tuple[CatalogColumn, ...] - full_engine: str | None = None - order_by: tuple[str, ...] = () - partition_by: str | None = None - ttl: str | None = None - settings: tuple[tuple[str, str], ...] = () - definition_sql: str | None = None - query_sql: str | None = None - source_relation_names: tuple[str, ...] = () - target_relation_name: str | None = None - stable_binding_name: str | None = None - ownership_generation: str | None = None - - @property - def storage_engine(self) -> str: - """Return the engine including storage arguments when available.""" - - return self.full_engine or self.engine - - @property - def source_relation_name(self) -> str | None: - """Return the first physical source when one exists.""" - - return self.source_relation_names[0] if self.source_relation_names else None - - -@dataclass(frozen=True) -class CatalogSnapshot: - """One immutable point-in-time catalog for an adapter and database.""" - - identity: CatalogIdentity - warehouse_timezone: str - relations: tuple[CatalogRelation, ...] - - def relation(self, name: str) -> CatalogRelation | None: - """Return one relation by unqualified name when it exists.""" - - return next((relation for relation in self.relations if relation.name == name), None) - - def relation_names(self) -> frozenset[str]: - """Return every observed unqualified relation name.""" - - return frozenset(relation.name for relation in self.relations) - - -@dataclass(frozen=True) -class InspectedActiveTableBinding: - """A stable logical view pointing at an active physical table.""" - - database: str - logical_name: str - physical_name: str - - -@dataclass(frozen=True) -class InspectedPhysicalTableCandidate: - """A deployment-suffixed physical model candidate for a logical root.""" - - database: str - logical_name: str - physical_name: str - object_type: str = "table" - - -@dataclass(frozen=True) -class InspectedManagedTableState: - """Managed table state used for active-deployment resolution.""" - - active_bindings: tuple[InspectedActiveTableBinding, ...] - physical_candidates: tuple[InspectedPhysicalTableCandidate, ...] - - -@dataclass(frozen=True, repr=False) -class AdapterConnectionConfig: - """Resolved, format-neutral connection settings for one adapter.""" - - host: str - port: int - username: str - password: str - database: str | None = None - settings: tuple[tuple[str, str], ...] = () - - def __repr__(self) -> str: - """Render the configuration without exposing the password.""" - - return ( - f"{type(self).__name__}(host={self.host!r}, port={self.port!r}, " - f"username={self.username!r}, password={REDACTED_SECRET_PLACEHOLDER!r}, " - f"database={self.database!r}, settings={self.settings!r})" - ) - - -@dataclass(frozen=True) -class AdapterQueryResult: - """A normalized query result returned by the adapter boundary.""" - - rows: tuple[tuple[object, ...], ...] - column_names: tuple[str, ...] = () - - def named_rows(self) -> tuple[Mapping[str, object], ...]: - """Return rows keyed by the query's column names.""" - - if not self.column_names: - if not self.rows: - return () - raise AdapterResultError("Query result does not include column names") - return tuple(dict(zip(self.column_names, row, strict=True)) for row in self.rows) - - -@dataclass(frozen=True) -class AdapterWarehouseDisk: - """Capacity and severity for one configured warehouse disk.""" - - name: str - path: str | None - disk_type: str | None - total_bytes: int | None - free_bytes: int | None - unreserved_bytes: int | None - keep_free_bytes: int | None - status: AdapterWarehouseHealthStatus | str - - def __post_init__(self) -> None: - object.__setattr__(self, "status", AdapterWarehouseHealthStatus(self.status)) - - -@dataclass(frozen=True) -class AdapterWarehouseMemory: - """Truthfully labelled warehouse process and execution-boundary memory.""" - - resident_bytes: int | None - host_total_bytes: int | None - cgroup_used_bytes: int | None - cgroup_limit_bytes: int | None - basis: str - pressure_fraction: float | None - - -@dataclass(frozen=True) -class AdapterWarehouseActivity: - """Current bounded warehouse work counts.""" - - active_queries: int | None - active_merges: int | None - incomplete_mutations: int | None - - -@dataclass(frozen=True) -class AdapterWarehouseStuckKafkaConsumer: - """A Kafka table that keeps failing without committing an offset.""" - - table: str - stuck_seconds: int | None - recent_errors: int - latest_error: str - - -@dataclass(frozen=True) -class AdapterWarehouseKafkaConsumers: - """Managed Kafka consumer health observed inside the warehouse.""" - - configured_tables: int - materialized_tables: int - materialized_table_names: tuple[str, ...] - polling_tables: int - exception_tables: int - stuck_consumers: tuple[AdapterWarehouseStuckKafkaConsumer, ...] = () - - -@dataclass(frozen=True) -class AdapterWarehouseDeadLetterModel: - """Quarantined rows of one model over the last 24 hours and its latest row.""" - - model_name: str - row_count: int - latest_failed_columns: tuple[str, ...] - latest_payload: str - latest_quarantined_at: str - - -@dataclass(frozen=True) -class AdapterWarehouseTable: - """Active physical footprint for one project-database table.""" - - name: str - rows: int | None - bytes_on_disk: int | None - active_parts: int | None - - -@dataclass(frozen=True) -class AdapterWarehouseHealth: - """Adapter-neutral point-in-time warehouse diagnostic snapshot.""" - - availability: AdapterWarehouseHealthAvailability | str - status: AdapterWarehouseHealthStatus | str - version: str | None - uptime_seconds: int | None - disks: tuple[AdapterWarehouseDisk, ...] - inode_total: int | None - inode_free: int | None - inode_status: AdapterWarehouseHealthStatus | str - memory: AdapterWarehouseMemory | None - activity: AdapterWarehouseActivity | None - tables: tuple[AdapterWarehouseTable, ...] | None - collection_duration_ms: int - kafka_consumers: AdapterWarehouseKafkaConsumers | None = None - dead_letters: tuple[AdapterWarehouseDeadLetterModel, ...] | None = None - capacity_warning_fraction: float | None = None - capacity_critical_fraction: float | None = None - warnings: tuple[str, ...] = () - stale: bool = False - measured_at: str | None = None - - def __post_init__(self) -> None: - object.__setattr__( - self, "availability", AdapterWarehouseHealthAvailability(self.availability) - ) - object.__setattr__(self, "status", AdapterWarehouseHealthStatus(self.status)) - object.__setattr__(self, "inode_status", AdapterWarehouseHealthStatus(self.inode_status)) - - -@dataclass(frozen=True) -class AdapterMutationResult: - """Warehouse-reported evidence for one executed mutation statement.""" - - written_rows: int | None = None - - -@dataclass(frozen=True) -class AdapterStatementProgress: - """Ephemeral warehouse telemetry for one currently executing statement.""" - - elapsed_seconds: float - read_rows: int - read_bytes: int - total_rows_approx: int - memory_usage_bytes: int - settings: tuple[tuple[str, str], ...] = () - - -@dataclass(frozen=True) -class AdapterQueryCancellation: - """Adapter-confirmed outcome for one exactly identified workflow query.""" - - query_id: str - supported: bool - query_found: bool - termination_confirmed: bool - detail: str | None = None - - -@dataclass(frozen=True) -class AdapterReplayOffsetRange: - """One captured partition span used only for approximate replay progress.""" - - partition: int - lower_offset: int - upper_offset: int - - -@dataclass(frozen=True) -class AdapterReplayOffsetProgressRequest: - """Target relation and immutable captured ranges for frontier observation.""" - - database: str - relation: str - partition_column: str - offset_column: str - ranges: tuple[AdapterReplayOffsetRange, ...] - - -@dataclass(frozen=True) -class AdapterReplayOffsetFrontier: - """Greatest committed target offset observed for one replay partition.""" - - partition: int - completed_offset: int +from sqltide._native import AdapterBindingReplacementRequest as AdapterBindingReplacementRequest +from sqltide._native import AdapterCapabilities as AdapterCapabilities +from sqltide._native import AdapterCapturedReplayRequest as AdapterCapturedReplayRequest +from sqltide._native import AdapterColumn as AdapterColumn +from sqltide._native import AdapterConnectionConfig as AdapterConnectionConfig +from sqltide._native import AdapterCurrentQualityNode as AdapterCurrentQualityNode +from sqltide._native import AdapterDeploymentInventory as AdapterDeploymentInventory +from sqltide._native import AdapterDeploymentRecord as AdapterDeploymentRecord +from sqltide._native import AdapterDeploymentReplayRequest as AdapterDeploymentReplayRequest +from sqltide._native import AdapterDeploymentWatermarkRecord as AdapterDeploymentWatermarkRecord +from sqltide._native import AdapterDirectFingerprintRecord as AdapterDirectFingerprintRecord +from sqltide._native import AdapterDirectFingerprintSnapshot as AdapterDirectFingerprintSnapshot +from sqltide._native import AdapterIdentity as AdapterIdentity +from sqltide._native import AdapterInvocationRecord as AdapterInvocationRecord +from sqltide._native import AdapterManagedSource as AdapterManagedSource +from sqltide._native import ( + AdapterManagedSourceRealizationRequest as AdapterManagedSourceRealizationRequest, +) +from sqltide._native import AdapterManifest as AdapterManifest +from sqltide._native import AdapterManifestResource as AdapterManifestResource +from sqltide._native import AdapterManifestSnapshot as AdapterManifestSnapshot +from sqltide._native import AdapterMaterializedView as AdapterMaterializedView +from sqltide._native import AdapterMetadataObjectKey as AdapterMetadataObjectKey +from sqltide._native import AdapterMetadataState as AdapterMetadataState +from sqltide._native import AdapterModelRealization as AdapterModelRealization +from sqltide._native import AdapterModelRealizationRequest as AdapterModelRealizationRequest +from sqltide._native import AdapterMutationResult as AdapterMutationResult +from sqltide._native import AdapterNodeResultRecord as AdapterNodeResultRecord +from sqltide._native import AdapterObjectStateRecord as AdapterObjectStateRecord +from sqltide._native import AdapterPhysicalRelationMapping as AdapterPhysicalRelationMapping +from sqltide._native import AdapterPreparedObjectMapping as AdapterPreparedObjectMapping +from sqltide._native import AdapterPublishEventRecord as AdapterPublishEventRecord +from sqltide._native import AdapterQueryCancellation as AdapterQueryCancellation +from sqltide._native import AdapterQueryResult as AdapterQueryResult +from sqltide._native import AdapterReadinessOffsetSummary as AdapterReadinessOffsetSummary +from sqltide._native import AdapterReadinessRequest as AdapterReadinessRequest +from sqltide._native import AdapterReadinessRootObservation as AdapterReadinessRootObservation +from sqltide._native import AdapterReadinessRootRequest as AdapterReadinessRootRequest +from sqltide._native import AdapterReadinessScalarSummary as AdapterReadinessScalarSummary +from sqltide._native import AdapterRefreshState as AdapterRefreshState +from sqltide._native import AdapterRelationCleanupRequest as AdapterRelationCleanupRequest +from sqltide._native import AdapterReplayBoundary as AdapterReplayBoundary +from sqltide._native import AdapterReplayColumns as AdapterReplayColumns +from sqltide._native import AdapterReplayCoverageRange as AdapterReplayCoverageRange +from sqltide._native import AdapterReplayCoverageRequest as AdapterReplayCoverageRequest +from sqltide._native import AdapterReplayLowerBound as AdapterReplayLowerBound +from sqltide._native import AdapterReplayOffsetFrontier as AdapterReplayOffsetFrontier +from sqltide._native import AdapterReplayOffsetProgressRequest as AdapterReplayOffsetProgressRequest +from sqltide._native import AdapterReplayOffsetRange as AdapterReplayOffsetRange +from sqltide._native import AdapterReplayQuery as AdapterReplayQuery +from sqltide._native import AdapterReplayRelations as AdapterReplayRelations +from sqltide._native import AdapterReplayRequest as AdapterReplayRequest +from sqltide._native import AdapterReplayWindow as AdapterReplayWindow +from sqltide._native import AdapterRunEventRecord as AdapterRunEventRecord +from sqltide._native import AdapterRunStatementRecord as AdapterRunStatementRecord +from sqltide._native import AdapterSensorCheckpointRecord as AdapterSensorCheckpointRecord +from sqltide._native import AdapterSensorLeaseRecord as AdapterSensorLeaseRecord +from sqltide._native import AdapterSensorOverrideRecord as AdapterSensorOverrideRecord +from sqltide._native import AdapterSensorState as AdapterSensorState +from sqltide._native import AdapterSensorStepRecord as AdapterSensorStepRecord +from sqltide._native import AdapterSensorTickRecord as AdapterSensorTickRecord +from sqltide._native import ( + AdapterSetDifferenceComparisonRequest as AdapterSetDifferenceComparisonRequest, +) +from sqltide._native import AdapterSetDifferenceTarget as AdapterSetDifferenceTarget +from sqltide._native import AdapterSourceRealization as AdapterSourceRealization +from sqltide._native import AdapterStableBinding as AdapterStableBinding +from sqltide._native import AdapterStableBindingRemoval as AdapterStableBindingRemoval +from sqltide._native import AdapterStableView as AdapterStableView +from sqltide._native import AdapterStatementProgress as AdapterStatementProgress +from sqltide._native import AdapterTable as AdapterTable +from sqltide._native import AdapterTargetMutationLock as AdapterTargetMutationLock +from sqltide._native import AdapterView as AdapterView +from sqltide._native import AdapterViewRealizationRequest as AdapterViewRealizationRequest +from sqltide._native import AdapterWarehouseActivity as AdapterWarehouseActivity +from sqltide._native import AdapterWarehouseDeadLetterModel as AdapterWarehouseDeadLetterModel +from sqltide._native import AdapterWarehouseDisk as AdapterWarehouseDisk +from sqltide._native import AdapterWarehouseHealth as AdapterWarehouseHealth +from sqltide._native import AdapterWarehouseKafkaConsumers as AdapterWarehouseKafkaConsumers +from sqltide._native import AdapterWarehouseMemory as AdapterWarehouseMemory +from sqltide._native import AdapterWarehouseStuckKafkaConsumer as AdapterWarehouseStuckKafkaConsumer +from sqltide._native import AdapterWarehouseTable as AdapterWarehouseTable +from sqltide._native import CatalogColumn as CatalogColumn +from sqltide._native import CatalogIdentity as CatalogIdentity +from sqltide._native import CatalogRelation as CatalogRelation +from sqltide._native import CatalogSnapshot as CatalogSnapshot +from sqltide._native import InspectedActiveTableBinding as InspectedActiveTableBinding +from sqltide._native import InspectedManagedTableState as InspectedManagedTableState +from sqltide._native import InspectedPhysicalTableCandidate as InspectedPhysicalTableCandidate diff --git a/tests/unit/src/sqltide/adapter/_test_types.py b/tests/unit/src/sqltide/adapter/_test_types.py index 95231836..fcae8721 100644 --- a/tests/unit/src/sqltide/adapter/_test_types.py +++ b/tests/unit/src/sqltide/adapter/_test_types.py @@ -1,5 +1,7 @@ from dataclasses import dataclass +from sqltide.adapter.models import AdapterConnectionConfig + @dataclass(frozen=True) class ResolveAdapterTestCase: @@ -50,3 +52,10 @@ class AdapterConnectionConfigRedactionTestCase: password: str expected_absent_fragment: str expected_present_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class AdapterConnectionConfigReprTestCase: + description: str + config: AdapterConnectionConfig + expected_repr: str diff --git a/tests/unit/src/sqltide/adapter/test_models.py b/tests/unit/src/sqltide/adapter/test_models.py index 397bf041..acb7b873 100644 --- a/tests/unit/src/sqltide/adapter/test_models.py +++ b/tests/unit/src/sqltide/adapter/test_models.py @@ -6,6 +6,7 @@ from sqltide.adapter.models import AdapterConnectionConfig, AdapterQueryResult from tests.unit.src.sqltide.adapter._test_types import ( AdapterConnectionConfigRedactionTestCase, + AdapterConnectionConfigReprTestCase, AdapterQueryResultDecodingTestCase, AdapterQueryResultErrorTestCase, ) @@ -97,3 +98,46 @@ def test_given_resolved_connection_config_when_rendering_then_it_redacts_the_pas expected_fragment: str for expected_fragment in test_case.expected_present_fragments: assert expected_fragment in rendered + + +@pytest.mark.parametrize( + "test_case", + [ + AdapterConnectionConfigReprTestCase( + description="renders every field but the password with default database and settings", + config=AdapterConnectionConfig( + host="localhost", + port=8123, + username="sqltide", + password="super-secret-password", + ), + expected_repr=( + "AdapterConnectionConfig(host='localhost', port=8123, username='sqltide', " + "password='***', database=None, settings=())" + ), + ), + AdapterConnectionConfigReprTestCase( + description="renders the database and settings and still hides a quoted password", + config=AdapterConnectionConfig( + host="warehouse.example.com", + port=9440, + username="orders_reader", + password="orders-'secret\"-value", + database="orders", + settings=(("max_threads", "4"),), + ), + expected_repr=( + "AdapterConnectionConfig(host='warehouse.example.com', port=9440, " + "username='orders_reader', password='***', database='orders', " + "settings=(('max_threads', '4'),))" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_resolved_connection_config_when_rendering_then_repr_and_str_match_exactly( + test_case: AdapterConnectionConfigReprTestCase, +) -> None: + rendered: tuple[str, str] = (repr(test_case.config), str(test_case.config)) + + assert rendered == (test_case.expected_repr, test_case.expected_repr) diff --git a/tests/unit/src/sqltide/compiler/compile/_test_types.py b/tests/unit/src/sqltide/compiler/compile/_test_types.py index 7040b276..51ecddf8 100644 --- a/tests/unit/src/sqltide/compiler/compile/_test_types.py +++ b/tests/unit/src/sqltide/compiler/compile/_test_types.py @@ -309,3 +309,10 @@ class LiveQuarantineShapeTestCase: description: str view_sql: str expected_quarantined: bool + + +@dataclass(frozen=True) +class CompiledModelShallowCopyTestCase: + description: str + transform_query: str + expected_type: type diff --git a/tests/unit/src/sqltide/compiler/compile/test_compile_pipeline.py b/tests/unit/src/sqltide/compiler/compile/test_compile_pipeline.py index e18aec82..6b8310cd 100644 --- a/tests/unit/src/sqltide/compiler/compile/test_compile_pipeline.py +++ b/tests/unit/src/sqltide/compiler/compile/test_compile_pipeline.py @@ -1,4 +1,5 @@ -from dataclasses import replace +import copy +from dataclasses import fields, replace from pathlib import Path from typing import cast @@ -56,6 +57,7 @@ from sqltide.compiler.pipeline.models import RealizedProject from tests.unit.src.sqltide.compiler.compile._test_types import ( CompileAdoptedSourceSharingTestCase, + CompiledModelShallowCopyTestCase, CompileModelNamingTestCase, CompileModelRetentionTestCase, CompileModelTtlDefaultTestCase, @@ -1899,3 +1901,28 @@ def test_given_consistent_shared_adopted_relation_when_compiling_then_preserves_ tuple(cast(ExternalTableSourceStep, source.source).table_name for source in project.sources) == test_case.expected_source_table_names ) + + +@pytest.mark.parametrize( + "test_case", + [ + CompiledModelShallowCopyTestCase( + description="a compiled table model copy keeps its class and shares every field", + transform_query='SELECT order_id::UInt64 AS order_id FROM __ref("orders")', + expected_type=CompiledTableModel, + ) + ], + ids=lambda case: case.description, +) +def test_given_compiled_table_model_when_shallow_copying_then_the_copy_shares_every_field( + test_case: CompiledModelShallowCopyTestCase, +) -> None: + model: CompiledModel = compile_test_pipeline( + build_inline_sql_pipeline(test_case.transform_query) + ).models[0] + + copied: CompiledModel = copy.copy(model) + + assert type(copied) is test_case.expected_type + assert copied == model + assert all(getattr(copied, field.name) is getattr(model, field.name) for field in fields(model)) diff --git a/tests/unit/src/sqltide/compiler/discovery/_test_types.py b/tests/unit/src/sqltide/compiler/discovery/_test_types.py index 169811cd..323771dc 100644 --- a/tests/unit/src/sqltide/compiler/discovery/_test_types.py +++ b/tests/unit/src/sqltide/compiler/discovery/_test_types.py @@ -1,9 +1,13 @@ from dataclasses import dataclass from pathlib import Path +from typing import TYPE_CHECKING from sqltide.compiler.discovery.models import KafkaSettings, SourceFreshnessPolicy from sqltide.compiler.discovery.types import ModelReferenceScope, PipelineMode +if TYPE_CHECKING: + from _typeshed import DataclassInstance + @dataclass(frozen=True) class DiscoverPipelinesTestCase: @@ -360,3 +364,18 @@ class KafkaSettingsReprTestCase: kafka_settings: KafkaSettings expected_repr_pattern: str expected_hidden_values: tuple[str, ...] + + +@dataclass(frozen=True) +class ShallowCopyTestCase: + description: str + instance: "DataclassInstance" + expected_type: type + + +@dataclass(frozen=True) +class ReplaceTestCase: + description: str + instance: "DataclassInstance" + changes: dict[str, object] + expected_instance: "DataclassInstance" diff --git a/tests/unit/src/sqltide/compiler/discovery/test_models.py b/tests/unit/src/sqltide/compiler/discovery/test_models.py index 23fd33e8..96bce77d 100644 --- a/tests/unit/src/sqltide/compiler/discovery/test_models.py +++ b/tests/unit/src/sqltide/compiler/discovery/test_models.py @@ -1,14 +1,26 @@ """Behavior tests for the authored model step specifications.""" +import copy +import operator import re +from dataclasses import fields import pytest from sqltide.compiler.discovery.exceptions import ProjectSpecError -from sqltide.compiler.discovery.models import KafkaSettings, TransformStep, ViewStep +from sqltide.compiler.discovery.models import ( + KafkaLandingStep, + KafkaSettings, + Pipeline, + RawConnectionConfig, + TransformStep, + ViewStep, +) from tests.unit.src.sqltide.compiler.discovery._test_types import ( KafkaSettingsReprTestCase, ModelStepErrorTestCase, + ReplaceTestCase, + ShallowCopyTestCase, ) QUERY_OR_FILE_MESSAGE: str = "Exactly one of 'query' or 'sql_file' must be provided" @@ -115,3 +127,106 @@ def test_given_kafka_settings_when_rendering_repr_then_no_field_value_is_shown( assert re.fullmatch(test_case.expected_repr_pattern, rendered) is not None assert not any(value in rendered for value in test_case.expected_hidden_values) + + +@pytest.mark.parametrize( + "test_case", + [ + ShallowCopyTestCase( + description="connection values keep their frozen configuration pairs", + instance=RawConnectionConfig( + values=( + ("host", "warehouse.example.com"), + ("settings", {"max_threads": [1, 2]}), + ) + ), + expected_type=RawConnectionConfig, + ), + ShallowCopyTestCase( + description="a transform keeps its settings mapping and ORDER BY tuple", + instance=TransformStep( + name="orders_enriched", + source="orders", + engine="MergeTree()", + order_by=["order_id"], + query="SELECT 1 AS order_id", + settings={"max_threads": "4"}, + ), + expected_type=TransformStep, + ), + ShallowCopyTestCase( + description="a pipeline keeps its transforms tuple", + instance=Pipeline( + name="orders", + source=KafkaLandingStep( + name="orders", + kafka=KafkaSettings(broker_list="kafka:9092", topic="source.orders"), + ), + transforms=[ + TransformStep( + name="orders_enriched", + source="orders", + engine="MergeTree()", + order_by=("order_id",), + query="SELECT 1 AS order_id", + ) + ], + ), + expected_type=Pipeline, + ), + ], + ids=lambda case: case.description, +) +def test_given_authored_model_when_shallow_copying_then_the_copy_shares_every_field( + test_case: ShallowCopyTestCase, +) -> None: + copied: object = copy.copy(test_case.instance) + + assert type(copied) is test_case.expected_type + assert copied == test_case.instance + assert all( + getattr(copied, field.name) is getattr(test_case.instance, field.name) + for field in fields(test_case.instance) + ) + + +@pytest.mark.parametrize( + "test_case", + [ + ReplaceTestCase( + description="a transform coerces its replaced ORDER BY and settings again", + instance=TransformStep( + name="orders_enriched", + source="orders", + engine="MergeTree()", + order_by=("order_id",), + query="SELECT 1 AS order_id", + ), + changes={"order_by": ["customer_id"], "settings": {"max_threads": "8"}}, + expected_instance=TransformStep( + name="orders_enriched", + source="orders", + engine="MergeTree()", + order_by=("customer_id",), + query="SELECT 1 AS order_id", + settings={"max_threads": "8"}, + ), + ), + ReplaceTestCase( + description="connection values are replaced as frozen configuration pairs", + instance=RawConnectionConfig(values=(("host", "warehouse.example.com"),)), + changes={"values": (("host", "replica.example.com"), ("port", 9000))}, + expected_instance=RawConnectionConfig( + values=(("host", "replica.example.com"), ("port", 9000)) + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_authored_model_when_replacing_through_copy_protocol_then_fields_are_rebuilt( + test_case: ReplaceTestCase, +) -> None: + replaced: object = operator.methodcaller("__replace__", **test_case.changes)(test_case.instance) + + assert type(replaced) is type(test_case.expected_instance) + assert replaced == test_case.expected_instance