From aa4bb4e91e58709eb6bcb575213e68d37405ab4f Mon Sep 17 00:00:00 2001 From: Morgan Wowk Date: Mon, 2 Nov 2026 14:41:41 -0800 Subject: [PATCH] feat(orchestrator): Add refresh duration and completion lag metrics Adds two histograms to the tangle.orchestrator meter: - container_execution.refresh.duration: each running container execution refresh's queries and processing time, by container_execution.refresh.phase. - container_execution.completion_lag.duration: the time from a container ending, as its launcher reports it, to the orchestrator committing its container execution's SUCCEEDED or FAILED status, by the committed container_execution.status. A SUCCEEDED container with missing outputs is committed, and recorded, as FAILED. Completion lag is recorded at most once per container execution: the running sweep only selects PENDING and RUNNING rows, and nothing moves a container execution back out of SUCCEEDED or FAILED. CANCELLED and SYSTEM_ERROR are set by the orchestrator itself, with no launcher-reported end, so they are not recorded. SKIPPED only applies to execution nodes, which never get a container execution. A missing or naive end time is skipped, and an end time ahead of the orchestrator's clock is recorded as 0 since OTel histograms drop negative values. The processing clock now starts where the queries clock stops, so the execution-node lookup is counted once, in queries, and the two phases add up to the refresh. --- .../instrumentation/metrics.py | 78 +++++++++++++++++ cloud_pipelines_backend/orchestrator_sql.py | 10 ++- tests/instrumentation/test_metrics.py | 83 +++++++++++++++++++ tests/test_orchestrator_sql.py | 74 +++++++++++++++++ 4 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 tests/instrumentation/test_metrics.py diff --git a/cloud_pipelines_backend/instrumentation/metrics.py b/cloud_pipelines_backend/instrumentation/metrics.py index 6f0ba877..e6e5528e 100644 --- a/cloud_pipelines_backend/instrumentation/metrics.py +++ b/cloud_pipelines_backend/instrumentation/metrics.py @@ -77,6 +77,51 @@ class MetricUnit(str, enum.Enum): unit=MetricUnit.EXECUTIONS, ) +container_execution_refresh_duration = orchestrator_meter.create_histogram( + name="container_execution.refresh.duration", + description=( + "Duration of one orchestrator refresh of a pending or running container" + " execution, split into its queries and its processing" + ), + unit=MetricUnit.SECONDS, + explicit_bucket_boundaries_advisory=[ + 0.005, + 0.01, + 0.025, + 0.05, + 0.1, + 0.25, + 0.5, + 1.0, + 2.5, + 5.0, + 10.0, + 30.0, + ], +) + +container_execution_completion_lag_duration = orchestrator_meter.create_histogram( + name="container_execution.completion_lag.duration", + description=( + "Time from a container ending, as reported by its launcher, to the" + " orchestrator committing its container execution's SUCCEEDED or FAILED status" + ), + unit=MetricUnit.SECONDS, + explicit_bucket_boundaries_advisory=[ + 0.5, + 1.0, + 2.0, + 5.0, + 10.0, + 30.0, + 60.0, + 120.0, + 300.0, + 600.0, + 1800.0, + ], +) + # --------------------------------------------------------------------------- # SQLAlchemy event listeners @@ -113,3 +158,36 @@ def _handle_before_commit(session: orm.Session) -> None: ) obj._status_changed = False execution_tracing.emit_execution_trace(execution=obj) + + +def record_refresh_duration( + *, queries_duration_ms: float, processing_duration_ms: float +) -> None: + """Records one running container execution refresh's queries and processing durations.""" + for phase, duration_ms in ( + ("queries", queries_duration_ms), + ("processing", processing_duration_ms), + ): + container_execution_refresh_duration.record( + duration_ms / 1000, + attributes={"container_execution.refresh.phase": phase}, + ) + + +def record_completion_lag( + *, + ended_at: datetime.datetime | None, + status: backend_types_sql.ContainerExecutionStatus, +) -> None: + """Records how long after its container ended a container execution's SUCCEEDED or FAILED status was committed.""" + if ended_at is None: + return + if ended_at.tzinfo is None: + ended_at = ended_at.replace(tzinfo=datetime.timezone.utc) + lag_seconds = ( + datetime.datetime.now(tz=datetime.timezone.utc) - ended_at + ).total_seconds() + container_execution_completion_lag_duration.record( + max(lag_seconds, 0.0), + attributes={"container_execution.status": status.value}, + ) diff --git a/cloud_pipelines_backend/orchestrator_sql.py b/cloud_pipelines_backend/orchestrator_sql.py index 00a0113a..178fb3bb 100644 --- a/cloud_pipelines_backend/orchestrator_sql.py +++ b/cloud_pipelines_backend/orchestrator_sql.py @@ -223,7 +223,6 @@ def internal_process_running_executions_queue(self, session: orm.Session): query_duration_ms = (time.monotonic_ns() - query_start_timestamp) / 1_000_000 if running_container_execution: self._running_executions_queue_idle = False - start_timestamp = time.monotonic_ns() execution_node_ids = list( session.scalars( @@ -241,6 +240,7 @@ def internal_process_running_executions_queue(self, session: orm.Session): queries_duration_ms = ( time.monotonic_ns() - query_start_timestamp ) / 1_000_000 + start_timestamp = time.monotonic_ns() _logger.info( f"Before processing running container execution. Queries duration {queries_duration_ms}ms." ) @@ -334,6 +334,10 @@ def internal_process_running_executions_queue(self, session: orm.Session): _logger.info( f"After processing running container execution (duration: {duration_ms}ms)" ) + app_metrics.record_refresh_duration( + queries_duration_ms=queries_duration_ms, + processing_duration_ms=duration_ms, + ) return True else: if not self._running_executions_queue_idle or query_duration_ms > 1000: @@ -1225,6 +1229,10 @@ def _maybe_preload_value( launcher_interfaces.ContainerStatus.SUCCEEDED, launcher_interfaces.ContainerStatus.FAILED, ): + app_metrics.record_completion_lag( + ended_at=reloaded_launched_container.ended_at, + status=container_execution.status, + ) try: launched_container.cleanup() except Exception: diff --git a/tests/instrumentation/test_metrics.py b/tests/instrumentation/test_metrics.py new file mode 100644 index 00000000..9cd808f9 --- /dev/null +++ b/tests/instrumentation/test_metrics.py @@ -0,0 +1,83 @@ +"""Tests for the orchestrator metric helpers.""" + +import datetime +from unittest import mock + +import pytest + +from cloud_pipelines_backend import backend_types_sql as bts +from cloud_pipelines_backend.instrumentation import metrics + + +@pytest.fixture() +def refresh_duration(monkeypatch: pytest.MonkeyPatch) -> mock.MagicMock: + histogram = mock.MagicMock() + monkeypatch.setattr(metrics, "container_execution_refresh_duration", histogram) + return histogram + + +@pytest.fixture() +def completion_lag_duration(monkeypatch: pytest.MonkeyPatch) -> mock.MagicMock: + histogram = mock.MagicMock() + monkeypatch.setattr( + metrics, "container_execution_completion_lag_duration", histogram + ) + return histogram + + +def _seconds_ago(seconds: float) -> datetime.datetime: + return datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta( + seconds=seconds + ) + + +def test_refresh_duration_records_each_phase_in_seconds(refresh_duration): + metrics.record_refresh_duration( + queries_duration_ms=250, processing_duration_ms=1500 + ) + + assert refresh_duration.record.call_args_list == [ + mock.call(0.25, attributes={"container_execution.refresh.phase": "queries"}), + mock.call(1.5, attributes={"container_execution.refresh.phase": "processing"}), + ] + + +def test_completion_lag_records_seconds_since_the_container_ended( + completion_lag_duration, +): + metrics.record_completion_lag( + ended_at=_seconds_ago(42), status=bts.ContainerExecutionStatus.FAILED + ) + + (seconds,), kwargs = completion_lag_duration.record.call_args + assert 42 <= seconds < 43 + assert kwargs == {"attributes": {"container_execution.status": "FAILED"}} + + +def test_completion_lag_clamps_an_end_time_in_the_future_to_zero( + completion_lag_duration, +): + metrics.record_completion_lag( + ended_at=_seconds_ago(-5), status=bts.ContainerExecutionStatus.SUCCEEDED + ) + + (seconds,), _ = completion_lag_duration.record.call_args + assert seconds == 0.0 + + +def test_completion_lag_treats_a_naive_end_time_as_utc(completion_lag_duration): + metrics.record_completion_lag( + ended_at=_seconds_ago(42).replace(tzinfo=None), + status=bts.ContainerExecutionStatus.SUCCEEDED, + ) + + (seconds,), _ = completion_lag_duration.record.call_args + assert 42 <= seconds < 43 + + +def test_completion_lag_skips_a_missing_end_time(completion_lag_duration): + metrics.record_completion_lag( + ended_at=None, status=bts.ContainerExecutionStatus.SUCCEEDED + ) + + completion_lag_duration.record.assert_not_called() diff --git a/tests/test_orchestrator_sql.py b/tests/test_orchestrator_sql.py index 77b4988e..bd757042 100644 --- a/tests/test_orchestrator_sql.py +++ b/tests/test_orchestrator_sql.py @@ -921,3 +921,77 @@ def test_non_retriable_termination_error_is_a_system_error(self) -> None: sql.select(bts.ContainerExecution) ) assert container_execution.status == bts.ContainerExecutionStatus.SYSTEM_ERROR + + +class TestRefreshMetrics: + _ENDED_AT = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc) + + def _refresh_once( + self, + *, + refreshed_status: launcher_interfaces.ContainerStatus | None = None, + outputs_exist: bool = True, + ): + """Launches a one-task run with one output, refreshes it once, and returns the mocked metrics module.""" + session_factory = _create_session_factory() + _create_pipeline_run( + session_factory, + _make_graph_task_spec( + tasks={ + "task": structures.TaskSpec( + component_ref=structures.ComponentReference( + spec=_make_container_component(outputs=["out"]) + ) + ) + } + ), + ) + launched_container_mock = _make_launched_container_mock() + _process_queued_executions(session_factory, launched_container_mock) + orchestrator = _make_running_orchestrator( + session_factory, launched_container_mock + ) + orchestrator._storage_provider.make_uri.return_value.get_reader.return_value.exists.return_value = ( + outputs_exist + ) + if refreshed_status: + orchestrator._launcher.get_refreshed_launched_container_from_dict.return_value = mock.MagicMock( + status=refreshed_status, + exit_code=0, + started_at=None, + ended_at=self._ENDED_AT, + launcher_error_message=None, + to_dict=lambda: {"foo": "bar"}, + ) + with mock.patch.object(orchestrator_sql, "app_metrics") as app_metrics: + orchestrator.internal_process_running_executions_queue( + session=session_factory() + ) + return app_metrics + + def test_every_refresh_records_its_duration(self) -> None: + app_metrics = self._refresh_once() + + app_metrics.record_refresh_duration.assert_called_once() + app_metrics.record_completion_lag.assert_not_called() + + def test_a_failed_container_records_its_completion_lag(self) -> None: + app_metrics = self._refresh_once( + refreshed_status=launcher_interfaces.ContainerStatus.FAILED + ) + + app_metrics.record_completion_lag.assert_called_once_with( + ended_at=self._ENDED_AT, status=bts.ContainerExecutionStatus.FAILED + ) + + def test_a_succeeded_container_missing_outputs_records_the_committed_failed_status( + self, + ) -> None: + app_metrics = self._refresh_once( + refreshed_status=launcher_interfaces.ContainerStatus.SUCCEEDED, + outputs_exist=False, + ) + + app_metrics.record_completion_lag.assert_called_once_with( + ended_at=self._ENDED_AT, status=bts.ContainerExecutionStatus.FAILED + )