Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 78 additions & 0 deletions cloud_pipelines_backend/instrumentation/metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,51 @@ class MetricUnit(str, enum.Enum):
unit=MetricUnit.EXECUTIONS,
)

container_execution_refresh_duration = orchestrator_meter.create_histogram(
name="container_execution.refresh.duration",
description=(
"Duration of one orchestrator refresh of a pending or running container"
" execution, split into its queries and its processing"
),
unit=MetricUnit.SECONDS,
explicit_bucket_boundaries_advisory=[
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0,
30.0,
],
)

container_execution_completion_lag_duration = orchestrator_meter.create_histogram(
name="container_execution.completion_lag.duration",
description=(
"Time from a container ending, as reported by its launcher, to the"
" orchestrator committing its container execution's SUCCEEDED or FAILED status"
),
unit=MetricUnit.SECONDS,
explicit_bucket_boundaries_advisory=[
0.5,
1.0,
2.0,
5.0,
10.0,
30.0,
60.0,
120.0,
300.0,
600.0,
1800.0,
],
)


# ---------------------------------------------------------------------------
# SQLAlchemy event listeners
Expand Down Expand Up @@ -113,3 +158,36 @@ def _handle_before_commit(session: orm.Session) -> None:
)
obj._status_changed = False
execution_tracing.emit_execution_trace(execution=obj)


def record_refresh_duration(
Comment thread
morgan-wowk marked this conversation as resolved.
Resolved
*, queries_duration_ms: float, processing_duration_ms: float
) -> None:
"""Records one running container execution refresh's queries and processing durations."""
for phase, duration_ms in (
Comment thread
morgan-wowk marked this conversation as resolved.
Resolved
("queries", queries_duration_ms),
("processing", processing_duration_ms),
):
container_execution_refresh_duration.record(
duration_ms / 1000,
attributes={"container_execution.refresh.phase": phase},
)


def record_completion_lag(
*,
ended_at: datetime.datetime | None,
status: backend_types_sql.ContainerExecutionStatus,
) -> None:
"""Records how long after its container ended a container execution's SUCCEEDED or FAILED status was committed."""
if ended_at is None:
return
if ended_at.tzinfo is None:
ended_at = ended_at.replace(tzinfo=datetime.timezone.utc)
lag_seconds = (
datetime.datetime.now(tz=datetime.timezone.utc) - ended_at
).total_seconds()
container_execution_completion_lag_duration.record(
max(lag_seconds, 0.0),
attributes={"container_execution.status": status.value},
)
10 changes: 9 additions & 1 deletion cloud_pipelines_backend/orchestrator_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,6 @@ def internal_process_running_executions_queue(self, session: orm.Session):
query_duration_ms = (time.monotonic_ns() - query_start_timestamp) / 1_000_000
if running_container_execution:
self._running_executions_queue_idle = False
start_timestamp = time.monotonic_ns()

execution_node_ids = list(
session.scalars(
Expand All @@ -241,6 +240,7 @@ def internal_process_running_executions_queue(self, session: orm.Session):
queries_duration_ms = (
time.monotonic_ns() - query_start_timestamp
) / 1_000_000
start_timestamp = time.monotonic_ns()
_logger.info(
f"Before processing running container execution. Queries duration {queries_duration_ms}ms."
)
Expand Down Expand Up @@ -334,6 +334,10 @@ def internal_process_running_executions_queue(self, session: orm.Session):
_logger.info(
f"After processing running container execution (duration: {duration_ms}ms)"
)
app_metrics.record_refresh_duration(
Comment thread
morgan-wowk marked this conversation as resolved.
Resolved
queries_duration_ms=queries_duration_ms,
processing_duration_ms=duration_ms,
)
return True
else:
if not self._running_executions_queue_idle or query_duration_ms > 1000:
Expand Down Expand Up @@ -1225,6 +1229,10 @@ def _maybe_preload_value(
launcher_interfaces.ContainerStatus.SUCCEEDED,
launcher_interfaces.ContainerStatus.FAILED,
):
app_metrics.record_completion_lag(
ended_at=reloaded_launched_container.ended_at,
status=container_execution.status,
)
try:
launched_container.cleanup()
except Exception:
Expand Down
83 changes: 83 additions & 0 deletions tests/instrumentation/test_metrics.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
"""Tests for the orchestrator metric helpers."""

import datetime
from unittest import mock

import pytest

from cloud_pipelines_backend import backend_types_sql as bts
from cloud_pipelines_backend.instrumentation import metrics


@pytest.fixture()
def refresh_duration(monkeypatch: pytest.MonkeyPatch) -> mock.MagicMock:
histogram = mock.MagicMock()
monkeypatch.setattr(metrics, "container_execution_refresh_duration", histogram)
return histogram


@pytest.fixture()
def completion_lag_duration(monkeypatch: pytest.MonkeyPatch) -> mock.MagicMock:
histogram = mock.MagicMock()
monkeypatch.setattr(
metrics, "container_execution_completion_lag_duration", histogram
)
return histogram


def _seconds_ago(seconds: float) -> datetime.datetime:
return datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta(
seconds=seconds
)


def test_refresh_duration_records_each_phase_in_seconds(refresh_duration):
metrics.record_refresh_duration(
queries_duration_ms=250, processing_duration_ms=1500
)

assert refresh_duration.record.call_args_list == [
mock.call(0.25, attributes={"container_execution.refresh.phase": "queries"}),
mock.call(1.5, attributes={"container_execution.refresh.phase": "processing"}),
]


def test_completion_lag_records_seconds_since_the_container_ended(
completion_lag_duration,
):
metrics.record_completion_lag(
ended_at=_seconds_ago(42), status=bts.ContainerExecutionStatus.FAILED
)

(seconds,), kwargs = completion_lag_duration.record.call_args
assert 42 <= seconds < 43
assert kwargs == {"attributes": {"container_execution.status": "FAILED"}}


def test_completion_lag_clamps_an_end_time_in_the_future_to_zero(
completion_lag_duration,
):
metrics.record_completion_lag(
ended_at=_seconds_ago(-5), status=bts.ContainerExecutionStatus.SUCCEEDED
)

(seconds,), _ = completion_lag_duration.record.call_args
assert seconds == 0.0


def test_completion_lag_treats_a_naive_end_time_as_utc(completion_lag_duration):
metrics.record_completion_lag(
ended_at=_seconds_ago(42).replace(tzinfo=None),
status=bts.ContainerExecutionStatus.SUCCEEDED,
)

(seconds,), _ = completion_lag_duration.record.call_args
assert 42 <= seconds < 43


def test_completion_lag_skips_a_missing_end_time(completion_lag_duration):
metrics.record_completion_lag(
ended_at=None, status=bts.ContainerExecutionStatus.SUCCEEDED
)

completion_lag_duration.record.assert_not_called()
74 changes: 74 additions & 0 deletions tests/test_orchestrator_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -921,3 +921,77 @@ def test_non_retriable_termination_error_is_a_system_error(self) -> None:
sql.select(bts.ContainerExecution)
)
assert container_execution.status == bts.ContainerExecutionStatus.SYSTEM_ERROR


class TestRefreshMetrics:
_ENDED_AT = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)

def _refresh_once(
self,
*,
refreshed_status: launcher_interfaces.ContainerStatus | None = None,
outputs_exist: bool = True,
):
"""Launches a one-task run with one output, refreshes it once, and returns the mocked metrics module."""
session_factory = _create_session_factory()
_create_pipeline_run(
session_factory,
_make_graph_task_spec(
tasks={
"task": structures.TaskSpec(
component_ref=structures.ComponentReference(
spec=_make_container_component(outputs=["out"])
)
)
}
),
)
launched_container_mock = _make_launched_container_mock()
_process_queued_executions(session_factory, launched_container_mock)
orchestrator = _make_running_orchestrator(
session_factory, launched_container_mock
)
orchestrator._storage_provider.make_uri.return_value.get_reader.return_value.exists.return_value = (
outputs_exist
)
if refreshed_status:
orchestrator._launcher.get_refreshed_launched_container_from_dict.return_value = mock.MagicMock(
status=refreshed_status,
exit_code=0,
started_at=None,
ended_at=self._ENDED_AT,
launcher_error_message=None,
to_dict=lambda: {"foo": "bar"},
)
with mock.patch.object(orchestrator_sql, "app_metrics") as app_metrics:
orchestrator.internal_process_running_executions_queue(
session=session_factory()
)
return app_metrics

def test_every_refresh_records_its_duration(self) -> None:
app_metrics = self._refresh_once()

app_metrics.record_refresh_duration.assert_called_once()
app_metrics.record_completion_lag.assert_not_called()

def test_a_failed_container_records_its_completion_lag(self) -> None:
app_metrics = self._refresh_once(
refreshed_status=launcher_interfaces.ContainerStatus.FAILED
)

app_metrics.record_completion_lag.assert_called_once_with(
ended_at=self._ENDED_AT, status=bts.ContainerExecutionStatus.FAILED
)

def test_a_succeeded_container_missing_outputs_records_the_committed_failed_status(
self,
) -> None:
app_metrics = self._refresh_once(
refreshed_status=launcher_interfaces.ContainerStatus.SUCCEEDED,
outputs_exist=False,
)

app_metrics.record_completion_lag.assert_called_once_with(
ended_at=self._ENDED_AT, status=bts.ContainerExecutionStatus.FAILED
)
Loading