Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 29 additions & 3 deletions cloud_pipelines_backend/orchestrator_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@

_T = typing.TypeVar("_T")

_HEARTBEAT_INTERVAL_SECONDS = 60

DYNAMIC_DATA_SECRET_KEY = "secret"
DYNAMIC_DATA_SECRET_NAME_KEY = "name"

Expand Down Expand Up @@ -90,6 +92,9 @@ def __init__(
_max_container_execution_refresh_error_retries
)
self._container_execution_refresh_error_counts: dict[Any, int] = {}
self._heartbeat_started_at = time.monotonic()
self._heartbeat_queued_execution_count = 0
self._heartbeat_running_container_execution_count = 0

self._max_queue_batch_size = _max_queue_batch_size
self._max_queue_batch_duration = _max_queue_batch_duration
Expand Down Expand Up @@ -136,6 +141,7 @@ def process_each_queue_once(self):
> self._max_queue_batch_duration
):
break
self._maybe_log_heartbeat()

def internal_process_queued_executions_queue(self, session: orm.Session):
_logger.debug("Selecting queued execution to process.)")
Expand All @@ -162,6 +168,7 @@ def internal_process_queued_executions_queue(self, session: orm.Session):
return False

self._queued_executions_queue_idle = False
self._heartbeat_queued_execution_count += 1
start_timestamp = time.monotonic_ns()

with contextual_logging.execution_logging_context(queued_execution):
Expand Down Expand Up @@ -228,6 +235,7 @@ def internal_process_running_executions_queue(self, session: orm.Session):
return False

self._running_executions_queue_idle = False
self._heartbeat_running_container_execution_count += 1

execution_node_ids = list(
session.scalars(
Expand All @@ -246,7 +254,7 @@ def internal_process_running_executions_queue(self, session: orm.Session):
time.monotonic_ns() - query_start_timestamp
) / 1_000_000
start_timestamp = time.monotonic_ns()
_logger.info(
_logger.debug(
f"Before processing running container execution. Queries duration {queries_duration_ms}ms."
)
try:
Expand Down Expand Up @@ -332,9 +340,13 @@ def internal_process_running_executions_queue(self, session: orm.Session):
)
finally:
duration_ms = (time.monotonic_ns() - start_timestamp) / 1_000_000
_logger.info(
_logger.debug(
f"After processing running container execution (duration: {duration_ms}ms)"
)
if queries_duration_ms > 1000 or duration_ms > 1000:
_logger.info(
f"Slow running container execution visit (queries: {queries_duration_ms}ms, processing: {duration_ms}ms)"
)
app_metrics.record_refresh_duration(
queries_duration_ms=queries_duration_ms,
processing_duration_ms=duration_ms,
Expand Down Expand Up @@ -1013,7 +1025,7 @@ def internal_process_one_running_execution(
reloaded_launched_container.status
)
if new_status == previous_status:
_logger.info(f"Container execution remains in {new_status} state.")
_logger.debug(f"Container execution remains in {new_status} state.")
return
_logger.info(
f"Container execution is now in state {new_status} (was {previous_status})."
Expand Down Expand Up @@ -1246,6 +1258,20 @@ def _maybe_preload_value(
f"Processing running container execution {container_execution.id}: Error doing launched container cleanup."
)

def _maybe_log_heartbeat(self):
"""Logs, about once a minute, how many queued and running executions the sweeps processed since the last one."""
elapsed_seconds = time.monotonic() - self._heartbeat_started_at
if elapsed_seconds < _HEARTBEAT_INTERVAL_SECONDS:
return
_logger.info(
f"Orchestrator heartbeat: processed {self._heartbeat_queued_execution_count} queued executions and "
f"{self._heartbeat_running_container_execution_count} running container executions in the last "
f"{elapsed_seconds:.0f}s."
)
self._heartbeat_started_at = time.monotonic()
self._heartbeat_queued_execution_count = 0
self._heartbeat_running_container_execution_count = 0


def _get_direct_downstream_executions(
session: orm.Session, execution: bts.ExecutionNode
Expand Down
60 changes: 60 additions & 0 deletions tests/test_orchestrator_sql.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""Tests for ``orchestrator_sql``."""

import datetime
import logging
from typing import Callable
from unittest import mock

Expand Down Expand Up @@ -1077,3 +1078,62 @@ def test_other_launcher_errors_are_still_system_errors(self) -> None:
app_metrics.execution_system_errors.add.assert_called_once_with(1)
bugsnag.notify.assert_called_once()
app_metrics.execution_launch_rejections.add.assert_not_called()


def _heartbeat_messages(caplog: pytest.LogCaptureFixture) -> list[str]:
return [
record.getMessage()
for record in caplog.records
if record.getMessage().startswith("Orchestrator heartbeat:")
]


class TestHeartbeat:
def test_idle_sweeps_log_a_heartbeat_once_the_interval_passes(
self, caplog: pytest.LogCaptureFixture
) -> None:
caplog.set_level(logging.INFO, logger=orchestrator_sql.__name__)
orchestrator = _make_orchestrator(
session_factory=_create_session_factory(),
launched_container_mock=_make_launched_container_mock(),
)

orchestrator.process_each_queue_once()
assert _heartbeat_messages(caplog) == []

orchestrator._heartbeat_started_at -= 61
orchestrator.process_each_queue_once()
assert len(_heartbeat_messages(caplog)) == 1
assert (
"processed 0 queued executions and 0 running container executions"
in _heartbeat_messages(caplog)[0]
)

def test_heartbeat_counts_processed_executions_and_then_resets(
self, caplog: pytest.LogCaptureFixture
) -> None:
caplog.set_level(logging.INFO, logger=orchestrator_sql.__name__)
session_factory = _create_session_factory()
_create_pipeline_run(session_factory, _single_task_pipeline())
orchestrator = _make_running_orchestrator(
session_factory, _make_launched_container_mock()
)

assert orchestrator.internal_process_queued_executions_queue(
session=session_factory()
)
assert orchestrator.internal_process_running_executions_queue(
session=session_factory()
)
orchestrator._heartbeat_started_at -= 61
orchestrator._maybe_log_heartbeat()
orchestrator._heartbeat_started_at -= 61
orchestrator._maybe_log_heartbeat()

first, second = _heartbeat_messages(caplog)
assert (
"processed 1 queued executions and 1 running container executions" in first
)
assert (
"processed 0 queued executions and 0 running container executions" in second
)
Loading