Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 65 additions & 34 deletions cloud_pipelines_backend/orchestrator_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,9 @@ def __init__(
_max_container_execution_refresh_error_retries: int = 3,
_max_queue_batch_size: int = 1,
_max_queue_batch_duration: datetime.timedelta = datetime.timedelta(),
_running_execution_refresh_interval: datetime.timedelta = datetime.timedelta(
seconds=30
),
queued_execution_interceptor: QueuedExecutionInterceptor | None = None,
):
self._session_factory = session_factory
Expand All @@ -93,49 +96,74 @@ def __init__(

self._max_queue_batch_size = _max_queue_batch_size
self._max_queue_batch_duration = _max_queue_batch_duration
self._running_execution_refresh_interval = _running_execution_refresh_interval
self._running_executions_queue_overdue = False
self._queued_execution_interceptor = queued_execution_interceptor

def run_loop(self):
while True:
sweep_again_now = False
try:
self.process_each_queue_once()
time.sleep(self._sleep_seconds_between_queue_sweeps)
sweep_again_now = self.process_each_queue_once()
except Exception as exc:
_logger.exception("Error while calling `process_each_queue_once`")
bugsnag_instrumentation.notify(exception=exc)
if not sweep_again_now:
time.sleep(self._sleep_seconds_between_queue_sweeps)

def process_each_queue_once(self):
queue_handlers = [
self.internal_process_queued_executions_queue,
self.internal_process_running_executions_queue,
]
for queue_handler in queue_handlers:
# Optimization: Process same queue multiple times so that multiple fast operations can be batched together.
# Before: Queued (fast), Running (slow), Queued (fast), Running (slow), Queued (fast), Running (slow)
# After: Queued (fast), Queued (fast), Queued (fast), Running (slow)
# After: Queued (fast), Queued (slow), Running (slow)
# Process same queue multiple times
# * For up to X milliseconds (e.g. 200ms)
# * Up to Y times (reduces blast radius of time-related bugs)
# * If the queue is not empty
# Using monotonic timer to avoid bugs when time changes
start_time_seconds = time.monotonic()
for _ in range(self._max_queue_batch_size):
try:
with self._session_factory() as session:
queue_had_items = queue_handler(session=session)
if not queue_had_items:
break
except Exception as exc:
_logger.exception(f"Error while executing {queue_handler=}")
bugsnag_instrumentation.notify(exception=exc)
if (
datetime.timedelta(
milliseconds=(time.monotonic() - start_time_seconds) * 1000
)
> self._max_queue_batch_duration
):
break
def process_each_queue_once(self) -> bool:
"""Processes each queue once; returns whether the next sweep should start without waiting."""
self._running_executions_queue_overdue = False
queued_queue_has_items = self._process_queue_batch(
self.internal_process_queued_executions_queue
)
running_queue_has_items = self._process_queue_batch(
self.internal_process_running_executions_queue
)
return queued_queue_has_items or (
running_queue_has_items and self._running_executions_queue_overdue
)

def _process_queue_batch(
self, queue_handler: typing.Callable[..., typing.Any]
) -> bool:
"""Runs a queue handler in a batch; returns whether any call in the batch found an item."""
# Optimization: Process same queue multiple times so that multiple fast operations can be batched together.
# Before: Queued (fast), Running (slow), Queued (fast), Running (slow), Queued (fast), Running (slow)
# After: Queued (fast), Queued (fast), Queued (fast), Running (slow)
# After: Queued (fast), Queued (slow), Running (slow)
# Process same queue multiple times
# * For up to X milliseconds (e.g. 200ms)
# * Up to Y times (reduces blast radius of time-related bugs)
# * If the queue is not empty
# Using monotonic timer to avoid bugs when time changes
start_time_seconds = time.monotonic()
queue_has_items = False
for _ in range(self._max_queue_batch_size):
try:
with self._session_factory() as session:
if not queue_handler(session=session):
break
queue_has_items = True
except Exception as exc:
_logger.exception(f"Error while executing {queue_handler=}")
bugsnag_instrumentation.notify(exception=exc)
if (
datetime.timedelta(
milliseconds=(time.monotonic() - start_time_seconds) * 1000
)
> self._max_queue_batch_duration
):
break
return queue_has_items

def _is_overdue(self, *, last_processed_at: datetime.datetime | None) -> bool:
"""Whether a container execution last refreshed at `last_processed_at` is due for another refresh."""
return (
last_processed_at is None
or last_processed_at
< _get_current_time() - self._running_execution_refresh_interval
)

def internal_process_queued_executions_queue(self, session: orm.Session):
_logger.debug("Selecting queued execution to process.)")
Expand Down Expand Up @@ -228,6 +256,9 @@ def internal_process_running_executions_queue(self, session: orm.Session):
return False

self._running_executions_queue_idle = False
self._running_executions_queue_overdue = self._is_overdue(
last_processed_at=running_container_execution.last_processed_at
)

execution_node_ids = list(
session.scalars(
Expand Down
159 changes: 159 additions & 0 deletions tests/test_orchestrator_sweep_pacing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
"""Tests for whether the orchestrator starts its next sweep without waiting."""

import datetime
from typing import Any, Callable
from unittest import mock

from sqlalchemy import orm
from sqlalchemy import sql

from cloud_pipelines_backend import api_server_sql
from cloud_pipelines_backend import backend_types_sql as bts
from cloud_pipelines_backend import component_structures as structures
from cloud_pipelines_backend import database_ops
from cloud_pipelines_backend import orchestrator_sql
from cloud_pipelines_backend.launchers import interfaces as launcher_interfaces


def _create_session_factory() -> Callable[[], orm.Session]:
db_engine = database_ops.create_db_engine_and_migrate_db(database_uri="sqlite://")
return lambda: orm.Session(bind=db_engine)


def _make_launched_container(launcher_data: dict[str, Any]) -> mock.MagicMock:
return mock.MagicMock(
status=launcher_interfaces.ContainerStatus.PENDING,
to_dict=lambda: dict(launcher_data),
)


def _container_task(image: str) -> structures.TaskSpec:
return structures.TaskSpec(
component_ref=structures.ComponentReference(
spec=structures.ComponentSpec(
implementation=structures.ContainerImplementation(
container=structures.ContainerSpec(image=image)
)
)
)
)


def _create_pipeline_run(
session_factory: Callable[[], orm.Session], task_count: int = 1
) -> None:
"""A pipeline run with ``task_count`` container tasks that do not share a cached container execution."""
pipeline_spec = structures.ComponentSpec(
implementation=structures.GraphImplementation(
graph=structures.GraphSpec(
tasks={
f"task{i}": _container_task(image=f"python:3.{i}")
for i in range(task_count)
}
)
),
)
api_server_sql.PipelineRunsApiService_Sql().create(
session=session_factory(),
root_task=structures.TaskSpec(
component_ref=structures.ComponentReference(spec=pipeline_spec)
),
created_by="user1",
)


def _make_orchestrator(
session_factory: Callable[[], orm.Session],
max_queue_batch_size: int = 1,
) -> orchestrator_sql.OrchestratorService_Sql:
launcher = mock.MagicMock(
launch_container_task=mock.MagicMock(
side_effect=lambda *a, **kw: _make_launched_container({"pod": "pod-0"})
),
deserialize_launched_container_from_dict=mock.MagicMock(
side_effect=_make_launched_container
),
get_refreshed_launched_container_from_dict=mock.MagicMock(
side_effect=_make_launched_container
),
)
return orchestrator_sql.OrchestratorService_Sql(
session_factory=session_factory,
launcher=launcher,
storage_provider=mock.MagicMock(),
data_root_uri="file:///tmp/artifacts",
logs_root_uri="file:///tmp/logs",
_max_queue_batch_size=max_queue_batch_size,
_max_queue_batch_duration=datetime.timedelta(minutes=1),
)


def _set_last_processed_at(
session_factory: Callable[[], orm.Session], last_processed_at: datetime.datetime
) -> None:
with session_factory() as session:
session.execute(
sql.update(bts.ContainerExecution).values(
last_processed_at=last_processed_at
)
)
session.commit()


def test_waits_when_there_is_no_work():
session_factory = _create_session_factory()

assert _make_orchestrator(session_factory).process_each_queue_once() is False


def test_sweeps_again_without_waiting_after_launching_a_queued_execution():
session_factory = _create_session_factory()
_create_pipeline_run(session_factory)

assert _make_orchestrator(session_factory).process_each_queue_once() is True


def test_waits_while_every_running_container_execution_was_refreshed_recently():
session_factory = _create_session_factory()
_create_pipeline_run(session_factory)
orchestrator = _make_orchestrator(session_factory)
orchestrator.process_each_queue_once()

assert orchestrator.process_each_queue_once() is False


def test_sweeps_again_without_waiting_while_a_running_container_execution_is_overdue():
session_factory = _create_session_factory()
_create_pipeline_run(session_factory, task_count=2)
orchestrator = _make_orchestrator(session_factory)
orchestrator.process_each_queue_once()
orchestrator.process_each_queue_once()
_set_last_processed_at(
session_factory,
datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta(minutes=5),
)

assert orchestrator.process_each_queue_once() is True


def test_waits_after_a_queue_handler_raises():
session_factory = _create_session_factory()
_create_pipeline_run(session_factory)
orchestrator = _make_orchestrator(session_factory)
with mock.patch.object(
orchestrator,
"internal_process_queued_executions_queue",
side_effect=RuntimeError("The database was unavailable"),
):
assert orchestrator.process_each_queue_once() is False


def test_sweeps_again_without_waiting_when_a_batch_raises_after_finding_work():
session_factory = _create_session_factory()
orchestrator = _make_orchestrator(session_factory, max_queue_batch_size=3)
with mock.patch.object(
orchestrator,
"internal_process_queued_executions_queue",
side_effect=[True, True, RuntimeError("The database was unavailable")],
):
assert orchestrator.process_each_queue_once() is True
Loading