diff --git a/cloud_pipelines_backend/orchestrator_sql.py b/cloud_pipelines_backend/orchestrator_sql.py index 246a607..6abbcd3 100644 --- a/cloud_pipelines_backend/orchestrator_sql.py +++ b/cloud_pipelines_backend/orchestrator_sql.py @@ -74,6 +74,9 @@ def __init__( _max_container_execution_refresh_error_retries: int = 3, _max_queue_batch_size: int = 1, _max_queue_batch_duration: datetime.timedelta = datetime.timedelta(), + _running_execution_refresh_interval: datetime.timedelta = datetime.timedelta( + seconds=30 + ), queued_execution_interceptor: QueuedExecutionInterceptor | None = None, ): self._session_factory = session_factory @@ -93,49 +96,74 @@ def __init__( self._max_queue_batch_size = _max_queue_batch_size self._max_queue_batch_duration = _max_queue_batch_duration + self._running_execution_refresh_interval = _running_execution_refresh_interval + self._running_executions_queue_overdue = False self._queued_execution_interceptor = queued_execution_interceptor def run_loop(self): while True: + sweep_again_now = False try: - self.process_each_queue_once() - time.sleep(self._sleep_seconds_between_queue_sweeps) + sweep_again_now = self.process_each_queue_once() except Exception as exc: _logger.exception("Error while calling `process_each_queue_once`") bugsnag_instrumentation.notify(exception=exc) + if not sweep_again_now: + time.sleep(self._sleep_seconds_between_queue_sweeps) - def process_each_queue_once(self): - queue_handlers = [ - self.internal_process_queued_executions_queue, - self.internal_process_running_executions_queue, - ] - for queue_handler in queue_handlers: - # Optimization: Process same queue multiple times so that multiple fast operations can be batched together. - # Before: Queued (fast), Running (slow), Queued (fast), Running (slow), Queued (fast), Running (slow) - # After: Queued (fast), Queued (fast), Queued (fast), Running (slow) - # After: Queued (fast), Queued (slow), Running (slow) - # Process same queue multiple times - # * For up to X milliseconds (e.g. 200ms) - # * Up to Y times (reduces blast radius of time-related bugs) - # * If the queue is not empty - # Using monotonic timer to avoid bugs when time changes - start_time_seconds = time.monotonic() - for _ in range(self._max_queue_batch_size): - try: - with self._session_factory() as session: - queue_had_items = queue_handler(session=session) - if not queue_had_items: - break - except Exception as exc: - _logger.exception(f"Error while executing {queue_handler=}") - bugsnag_instrumentation.notify(exception=exc) - if ( - datetime.timedelta( - milliseconds=(time.monotonic() - start_time_seconds) * 1000 - ) - > self._max_queue_batch_duration - ): - break + def process_each_queue_once(self) -> bool: + """Processes each queue once; returns whether the next sweep should start without waiting.""" + self._running_executions_queue_overdue = False + queued_queue_has_items = self._process_queue_batch( + self.internal_process_queued_executions_queue + ) + running_queue_has_items = self._process_queue_batch( + self.internal_process_running_executions_queue + ) + return queued_queue_has_items or ( + running_queue_has_items and self._running_executions_queue_overdue + ) + + def _process_queue_batch( + self, queue_handler: typing.Callable[..., typing.Any] + ) -> bool: + """Runs a queue handler in a batch; returns whether any call in the batch found an item.""" + # Optimization: Process same queue multiple times so that multiple fast operations can be batched together. + # Before: Queued (fast), Running (slow), Queued (fast), Running (slow), Queued (fast), Running (slow) + # After: Queued (fast), Queued (fast), Queued (fast), Running (slow) + # After: Queued (fast), Queued (slow), Running (slow) + # Process same queue multiple times + # * For up to X milliseconds (e.g. 200ms) + # * Up to Y times (reduces blast radius of time-related bugs) + # * If the queue is not empty + # Using monotonic timer to avoid bugs when time changes + start_time_seconds = time.monotonic() + queue_has_items = False + for _ in range(self._max_queue_batch_size): + try: + with self._session_factory() as session: + if not queue_handler(session=session): + break + queue_has_items = True + except Exception as exc: + _logger.exception(f"Error while executing {queue_handler=}") + bugsnag_instrumentation.notify(exception=exc) + if ( + datetime.timedelta( + milliseconds=(time.monotonic() - start_time_seconds) * 1000 + ) + > self._max_queue_batch_duration + ): + break + return queue_has_items + + def _is_overdue(self, *, last_processed_at: datetime.datetime | None) -> bool: + """Whether a container execution last refreshed at `last_processed_at` is due for another refresh.""" + return ( + last_processed_at is None + or last_processed_at + < _get_current_time() - self._running_execution_refresh_interval + ) def internal_process_queued_executions_queue(self, session: orm.Session): _logger.debug("Selecting queued execution to process.)") @@ -228,6 +256,9 @@ def internal_process_running_executions_queue(self, session: orm.Session): return False self._running_executions_queue_idle = False + self._running_executions_queue_overdue = self._is_overdue( + last_processed_at=running_container_execution.last_processed_at + ) execution_node_ids = list( session.scalars( diff --git a/tests/test_orchestrator_sweep_pacing.py b/tests/test_orchestrator_sweep_pacing.py new file mode 100644 index 0000000..924c93e --- /dev/null +++ b/tests/test_orchestrator_sweep_pacing.py @@ -0,0 +1,159 @@ +"""Tests for whether the orchestrator starts its next sweep without waiting.""" + +import datetime +from typing import Any, Callable +from unittest import mock + +from sqlalchemy import orm +from sqlalchemy import sql + +from cloud_pipelines_backend import api_server_sql +from cloud_pipelines_backend import backend_types_sql as bts +from cloud_pipelines_backend import component_structures as structures +from cloud_pipelines_backend import database_ops +from cloud_pipelines_backend import orchestrator_sql +from cloud_pipelines_backend.launchers import interfaces as launcher_interfaces + + +def _create_session_factory() -> Callable[[], orm.Session]: + db_engine = database_ops.create_db_engine_and_migrate_db(database_uri="sqlite://") + return lambda: orm.Session(bind=db_engine) + + +def _make_launched_container(launcher_data: dict[str, Any]) -> mock.MagicMock: + return mock.MagicMock( + status=launcher_interfaces.ContainerStatus.PENDING, + to_dict=lambda: dict(launcher_data), + ) + + +def _container_task(image: str) -> structures.TaskSpec: + return structures.TaskSpec( + component_ref=structures.ComponentReference( + spec=structures.ComponentSpec( + implementation=structures.ContainerImplementation( + container=structures.ContainerSpec(image=image) + ) + ) + ) + ) + + +def _create_pipeline_run( + session_factory: Callable[[], orm.Session], task_count: int = 1 +) -> None: + """A pipeline run with ``task_count`` container tasks that do not share a cached container execution.""" + pipeline_spec = structures.ComponentSpec( + implementation=structures.GraphImplementation( + graph=structures.GraphSpec( + tasks={ + f"task{i}": _container_task(image=f"python:3.{i}") + for i in range(task_count) + } + ) + ), + ) + api_server_sql.PipelineRunsApiService_Sql().create( + session=session_factory(), + root_task=structures.TaskSpec( + component_ref=structures.ComponentReference(spec=pipeline_spec) + ), + created_by="user1", + ) + + +def _make_orchestrator( + session_factory: Callable[[], orm.Session], + max_queue_batch_size: int = 1, +) -> orchestrator_sql.OrchestratorService_Sql: + launcher = mock.MagicMock( + launch_container_task=mock.MagicMock( + side_effect=lambda *a, **kw: _make_launched_container({"pod": "pod-0"}) + ), + deserialize_launched_container_from_dict=mock.MagicMock( + side_effect=_make_launched_container + ), + get_refreshed_launched_container_from_dict=mock.MagicMock( + side_effect=_make_launched_container + ), + ) + return orchestrator_sql.OrchestratorService_Sql( + session_factory=session_factory, + launcher=launcher, + storage_provider=mock.MagicMock(), + data_root_uri="file:///tmp/artifacts", + logs_root_uri="file:///tmp/logs", + _max_queue_batch_size=max_queue_batch_size, + _max_queue_batch_duration=datetime.timedelta(minutes=1), + ) + + +def _set_last_processed_at( + session_factory: Callable[[], orm.Session], last_processed_at: datetime.datetime +) -> None: + with session_factory() as session: + session.execute( + sql.update(bts.ContainerExecution).values( + last_processed_at=last_processed_at + ) + ) + session.commit() + + +def test_waits_when_there_is_no_work(): + session_factory = _create_session_factory() + + assert _make_orchestrator(session_factory).process_each_queue_once() is False + + +def test_sweeps_again_without_waiting_after_launching_a_queued_execution(): + session_factory = _create_session_factory() + _create_pipeline_run(session_factory) + + assert _make_orchestrator(session_factory).process_each_queue_once() is True + + +def test_waits_while_every_running_container_execution_was_refreshed_recently(): + session_factory = _create_session_factory() + _create_pipeline_run(session_factory) + orchestrator = _make_orchestrator(session_factory) + orchestrator.process_each_queue_once() + + assert orchestrator.process_each_queue_once() is False + + +def test_sweeps_again_without_waiting_while_a_running_container_execution_is_overdue(): + session_factory = _create_session_factory() + _create_pipeline_run(session_factory, task_count=2) + orchestrator = _make_orchestrator(session_factory) + orchestrator.process_each_queue_once() + orchestrator.process_each_queue_once() + _set_last_processed_at( + session_factory, + datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta(minutes=5), + ) + + assert orchestrator.process_each_queue_once() is True + + +def test_waits_after_a_queue_handler_raises(): + session_factory = _create_session_factory() + _create_pipeline_run(session_factory) + orchestrator = _make_orchestrator(session_factory) + with mock.patch.object( + orchestrator, + "internal_process_queued_executions_queue", + side_effect=RuntimeError("The database was unavailable"), + ): + assert orchestrator.process_each_queue_once() is False + + +def test_sweeps_again_without_waiting_when_a_batch_raises_after_finding_work(): + session_factory = _create_session_factory() + orchestrator = _make_orchestrator(session_factory, max_queue_batch_size=3) + with mock.patch.object( + orchestrator, + "internal_process_queued_executions_queue", + side_effect=[True, True, RuntimeError("The database was unavailable")], + ): + assert orchestrator.process_each_queue_once() is True