diff --git a/cloud_pipelines_backend/api_server_sql.py b/cloud_pipelines_backend/api_server_sql.py index 6ca629f..8761fcd 100644 --- a/cloud_pipelines_backend/api_server_sql.py +++ b/cloud_pipelines_backend/api_server_sql.py @@ -25,6 +25,10 @@ T = typing.TypeVar("T") +_CANCELLED_CONTAINER_EXECUTION_LAST_PROCESSED_AT = datetime.datetime( + 1970, 1, 1, tzinfo=datetime.timezone.utc +) + class ApiServiceError(RuntimeError): pass @@ -243,6 +247,10 @@ def terminate( if execution_node.extra_data is None: execution_node.extra_data = {} execution_node.extra_data["desired_state"] = "TERMINATED" + if execution_node.container_execution is not None: + execution_node.container_execution.last_processed_at = ( + _CANCELLED_CONTAINER_EXECUTION_LAST_PROCESSED_AT + ) session.commit() # Note: This method must be last to not shadow the "list" type diff --git a/cloud_pipelines_backend/orchestrator_sql.py b/cloud_pipelines_backend/orchestrator_sql.py index 6abbcd3..e0dc8fa 100644 --- a/cloud_pipelines_backend/orchestrator_sql.py +++ b/cloud_pipelines_backend/orchestrator_sql.py @@ -75,7 +75,7 @@ def __init__( _max_queue_batch_size: int = 1, _max_queue_batch_duration: datetime.timedelta = datetime.timedelta(), _running_execution_refresh_interval: datetime.timedelta = datetime.timedelta( - seconds=30 + seconds=60 ), queued_execution_interceptor: QueuedExecutionInterceptor | None = None, ): @@ -97,7 +97,6 @@ def __init__( self._max_queue_batch_size = _max_queue_batch_size self._max_queue_batch_duration = _max_queue_batch_duration self._running_execution_refresh_interval = _running_execution_refresh_interval - self._running_executions_queue_overdue = False self._queued_execution_interceptor = queued_execution_interceptor def run_loop(self): @@ -113,16 +112,13 @@ def run_loop(self): def process_each_queue_once(self) -> bool: """Processes each queue once; returns whether the next sweep should start without waiting.""" - self._running_executions_queue_overdue = False queued_queue_has_items = self._process_queue_batch( self.internal_process_queued_executions_queue ) running_queue_has_items = self._process_queue_batch( self.internal_process_running_executions_queue ) - return queued_queue_has_items or ( - running_queue_has_items and self._running_executions_queue_overdue - ) + return queued_queue_has_items or running_queue_has_items def _process_queue_batch( self, queue_handler: typing.Callable[..., typing.Any] @@ -157,14 +153,6 @@ def _process_queue_batch( break return queue_has_items - def _is_overdue(self, *, last_processed_at: datetime.datetime | None) -> bool: - """Whether a container execution last refreshed at `last_processed_at` is due for another refresh.""" - return ( - last_processed_at is None - or last_processed_at - < _get_current_time() - self._running_execution_refresh_interval - ) - def internal_process_queued_executions_queue(self, session: orm.Session): _logger.debug("Selecting queued execution to process.)") @@ -237,7 +225,12 @@ def internal_process_running_executions_queue(self, session: orm.Session): bts.ContainerExecutionStatus.PENDING, bts.ContainerExecutionStatus.RUNNING, ) - ) + ), + sql.or_( + bts.ContainerExecution.last_processed_at.is_(None), + bts.ContainerExecution.last_processed_at + < _get_current_time() - self._running_execution_refresh_interval, + ), ) .order_by(bts.ContainerExecution.last_processed_at) .limit(1) @@ -256,9 +249,6 @@ def internal_process_running_executions_queue(self, session: orm.Session): return False self._running_executions_queue_idle = False - self._running_executions_queue_overdue = self._is_overdue( - last_processed_at=running_container_execution.last_processed_at - ) execution_node_ids = list( session.scalars( diff --git a/tests/test_container_execution_refresh_retries.py b/tests/test_container_execution_refresh_retries.py index 3dd2c1b..ec3bb02 100644 --- a/tests/test_container_execution_refresh_retries.py +++ b/tests/test_container_execution_refresh_retries.py @@ -1,5 +1,6 @@ """Tests for the refresh-failure budget in the running-executions queue.""" +import datetime from typing import Any, Callable from unittest import mock @@ -124,6 +125,7 @@ def _make_orchestrator( data_root_uri="file:///tmp/artifacts", logs_root_uri="file:///tmp/logs", _max_container_execution_refresh_error_retries=max_failures, + _running_execution_refresh_interval=datetime.timedelta(0), ) diff --git a/tests/test_orchestrator_failed_log_upload.py b/tests/test_orchestrator_failed_log_upload.py index 6198fd4..f2ff3b9 100644 --- a/tests/test_orchestrator_failed_log_upload.py +++ b/tests/test_orchestrator_failed_log_upload.py @@ -133,6 +133,7 @@ def test_failed_execution_survives_log_upload_failure() -> None: storage_provider=mock.MagicMock(), data_root_uri="file:///tmp/artifacts", logs_root_uri="file:///tmp/logs", + _running_execution_refresh_interval=datetime.timedelta(0), ) # Launch the queued executions: `upstream` becomes PENDING, `downstream` stays diff --git a/tests/test_orchestrator_sql.py b/tests/test_orchestrator_sql.py index bd75704..5d99ad7 100644 --- a/tests/test_orchestrator_sql.py +++ b/tests/test_orchestrator_sql.py @@ -113,6 +113,7 @@ def _make_orchestrator( data_root_uri="file:///tmp/artifacts", logs_root_uri="file:///tmp/logs", queued_execution_interceptor=queued_execution_interceptor, + _running_execution_refresh_interval=datetime.timedelta(0), ) diff --git a/tests/test_orchestrator_sweep_pacing.py b/tests/test_orchestrator_sweep_pacing.py index 924c93e..4f438fa 100644 --- a/tests/test_orchestrator_sweep_pacing.py +++ b/tests/test_orchestrator_sweep_pacing.py @@ -157,3 +157,62 @@ def test_sweeps_again_without_waiting_when_a_batch_raises_after_finding_work(): side_effect=[True, True, RuntimeError("The database was unavailable")], ): assert orchestrator.process_each_queue_once() is True + + +def test_skips_a_container_execution_refreshed_within_the_interval(): + session_factory = _create_session_factory() + _create_pipeline_run(session_factory) + orchestrator = _make_orchestrator(session_factory) + orchestrator.process_each_queue_once() + _set_last_processed_at( + session_factory, + datetime.datetime.now(tz=datetime.timezone.utc) + - datetime.timedelta(seconds=59), + ) + + assert ( + orchestrator.internal_process_running_executions_queue( + session=session_factory() + ) + is False + ) + orchestrator._launcher.get_refreshed_launched_container_from_dict.assert_not_called() + + +def test_refreshes_a_container_execution_once_the_interval_passed(): + session_factory = _create_session_factory() + _create_pipeline_run(session_factory) + orchestrator = _make_orchestrator(session_factory) + orchestrator.process_each_queue_once() + _set_last_processed_at( + session_factory, + datetime.datetime.now(tz=datetime.timezone.utc) + - datetime.timedelta(seconds=61), + ) + + assert ( + orchestrator.internal_process_running_executions_queue( + session=session_factory() + ) + is True + ) + orchestrator._launcher.get_refreshed_launched_container_from_dict.assert_called_once() + + +def test_cancelling_a_run_refreshes_its_container_execution_without_waiting(): + session_factory = _create_session_factory() + _create_pipeline_run(session_factory) + orchestrator = _make_orchestrator(session_factory) + orchestrator.process_each_queue_once() + with session_factory() as session: + pipeline_run_id = session.scalar(sql.select(bts.PipelineRun.id)) + api_server_sql.PipelineRunsApiService_Sql().terminate( + session=session, id=pipeline_run_id, skip_user_check=True + ) + + assert ( + orchestrator.internal_process_running_executions_queue( + session=session_factory() + ) + is True + )