From 89e3dbb4311ba63440fc120a2cff42a140bdde40 Mon Sep 17 00:00:00 2001 From: Morgan Wowk Date: Tue, 17 Nov 2026 17:22:30 -0700 Subject: [PATCH] feat(orchestrator): Refresh each running container execution at most once a minute unless it changed The running queue visited the oldest active row on every sweep, even when it had just been refreshed, so visits, DB queries and launcher API calls grew with sweep rate rather than with what changed. The running-queue select now skips rows refreshed within _running_execution_refresh_interval, whose default goes from 30s to 60s. Rows that need attention still go first: - rows a launcher reported as changed are moved further back than 60s; - finished completion prefetches are moved to 1970-01-01; - cancelling a run now moves its active container executions to 1970-01-01, so termination does not wait for the interval. Because every selected row is now overdue, a sweep that refreshed anything starts the next one without waiting, and the separate overdue check is gone. Changes nobody reports (launchers without change notifications, failed refresh retries) wait up to the interval. --- cloud_pipelines_backend/api_server_sql.py | 8 +++ cloud_pipelines_backend/orchestrator_sql.py | 26 +++----- ...est_container_execution_refresh_retries.py | 2 + tests/test_orchestrator_failed_log_upload.py | 1 + tests/test_orchestrator_sql.py | 1 + tests/test_orchestrator_sweep_pacing.py | 59 +++++++++++++++++++ 6 files changed, 79 insertions(+), 18 deletions(-) diff --git a/cloud_pipelines_backend/api_server_sql.py b/cloud_pipelines_backend/api_server_sql.py index 6ca629fa..8761fcd9 100644 --- a/cloud_pipelines_backend/api_server_sql.py +++ b/cloud_pipelines_backend/api_server_sql.py @@ -25,6 +25,10 @@ T = typing.TypeVar("T") +_CANCELLED_CONTAINER_EXECUTION_LAST_PROCESSED_AT = datetime.datetime( + 1970, 1, 1, tzinfo=datetime.timezone.utc +) + class ApiServiceError(RuntimeError): pass @@ -243,6 +247,10 @@ def terminate( if execution_node.extra_data is None: execution_node.extra_data = {} execution_node.extra_data["desired_state"] = "TERMINATED" + if execution_node.container_execution is not None: + execution_node.container_execution.last_processed_at = ( + _CANCELLED_CONTAINER_EXECUTION_LAST_PROCESSED_AT + ) session.commit() # Note: This method must be last to not shadow the "list" type diff --git a/cloud_pipelines_backend/orchestrator_sql.py b/cloud_pipelines_backend/orchestrator_sql.py index 6abbcd3e..e0dc8fa3 100644 --- a/cloud_pipelines_backend/orchestrator_sql.py +++ b/cloud_pipelines_backend/orchestrator_sql.py @@ -75,7 +75,7 @@ def __init__( _max_queue_batch_size: int = 1, _max_queue_batch_duration: datetime.timedelta = datetime.timedelta(), _running_execution_refresh_interval: datetime.timedelta = datetime.timedelta( - seconds=30 + seconds=60 ), queued_execution_interceptor: QueuedExecutionInterceptor | None = None, ): @@ -97,7 +97,6 @@ def __init__( self._max_queue_batch_size = _max_queue_batch_size self._max_queue_batch_duration = _max_queue_batch_duration self._running_execution_refresh_interval = _running_execution_refresh_interval - self._running_executions_queue_overdue = False self._queued_execution_interceptor = queued_execution_interceptor def run_loop(self): @@ -113,16 +112,13 @@ def run_loop(self): def process_each_queue_once(self) -> bool: """Processes each queue once; returns whether the next sweep should start without waiting.""" - self._running_executions_queue_overdue = False queued_queue_has_items = self._process_queue_batch( self.internal_process_queued_executions_queue ) running_queue_has_items = self._process_queue_batch( self.internal_process_running_executions_queue ) - return queued_queue_has_items or ( - running_queue_has_items and self._running_executions_queue_overdue - ) + return queued_queue_has_items or running_queue_has_items def _process_queue_batch( self, queue_handler: typing.Callable[..., typing.Any] @@ -157,14 +153,6 @@ def _process_queue_batch( break return queue_has_items - def _is_overdue(self, *, last_processed_at: datetime.datetime | None) -> bool: - """Whether a container execution last refreshed at `last_processed_at` is due for another refresh.""" - return ( - last_processed_at is None - or last_processed_at - < _get_current_time() - self._running_execution_refresh_interval - ) - def internal_process_queued_executions_queue(self, session: orm.Session): _logger.debug("Selecting queued execution to process.)") @@ -237,7 +225,12 @@ def internal_process_running_executions_queue(self, session: orm.Session): bts.ContainerExecutionStatus.PENDING, bts.ContainerExecutionStatus.RUNNING, ) - ) + ), + sql.or_( + bts.ContainerExecution.last_processed_at.is_(None), + bts.ContainerExecution.last_processed_at + < _get_current_time() - self._running_execution_refresh_interval, + ), ) .order_by(bts.ContainerExecution.last_processed_at) .limit(1) @@ -256,9 +249,6 @@ def internal_process_running_executions_queue(self, session: orm.Session): return False self._running_executions_queue_idle = False - self._running_executions_queue_overdue = self._is_overdue( - last_processed_at=running_container_execution.last_processed_at - ) execution_node_ids = list( session.scalars( diff --git a/tests/test_container_execution_refresh_retries.py b/tests/test_container_execution_refresh_retries.py index 3dd2c1b6..ec3bb028 100644 --- a/tests/test_container_execution_refresh_retries.py +++ b/tests/test_container_execution_refresh_retries.py @@ -1,5 +1,6 @@ """Tests for the refresh-failure budget in the running-executions queue.""" +import datetime from typing import Any, Callable from unittest import mock @@ -124,6 +125,7 @@ def _make_orchestrator( data_root_uri="file:///tmp/artifacts", logs_root_uri="file:///tmp/logs", _max_container_execution_refresh_error_retries=max_failures, + _running_execution_refresh_interval=datetime.timedelta(0), ) diff --git a/tests/test_orchestrator_failed_log_upload.py b/tests/test_orchestrator_failed_log_upload.py index 6198fd40..f2ff3b9a 100644 --- a/tests/test_orchestrator_failed_log_upload.py +++ b/tests/test_orchestrator_failed_log_upload.py @@ -133,6 +133,7 @@ def test_failed_execution_survives_log_upload_failure() -> None: storage_provider=mock.MagicMock(), data_root_uri="file:///tmp/artifacts", logs_root_uri="file:///tmp/logs", + _running_execution_refresh_interval=datetime.timedelta(0), ) # Launch the queued executions: `upstream` becomes PENDING, `downstream` stays diff --git a/tests/test_orchestrator_sql.py b/tests/test_orchestrator_sql.py index bd757042..5d99ad74 100644 --- a/tests/test_orchestrator_sql.py +++ b/tests/test_orchestrator_sql.py @@ -113,6 +113,7 @@ def _make_orchestrator( data_root_uri="file:///tmp/artifacts", logs_root_uri="file:///tmp/logs", queued_execution_interceptor=queued_execution_interceptor, + _running_execution_refresh_interval=datetime.timedelta(0), ) diff --git a/tests/test_orchestrator_sweep_pacing.py b/tests/test_orchestrator_sweep_pacing.py index 924c93e5..4f438fa3 100644 --- a/tests/test_orchestrator_sweep_pacing.py +++ b/tests/test_orchestrator_sweep_pacing.py @@ -157,3 +157,62 @@ def test_sweeps_again_without_waiting_when_a_batch_raises_after_finding_work(): side_effect=[True, True, RuntimeError("The database was unavailable")], ): assert orchestrator.process_each_queue_once() is True + + +def test_skips_a_container_execution_refreshed_within_the_interval(): + session_factory = _create_session_factory() + _create_pipeline_run(session_factory) + orchestrator = _make_orchestrator(session_factory) + orchestrator.process_each_queue_once() + _set_last_processed_at( + session_factory, + datetime.datetime.now(tz=datetime.timezone.utc) + - datetime.timedelta(seconds=59), + ) + + assert ( + orchestrator.internal_process_running_executions_queue( + session=session_factory() + ) + is False + ) + orchestrator._launcher.get_refreshed_launched_container_from_dict.assert_not_called() + + +def test_refreshes_a_container_execution_once_the_interval_passed(): + session_factory = _create_session_factory() + _create_pipeline_run(session_factory) + orchestrator = _make_orchestrator(session_factory) + orchestrator.process_each_queue_once() + _set_last_processed_at( + session_factory, + datetime.datetime.now(tz=datetime.timezone.utc) + - datetime.timedelta(seconds=61), + ) + + assert ( + orchestrator.internal_process_running_executions_queue( + session=session_factory() + ) + is True + ) + orchestrator._launcher.get_refreshed_launched_container_from_dict.assert_called_once() + + +def test_cancelling_a_run_refreshes_its_container_execution_without_waiting(): + session_factory = _create_session_factory() + _create_pipeline_run(session_factory) + orchestrator = _make_orchestrator(session_factory) + orchestrator.process_each_queue_once() + with session_factory() as session: + pipeline_run_id = session.scalar(sql.select(bts.PipelineRun.id)) + api_server_sql.PipelineRunsApiService_Sql().terminate( + session=session, id=pipeline_run_id, skip_user_check=True + ) + + assert ( + orchestrator.internal_process_running_executions_queue( + session=session_factory() + ) + is True + )