From 81c3c3368bf7248c682794ac387f9f7c9400c6f8 Mon Sep 17 00:00:00 2001 From: Morgan Wowk Date: Tue, 3 Nov 2026 10:59:28 -0700 Subject: [PATCH] feat(orchestrator): Prefetch a finished container's log and outputs off the loop thread Committing a SUCCEEDED or FAILED container execution uploads its log and reads every output (exists, get_info, small values) on the loop thread, which delays every other refresh behind that I/O. With the new internal _completion_prefetch_max_workers set, the first visit that sees an ended container starts that I/O on a worker thread and returns, leaving the status unchanged. When the prefetch finishes, it records the container execution as done, and the loop thread moves every done one that is still pending or running to the front of the running queue before its next selection. The next visit commits it from the prefetched results. Any call the prefetch could not make, and any output it did not find, is made inline as before, so the outcome is the same with the prefetch on or off. The visits after a prefetch starts reuse the ended container that started it rather than refreshing it again, since an ended status is final. Cancelling or a system error forgets the prefetch. shutdown_completion_prefetches() cancels prefetches that have not started and waits a bounded time for the running ones. The prefetch is off by default (0 workers). --- cloud_pipelines_backend/orchestrator_sql.py | 277 ++++++++- .../test_orchestrator_completion_prefetch.py | 587 ++++++++++++++++++ 2 files changed, 834 insertions(+), 30 deletions(-) create mode 100644 tests/test_orchestrator_completion_prefetch.py diff --git a/cloud_pipelines_backend/orchestrator_sql.py b/cloud_pipelines_backend/orchestrator_sql.py index 246a6077..6cb90370 100644 --- a/cloud_pipelines_backend/orchestrator_sql.py +++ b/cloud_pipelines_backend/orchestrator_sql.py @@ -1,9 +1,14 @@ +import concurrent.futures +import contextlib import copy +import dataclasses +import functools import json import datetime import logging import itertools import os +import threading import time import traceback import typing @@ -36,6 +41,10 @@ DYNAMIC_DATA_SECRET_KEY = "secret" DYNAMIC_DATA_SECRET_NAME_KEY = "name" +_FRONT_OF_RUNNING_QUEUE_LAST_PROCESSED_AT = datetime.datetime( + 1970, 1, 1, tzinfo=datetime.timezone.utc +) + class OrchestratorError(RuntimeError): pass @@ -72,6 +81,7 @@ def __init__( *, # Internal/experimental: _max_container_execution_refresh_error_retries: int = 3, + _completion_prefetch_max_workers: int = 0, _max_queue_batch_size: int = 1, _max_queue_batch_duration: datetime.timedelta = datetime.timedelta(), queued_execution_interceptor: QueuedExecutionInterceptor | None = None, @@ -94,6 +104,20 @@ def __init__( self._max_queue_batch_size = _max_queue_batch_size self._max_queue_batch_duration = _max_queue_batch_duration self._queued_execution_interceptor = queued_execution_interceptor + self._completion_prefetch_executor = None + if _completion_prefetch_max_workers: + self._completion_prefetch_executor = concurrent.futures.ThreadPoolExecutor( + max_workers=_completion_prefetch_max_workers, + thread_name_prefix="completion-prefetch", + ) + self._completion_prefetches: dict[ + str, concurrent.futures.Future[_CompletionPrefetch] + ] = {} + self._ended_launched_containers: dict[ + str, launcher_interfaces.LaunchedContainer + ] = {} + self._finished_completion_prefetch_ids: set[str] = set() + self._finished_completion_prefetch_ids_lock = threading.Lock() def run_loop(self): while True: @@ -198,6 +222,7 @@ def internal_process_queued_executions_queue(self, session: orm.Session): def internal_process_running_executions_queue(self, session: orm.Session): _logger.debug("Selecting running container execution to process.)") + self._move_finished_completion_prefetches_to_front(session=session) # Select only the ID to avoid loading large JSON columns into the sort buffer. query_start_timestamp = time.monotonic_ns() @@ -301,6 +326,9 @@ def internal_process_running_executions_queue(self, session: orm.Session): self._container_execution_refresh_error_counts.pop( running_container_execution.id, None ) + self._forget_completion_prefetch( + container_execution_id=running_container_execution.id + ) running_container_execution.status = ( bts.ContainerExecutionStatus.SYSTEM_ERROR ) @@ -963,6 +991,9 @@ def internal_process_one_running_execution( # We need to mark the execution as CANCELLED otherwise orchestrator will continue polling it. container_execution.status = bts.ContainerExecutionStatus.CANCELLED terminated = True + self._forget_completion_prefetch( + container_execution_id=container_execution.id + ) # Marking the execution nodes as cancelled only after the launched container is successfully terminated for execution_node in votes_to_terminate: @@ -985,10 +1016,25 @@ def internal_process_one_running_execution( if terminated: return - # Asking the launcher to refresh the container state. - reloaded_launched_container: launcher_interfaces.LaunchedContainer = ( - self._launcher.get_refreshed_launched_container_from_dict(launcher_data) + reloaded_launched_container: launcher_interfaces.LaunchedContainer | None = ( + self._ended_launched_containers.get(container_execution.id) ) + if reloaded_launched_container is None: + # Asking the launcher to refresh the container state. + reloaded_launched_container = ( + self._launcher.get_refreshed_launched_container_from_dict(launcher_data) + ) + prefetch = _CompletionPrefetch() + if ( + self._completion_prefetch_executor is not None + and reloaded_launched_container.status in _ENDED_CONTAINER_STATUSES + ): + prefetch = self._take_completion_prefetch( + container_execution=container_execution, + launched_container=reloaded_launched_container, + ) + if prefetch is None: + return current_time = _get_current_time() # Saving the updated launcher data reloaded_launcher_data = reloaded_launched_container.to_dict() @@ -1034,12 +1080,13 @@ def internal_process_one_running_execution( # Don't fail the execution if log upload fails. # Logs are important, but not so important that we should fail a successfully completed container execution. - try: - _retry(reloaded_launched_container.upload_log) - except Exception as ex: - _logger.exception( - f"! Error during `LaunchedContainer.upload_log` call: {ex}." - ) + if not prefetch.log_uploaded: + try: + _retry(reloaded_launched_container.upload_log) + except Exception as ex: + _logger.exception( + f"! Error during `LaunchedContainer.upload_log` call: {ex}." + ) _MAX_PRELOAD_VALUE_SIZE = 255 @@ -1077,8 +1124,14 @@ def _maybe_preload_value( missing_output_names = [ output_name for output_name, uri in output_artifact_uris.items() - if not _retry( - lambda: self._storage_provider.make_uri(uri).get_reader().exists() + if not _prefetched_or_call( + prefetch.output_exists, + uri, + lambda: _retry( + lambda: self._storage_provider.make_uri(uri) + .get_reader() + .exists() + ), ) ] @@ -1105,12 +1158,16 @@ def _maybe_preload_value( "orchestrator.get_output_artifact_info" ): output_artifact_data_info_map = { - output_name: _retry( - lambda: ( - self._storage_provider.make_uri(uri) - .get_reader() - .get_info() - ) + output_name: _prefetched_or_call( + prefetch.output_data_infos, + uri, + lambda: _retry( + lambda: ( + self._storage_provider.make_uri(uri) + .get_reader() + .get_info() + ) + ), ) for output_name, uri in output_artifact_uris.items() } @@ -1126,13 +1183,17 @@ def _maybe_preload_value( ][0], uri=output_artifact_uris[output_name], # Preloading artifact value is it's small enough (e.g. <=255 bytes) - value=_retry( - lambda: _maybe_get_small_artifact_value( - uri_reader=self._storage_provider.make_uri( - output_artifact_uris[output_name] - ).get_reader(), - data_info=data_info, - ) + value=_prefetched_or_call( + prefetch.output_values, + output_artifact_uris[output_name], + lambda: _retry( + lambda: _maybe_get_small_artifact_value( + uri_reader=self._storage_provider.make_uri( + output_artifact_uris[output_name] + ).get_reader(), + data_info=data_info, + ) + ), ), created_at=current_time, ) @@ -1184,12 +1245,13 @@ def _maybe_preload_value( # Logs are best-effort; losing them is never a reason to fail an # execution. The launcher decides what a lost log means; the # orchestrator only records that the upload did not succeed. - try: - _retry(reloaded_launched_container.upload_log) - except Exception as ex: - _logger.exception( - f"! Error during `LaunchedContainer.upload_log` call: {ex}." - ) + if not prefetch.log_uploaded: + try: + _retry(reloaded_launched_container.upload_log) + except Exception as ex: + _logger.exception( + f"! Error during `LaunchedContainer.upload_log` call: {ex}." + ) # Skip downstream executions for execution_node in execution_nodes: execution_node.container_execution_status = ( @@ -1234,6 +1296,161 @@ def _maybe_preload_value( f"Processing running container execution {container_execution.id}: Error doing launched container cleanup." ) + def shutdown_completion_prefetches(self, *, timeout_seconds: float) -> bool: + """Cancels the completion prefetches that have not started and waits up to `timeout_seconds` for the running ones; returns whether they all finished.""" + if self._completion_prefetch_executor is None: + return True + self._completion_prefetch_executor.shutdown(wait=False, cancel_futures=True) + _, not_done = concurrent.futures.wait( + self._completion_prefetches.values(), timeout=timeout_seconds + ) + return not not_done + + def _take_completion_prefetch( + self, + *, + container_execution: bts.ContainerExecution, + launched_container: launcher_interfaces.LaunchedContainer, + ) -> "_CompletionPrefetch | None": + """Returns and forgets the finished prefetch for this container execution, or starts one and returns None.""" + future = self._completion_prefetches.get(container_execution.id) + if future is None: + output_uris = [] + if ( + launched_container.status + == launcher_interfaces.ContainerStatus.SUCCEEDED + ): + output_uris = [ + output_artifact_info_dict["uri"] + for output_artifact_info_dict in ( + container_execution.output_artifact_data_map or {} + ).values() + ] + future = self._completion_prefetch_executor.submit( + _prefetch_completion, + launched_container=launched_container, + storage_provider=self._storage_provider, + output_uris=output_uris, + ) + future.add_done_callback( + functools.partial( + self._record_finished_completion_prefetch, container_execution.id + ) + ) + self._completion_prefetches[container_execution.id] = future + self._ended_launched_containers[container_execution.id] = launched_container + return None + if not future.done(): + return None + del self._completion_prefetches[container_execution.id] + self._ended_launched_containers.pop(container_execution.id, None) + try: + return future.result() + except Exception: + _logger.exception("Completion prefetch failed; processing without it.") + return _CompletionPrefetch() + + def _forget_completion_prefetch(self, *, container_execution_id: str) -> None: + """Forgets the prefetch of a container execution that left the running queue, cancelling it if it has not started.""" + self._ended_launched_containers.pop(container_execution_id, None) + future = self._completion_prefetches.pop(container_execution_id, None) + if future is not None: + future.cancel() + + def _record_finished_completion_prefetch( + self, + container_execution_id: str, + future: "concurrent.futures.Future[_CompletionPrefetch]", + ) -> None: + """Records that the container execution's prefetch is done, for the loop thread to move it to the front of the running queue.""" + if future.cancelled(): + return + with self._finished_completion_prefetch_ids_lock: + self._finished_completion_prefetch_ids.add(container_execution_id) + + def _move_finished_completion_prefetches_to_front( + self, *, session: orm.Session + ) -> None: + """Moves every pending or running container execution whose prefetch is done to the front of the running queue, so the next visit commits it.""" + with self._finished_completion_prefetch_ids_lock: + container_execution_ids = list(self._finished_completion_prefetch_ids) + self._finished_completion_prefetch_ids.clear() + if not container_execution_ids: + return + session.execute( + sql.update(bts.ContainerExecution) + .where( + bts.ContainerExecution.id.in_(container_execution_ids), + bts.ContainerExecution.status.in_( + ( + bts.ContainerExecutionStatus.PENDING, + bts.ContainerExecutionStatus.RUNNING, + ) + ), + ) + .values(last_processed_at=_FRONT_OF_RUNNING_QUEUE_LAST_PROCESSED_AT) + ) + session.commit() + + +_ENDED_CONTAINER_STATUSES = ( + launcher_interfaces.ContainerStatus.SUCCEEDED, + launcher_interfaces.ContainerStatus.FAILED, +) + + +@dataclasses.dataclass +class _CompletionPrefetch: + """The finished-container calls a prefetch completed; each call missing here is made inline instead.""" + + log_uploaded: bool = False + output_exists: dict[str, bool] = dataclasses.field(default_factory=dict) + output_data_infos: dict[str, storage_provider_interfaces.DataInfo] = ( + dataclasses.field(default_factory=dict) + ) + output_values: dict[str, str] = dataclasses.field(default_factory=dict) + + +def _prefetch_completion( + *, + launched_container: launcher_interfaces.LaunchedContainer, + storage_provider: storage_provider_interfaces.StorageProvider, + output_uris: list[str], +) -> _CompletionPrefetch: + """Makes the log upload and output reads that committing a finished container needs, recording each one that succeeds.""" + prefetch = _CompletionPrefetch() + with contextlib.suppress(Exception): + _retry(launched_container.upload_log) + prefetch.log_uploaded = True + + readers = {uri: storage_provider.make_uri(uri).get_reader() for uri in output_uris} + with contextlib.suppress(Exception): + for uri, reader in readers.items(): + if _retry(reader.exists): + prefetch.output_exists[uri] = True + if not all(prefetch.output_exists.get(uri) for uri in output_uris): + return prefetch + + with contextlib.suppress(Exception): + for uri, reader in readers.items(): + prefetch.output_data_infos[uri] = _retry(reader.get_info) + for uri, data_info in prefetch.output_data_infos.items(): + value = _maybe_get_small_artifact_value( + uri_reader=readers[uri], data_info=data_info + ) + if value is not None: + prefetch.output_values[uri] = value + return prefetch + + +def _prefetched_or_call( + prefetched: dict[str, _T], uri: str, call: typing.Callable[[], _T] +) -> _T: + """Returns the prefetched result for `uri`, or makes the call when the prefetch has none.""" + if uri in prefetched: + return prefetched[uri] + return call() + def _get_direct_downstream_executions( session: orm.Session, execution: bts.ExecutionNode diff --git a/tests/test_orchestrator_completion_prefetch.py b/tests/test_orchestrator_completion_prefetch.py new file mode 100644 index 00000000..390edfc6 --- /dev/null +++ b/tests/test_orchestrator_completion_prefetch.py @@ -0,0 +1,587 @@ +"""Tests for the completion prefetch. + +With the prefetch on, a finished container's log upload and output reads run on the prefetch workers, never on the loop +thread, and the container gets the same terminal outcome it gets with the prefetch off. +""" + +import concurrent.futures +import datetime +import threading +from unittest import mock + +import pytest +from cloud_pipelines.orchestration.storage_providers import ( + interfaces as storage_interfaces, +) +from sqlalchemy import orm, sql + +from cloud_pipelines_backend import api_server_sql +from cloud_pipelines_backend import backend_types_sql as bts +from cloud_pipelines_backend import component_structures as structures +from cloud_pipelines_backend import database_ops +from cloud_pipelines_backend import orchestrator_sql +from cloud_pipelines_backend.launchers import interfaces as launcher_interfaces + +_TS = datetime.datetime(2026, 1, 1, 12, 0, 0, tzinfo=datetime.timezone.utc) +_LAUNCHER_DATA = {"pod": "pod-0"} + + +@pytest.fixture(autouse=True) +def _no_retry_sleeps(): + """Patches out `_retry` sleeps, on the loop thread and the prefetch workers alike.""" + with mock.patch("time.sleep"): + yield + + +class _FakeStorage: + """A storage provider whose every output is a small file, recording which thread made each call.""" + + def __init__(self, *, exists: bool = True, error: Exception | None = None): + self.exists_result = exists + self.error = error + self.calls: list[tuple[str, int]] = [] + + def make_uri(self, uri: str) -> mock.MagicMock: + """An accessor whose reader answers for `uri`.""" + reader = mock.MagicMock() + reader.exists.side_effect = lambda: self._call("exists", self.exists_result) + reader.get_info.side_effect = lambda: self._call( + "get_info", + storage_interfaces.DataInfo( + total_size=5, is_dir=False, hashes={"md5": "abc"} + ), + ) + reader.download_as_bytes.side_effect = lambda: self._call( + "download_as_bytes", b"hello" + ) + accessor = mock.MagicMock() + accessor.get_reader.return_value = reader + return accessor + + def _call(self, name: str, result): + """Records the call and its thread, then returns `result` or raises the configured error.""" + self.calls.append((name, threading.get_ident())) + if self.error is not None: + raise self.error + return result + + +def _container_component( + *, inputs: list[str] | None = None, outputs: list[str] | None = None +) -> structures.ComponentSpec: + """A container component with the given input and output names.""" + return structures.ComponentSpec( + inputs=[structures.InputSpec(name=name) for name in (inputs or [])], + outputs=[structures.OutputSpec(name=name) for name in (outputs or [])], + implementation=structures.ContainerImplementation( + container=structures.ContainerSpec(image="python") + ), + ) + + +def _upstream_downstream_pipeline() -> structures.TaskSpec: + """`upstream` (produces `out`) feeding `downstream` (consumes `in`).""" + pipeline_spec = structures.ComponentSpec( + implementation=structures.GraphImplementation( + graph=structures.GraphSpec( + tasks={ + "upstream": structures.TaskSpec( + component_ref=structures.ComponentReference( + spec=_container_component(outputs=["out"]) + ), + ), + "downstream": structures.TaskSpec( + component_ref=structures.ComponentReference( + spec=_container_component(inputs=["in"]) + ), + arguments={ + "in": structures.TaskOutputArgument( + task_output=structures.TaskOutputReference( + task_id="upstream", output_name="out" + ) + ) + }, + ), + } + ) + ), + ) + return structures.TaskSpec( + component_ref=structures.ComponentReference(spec=pipeline_spec) + ) + + +def _launched_container( + status: launcher_interfaces.ContainerStatus, +) -> mock.MagicMock: + """A launched container in `status` that records the thread of each log upload.""" + container = mock.MagicMock( + status=status, + exit_code=0, + started_at=_TS, + ended_at=_TS, + launcher_error_message=None, + to_dict=lambda: dict(_LAUNCHER_DATA), + ) + container.upload_threads = [] + container.upload_log.side_effect = lambda: container.upload_threads.append( + threading.get_ident() + ) + return container + + +class _Scenario: + """One pipeline whose upstream container finishes as `ended_container`, with the completion prefetch on or off.""" + + def __init__( + self, + *, + prefetch: bool, + ended_container: mock.MagicMock, + storage: _FakeStorage, + ): + db_engine = database_ops.create_db_engine_and_migrate_db( + database_uri="sqlite://" + ) + self.session_factory = lambda: orm.Session(bind=db_engine) + api_server_sql.PipelineRunsApiService_Sql().create( + session=self.session_factory(), + root_task=_upstream_downstream_pipeline(), + created_by="user1", + ) + launcher = mock.MagicMock() + launcher.launch_container_task.side_effect = lambda *a, **kw: ( + _launched_container(launcher_interfaces.ContainerStatus.PENDING) + ) + launcher.deserialize_launched_container_from_dict.side_effect = ( + lambda data: _launched_container( + launcher_interfaces.ContainerStatus.PENDING + ) + ) + launcher.get_refreshed_launched_container_from_dict.side_effect = ( + lambda data: ended_container + ) + self.launcher = launcher + self.orchestrator = orchestrator_sql.OrchestratorService_Sql( + session_factory=self.session_factory, + launcher=launcher, + storage_provider=storage, + data_root_uri="file:///tmp/artifacts", + logs_root_uri="file:///tmp/logs", + _completion_prefetch_max_workers=1 if prefetch else 0, + ) + session = self.session_factory() + while self.orchestrator.internal_process_queued_executions_queue( + session=session + ): + pass + + def visit_running(self) -> None: + """Processes the running queue once.""" + self.orchestrator.internal_process_running_executions_queue( + session=self.session_factory() + ) + + def wait_for_prefetches(self) -> None: + """Blocks until every in-flight prefetch and its done callback have finished, then starts a fresh executor.""" + if self.orchestrator._completion_prefetch_executor is None: + return + self.orchestrator._completion_prefetch_executor.shutdown(wait=True) + self.orchestrator._completion_prefetch_executor = ( + concurrent.futures.ThreadPoolExecutor(max_workers=1) + ) + + def run_to_completion(self) -> None: + """Visits the ended container until it leaves PENDING, at most three times.""" + for _ in range(3): + self.visit_running() + if self.upstream_status() != bts.ContainerExecutionStatus.PENDING: + return + self.wait_for_prefetches() + + def node(self, task_id: str) -> bts.ExecutionNode: + """The execution node of `task_id`.""" + return self.session_factory().scalar( + sql.select(bts.ExecutionNode).where( + bts.ExecutionNode.task_id_in_parent_execution == task_id + ) + ) + + def upstream_status(self) -> bts.ContainerExecutionStatus: + """The upstream node's status.""" + return self.node("upstream").container_execution_status + + def upstream_last_processed_at(self) -> datetime.datetime: + """When the upstream container execution was last processed.""" + session = self.session_factory() + return session.get( + bts.ContainerExecution, self.node("upstream").container_execution_id + ).last_processed_at + + def set_upstream_last_processed_at(self, value: datetime.datetime) -> None: + """Sets when the upstream container execution was last processed.""" + with self.session_factory() as session: + session.get( + bts.ContainerExecution, + self.node("upstream").container_execution_id, + ).last_processed_at = value + session.commit() + + def outcome(self) -> dict: + """Everything the finished container's processing decides.""" + session = self.session_factory() + upstream = self.node("upstream") + container_execution = session.get( + bts.ContainerExecution, upstream.container_execution_id + ) + artifacts = [ + (data.total_size, data.is_dir, data.hash, data.value) + for data in session.scalars(sql.select(bts.ArtifactData)).all() + if data.uri + in [ + info["uri"] + for info in container_execution.output_artifact_data_map.values() + ] + ] + return { + "upstream": upstream.container_execution_status, + "downstream": self.node("downstream").container_execution_status, + "container": container_execution.status, + "error": (container_execution.extra_data or {}).get( + bts.CONTAINER_EXECUTION_EXTRA_DATA_ORCHESTRATION_ERROR_MESSAGE_KEY + ), + "artifacts": artifacts, + } + + +def _run(*, prefetch: bool, status, storage) -> _Scenario: + """Runs the scenario until the ended upstream container gets its terminal status.""" + scenario = _Scenario( + prefetch=prefetch, + ended_container=_launched_container(status), + storage=storage, + ) + scenario.run_to_completion() + return scenario + + +class TestPrefetchedCompletion: + def test_first_visit_starts_the_prefetch_and_keeps_the_status(self): + storage = _FakeStorage() + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=storage, + ) + with mock.patch.object( + scenario.orchestrator, "_completion_prefetch_executor" + ) as executor: + scenario.visit_running() + + executor.submit.assert_called_once() + assert scenario.upstream_status() == bts.ContainerExecutionStatus.PENDING + assert storage.calls == [] + + def test_terminal_status_lands_on_the_visit_after_the_prefetch(self): + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=_FakeStorage(), + ) + + scenario.visit_running() + assert scenario.upstream_status() == bts.ContainerExecutionStatus.PENDING + scenario.wait_for_prefetches() + scenario.visit_running() + + assert scenario.upstream_status() == bts.ContainerExecutionStatus.SUCCEEDED + assert scenario.orchestrator._completion_prefetches == {} + + def test_finished_prefetch_moves_the_execution_to_the_front_of_the_queue_on_the_loop_thread( + self, + ): + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=_FakeStorage(), + ) + + scenario.visit_running() + scenario.wait_for_prefetches() + assert scenario.upstream_last_processed_at().year != 1970 + scenario.orchestrator._move_finished_completion_prefetches_to_front( + session=scenario.session_factory() + ) + + assert scenario.upstream_last_processed_at().year == 1970 + + def test_output_that_was_missing_during_the_prefetch_is_read_again(self): + storage = _FakeStorage(exists=False) + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=storage, + ) + + scenario.visit_running() + scenario.wait_for_prefetches() + storage.exists_result = True + scenario.visit_running() + + assert scenario.upstream_status() == bts.ContainerExecutionStatus.SUCCEEDED + + def test_commits_the_ended_container_without_refreshing_it_again(self): + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=_FakeStorage(), + ) + scenario.visit_running() + scenario.wait_for_prefetches() + scenario.launcher.get_refreshed_launched_container_from_dict.side_effect = ( + RuntimeError("The pod is gone") + ) + + scenario.visit_running() + + assert scenario.upstream_status() == bts.ContainerExecutionStatus.SUCCEEDED + assert scenario.orchestrator._ended_launched_containers == {} + + def test_system_error_forgets_the_prefetch(self): + ended_container = _launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ) + scenario = _Scenario( + prefetch=True, ended_container=ended_container, storage=_FakeStorage() + ) + scenario.visit_running() + scenario.wait_for_prefetches() + + def _fail_to_serialize(): + raise RuntimeError("The launcher data could not be serialized") + + ended_container.to_dict = _fail_to_serialize + + scenario.visit_running() + + assert scenario.upstream_status() == bts.ContainerExecutionStatus.SYSTEM_ERROR + assert scenario.orchestrator._completion_prefetches == {} + assert scenario.orchestrator._ended_launched_containers == {} + + def test_pending_prefetch_visit_moves_the_execution_back_in_the_queue(self): + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=_FakeStorage(), + ) + scenario.orchestrator._completion_prefetches[ + scenario.node("upstream").container_execution_id + ] = mock.MagicMock(**{"done.return_value": False}) + scenario.set_upstream_last_processed_at( + datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) + ) + + scenario.visit_running() + + assert scenario.upstream_last_processed_at().year != 1970 + + def test_shutdown_stops_new_prefetches(self): + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=_FakeStorage(), + ) + + scenario.orchestrator.shutdown_completion_prefetches(timeout_seconds=0) + + with pytest.raises(RuntimeError): + scenario.orchestrator._completion_prefetch_executor.submit(lambda: None) + + def test_shutdown_reports_a_prefetch_still_running(self): + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=_FakeStorage(), + ) + release = threading.Event() + scenario.orchestrator._completion_prefetches["hung"] = ( + scenario.orchestrator._completion_prefetch_executor.submit(release.wait) + ) + + try: + assert not scenario.orchestrator.shutdown_completion_prefetches( + timeout_seconds=0.05 + ) + finally: + release.set() + + def test_shutdown_reports_finished_prefetches(self): + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=_FakeStorage(), + ) + scenario.visit_running() + assert scenario.orchestrator._completion_prefetches + + assert scenario.orchestrator.shutdown_completion_prefetches(timeout_seconds=5) + + def test_loop_thread_does_no_storage_or_log_io(self): + storage = _FakeStorage() + scenario = _run( + prefetch=True, + status=launcher_interfaces.ContainerStatus.SUCCEEDED, + storage=storage, + ) + ended_container = scenario.launcher.get_refreshed_launched_container_from_dict( + None + ) + loop_thread = threading.get_ident() + + assert sorted({name for name, _ in storage.calls}) == [ + "download_as_bytes", + "exists", + "get_info", + ] + assert all(thread != loop_thread for _, thread in storage.calls) + assert ended_container.upload_threads + assert loop_thread not in ended_container.upload_threads + + def test_refreshes_once_per_visit(self): + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.RUNNING + ), + storage=_FakeStorage(), + ) + + scenario.visit_running() + + assert ( + scenario.launcher.get_refreshed_launched_container_from_dict.call_count == 1 + ) + assert scenario.upstream_status() == bts.ContainerExecutionStatus.RUNNING + + +@pytest.mark.parametrize( + ("status", "storage_kwargs"), + [ + (launcher_interfaces.ContainerStatus.SUCCEEDED, {}), + (launcher_interfaces.ContainerStatus.SUCCEEDED, {"exists": False}), + (launcher_interfaces.ContainerStatus.FAILED, {}), + ( + launcher_interfaces.ContainerStatus.SUCCEEDED, + {"error": RuntimeError("storage is down")}, + ), + ], + ids=["succeeded", "missing-outputs", "failed", "storage-errors"], +) +def test_outcome_matches_the_orchestrator_without_the_prefetch(status, storage_kwargs): + without_prefetch = _run( + prefetch=False, + status=status, + storage=_FakeStorage(**storage_kwargs), + ) + with_prefetch = _run( + prefetch=True, + status=status, + storage=_FakeStorage(**storage_kwargs), + ) + + assert with_prefetch.outcome() == without_prefetch.outcome() + assert with_prefetch.upstream_status() != bts.ContainerExecutionStatus.PENDING + + +def test_failed_log_upload_is_retried_inline(): + container = _launched_container(launcher_interfaces.ContainerStatus.FAILED) + container.upload_log.side_effect = RuntimeError("pod log is gone") + scenario = _Scenario( + prefetch=True, + ended_container=container, + storage=_FakeStorage(), + ) + + scenario.run_to_completion() + + assert scenario.upstream_status() == bts.ContainerExecutionStatus.FAILED + assert container.upload_log.call_count == 10 + + +def test_cancelling_forgets_the_prefetch(): + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=_FakeStorage(), + ) + scenario.visit_running() + scenario.wait_for_prefetches() + with scenario.session_factory() as session: + upstream = scenario.node("upstream") + session.get(bts.ExecutionNode, upstream.id).extra_data = { + "desired_state": "TERMINATED" + } + session.commit() + + scenario.visit_running() + + assert scenario.upstream_status() == bts.ContainerExecutionStatus.CANCELLED + assert scenario.orchestrator._completion_prefetches == {} + + +def test_finished_prefetch_does_not_move_an_ended_execution(): + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=_FakeStorage(), + ) + scenario.visit_running() + scenario.wait_for_prefetches() + scenario.visit_running() + assert scenario.upstream_status() == bts.ContainerExecutionStatus.SUCCEEDED + scenario.orchestrator._finished_completion_prefetch_ids.add( + scenario.node("upstream").container_execution_id + ) + + scenario.orchestrator._move_finished_completion_prefetches_to_front( + session=scenario.session_factory() + ) + + assert scenario.upstream_last_processed_at().year != 1970 + + +def test_without_the_prefetch_the_first_visit_commits_the_status(): + scenario = _Scenario( + prefetch=False, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=_FakeStorage(), + ) + + scenario.visit_running() + + assert scenario.upstream_status() == bts.ContainerExecutionStatus.SUCCEEDED + assert scenario.orchestrator.shutdown_completion_prefetches(timeout_seconds=0)