diff --git a/cloud_pipelines_backend/orchestrator_sql.py b/cloud_pipelines_backend/orchestrator_sql.py index 246a607..6cb9037 100644 --- a/cloud_pipelines_backend/orchestrator_sql.py +++ b/cloud_pipelines_backend/orchestrator_sql.py @@ -1,9 +1,14 @@ +import concurrent.futures +import contextlib import copy +import dataclasses +import functools import json import datetime import logging import itertools import os +import threading import time import traceback import typing @@ -36,6 +41,10 @@ DYNAMIC_DATA_SECRET_KEY = "secret" DYNAMIC_DATA_SECRET_NAME_KEY = "name" +_FRONT_OF_RUNNING_QUEUE_LAST_PROCESSED_AT = datetime.datetime( + 1970, 1, 1, tzinfo=datetime.timezone.utc +) + class OrchestratorError(RuntimeError): pass @@ -72,6 +81,7 @@ def __init__( *, # Internal/experimental: _max_container_execution_refresh_error_retries: int = 3, + _completion_prefetch_max_workers: int = 0, _max_queue_batch_size: int = 1, _max_queue_batch_duration: datetime.timedelta = datetime.timedelta(), queued_execution_interceptor: QueuedExecutionInterceptor | None = None, @@ -94,6 +104,20 @@ def __init__( self._max_queue_batch_size = _max_queue_batch_size self._max_queue_batch_duration = _max_queue_batch_duration self._queued_execution_interceptor = queued_execution_interceptor + self._completion_prefetch_executor = None + if _completion_prefetch_max_workers: + self._completion_prefetch_executor = concurrent.futures.ThreadPoolExecutor( + max_workers=_completion_prefetch_max_workers, + thread_name_prefix="completion-prefetch", + ) + self._completion_prefetches: dict[ + str, concurrent.futures.Future[_CompletionPrefetch] + ] = {} + self._ended_launched_containers: dict[ + str, launcher_interfaces.LaunchedContainer + ] = {} + self._finished_completion_prefetch_ids: set[str] = set() + self._finished_completion_prefetch_ids_lock = threading.Lock() def run_loop(self): while True: @@ -198,6 +222,7 @@ def internal_process_queued_executions_queue(self, session: orm.Session): def internal_process_running_executions_queue(self, session: orm.Session): _logger.debug("Selecting running container execution to process.)") + self._move_finished_completion_prefetches_to_front(session=session) # Select only the ID to avoid loading large JSON columns into the sort buffer. query_start_timestamp = time.monotonic_ns() @@ -301,6 +326,9 @@ def internal_process_running_executions_queue(self, session: orm.Session): self._container_execution_refresh_error_counts.pop( running_container_execution.id, None ) + self._forget_completion_prefetch( + container_execution_id=running_container_execution.id + ) running_container_execution.status = ( bts.ContainerExecutionStatus.SYSTEM_ERROR ) @@ -963,6 +991,9 @@ def internal_process_one_running_execution( # We need to mark the execution as CANCELLED otherwise orchestrator will continue polling it. container_execution.status = bts.ContainerExecutionStatus.CANCELLED terminated = True + self._forget_completion_prefetch( + container_execution_id=container_execution.id + ) # Marking the execution nodes as cancelled only after the launched container is successfully terminated for execution_node in votes_to_terminate: @@ -985,10 +1016,25 @@ def internal_process_one_running_execution( if terminated: return - # Asking the launcher to refresh the container state. - reloaded_launched_container: launcher_interfaces.LaunchedContainer = ( - self._launcher.get_refreshed_launched_container_from_dict(launcher_data) + reloaded_launched_container: launcher_interfaces.LaunchedContainer | None = ( + self._ended_launched_containers.get(container_execution.id) ) + if reloaded_launched_container is None: + # Asking the launcher to refresh the container state. + reloaded_launched_container = ( + self._launcher.get_refreshed_launched_container_from_dict(launcher_data) + ) + prefetch = _CompletionPrefetch() + if ( + self._completion_prefetch_executor is not None + and reloaded_launched_container.status in _ENDED_CONTAINER_STATUSES + ): + prefetch = self._take_completion_prefetch( + container_execution=container_execution, + launched_container=reloaded_launched_container, + ) + if prefetch is None: + return current_time = _get_current_time() # Saving the updated launcher data reloaded_launcher_data = reloaded_launched_container.to_dict() @@ -1034,12 +1080,13 @@ def internal_process_one_running_execution( # Don't fail the execution if log upload fails. # Logs are important, but not so important that we should fail a successfully completed container execution. - try: - _retry(reloaded_launched_container.upload_log) - except Exception as ex: - _logger.exception( - f"! Error during `LaunchedContainer.upload_log` call: {ex}." - ) + if not prefetch.log_uploaded: + try: + _retry(reloaded_launched_container.upload_log) + except Exception as ex: + _logger.exception( + f"! Error during `LaunchedContainer.upload_log` call: {ex}." + ) _MAX_PRELOAD_VALUE_SIZE = 255 @@ -1077,8 +1124,14 @@ def _maybe_preload_value( missing_output_names = [ output_name for output_name, uri in output_artifact_uris.items() - if not _retry( - lambda: self._storage_provider.make_uri(uri).get_reader().exists() + if not _prefetched_or_call( + prefetch.output_exists, + uri, + lambda: _retry( + lambda: self._storage_provider.make_uri(uri) + .get_reader() + .exists() + ), ) ] @@ -1105,12 +1158,16 @@ def _maybe_preload_value( "orchestrator.get_output_artifact_info" ): output_artifact_data_info_map = { - output_name: _retry( - lambda: ( - self._storage_provider.make_uri(uri) - .get_reader() - .get_info() - ) + output_name: _prefetched_or_call( + prefetch.output_data_infos, + uri, + lambda: _retry( + lambda: ( + self._storage_provider.make_uri(uri) + .get_reader() + .get_info() + ) + ), ) for output_name, uri in output_artifact_uris.items() } @@ -1126,13 +1183,17 @@ def _maybe_preload_value( ][0], uri=output_artifact_uris[output_name], # Preloading artifact value is it's small enough (e.g. <=255 bytes) - value=_retry( - lambda: _maybe_get_small_artifact_value( - uri_reader=self._storage_provider.make_uri( - output_artifact_uris[output_name] - ).get_reader(), - data_info=data_info, - ) + value=_prefetched_or_call( + prefetch.output_values, + output_artifact_uris[output_name], + lambda: _retry( + lambda: _maybe_get_small_artifact_value( + uri_reader=self._storage_provider.make_uri( + output_artifact_uris[output_name] + ).get_reader(), + data_info=data_info, + ) + ), ), created_at=current_time, ) @@ -1184,12 +1245,13 @@ def _maybe_preload_value( # Logs are best-effort; losing them is never a reason to fail an # execution. The launcher decides what a lost log means; the # orchestrator only records that the upload did not succeed. - try: - _retry(reloaded_launched_container.upload_log) - except Exception as ex: - _logger.exception( - f"! Error during `LaunchedContainer.upload_log` call: {ex}." - ) + if not prefetch.log_uploaded: + try: + _retry(reloaded_launched_container.upload_log) + except Exception as ex: + _logger.exception( + f"! Error during `LaunchedContainer.upload_log` call: {ex}." + ) # Skip downstream executions for execution_node in execution_nodes: execution_node.container_execution_status = ( @@ -1234,6 +1296,161 @@ def _maybe_preload_value( f"Processing running container execution {container_execution.id}: Error doing launched container cleanup." ) + def shutdown_completion_prefetches(self, *, timeout_seconds: float) -> bool: + """Cancels the completion prefetches that have not started and waits up to `timeout_seconds` for the running ones; returns whether they all finished.""" + if self._completion_prefetch_executor is None: + return True + self._completion_prefetch_executor.shutdown(wait=False, cancel_futures=True) + _, not_done = concurrent.futures.wait( + self._completion_prefetches.values(), timeout=timeout_seconds + ) + return not not_done + + def _take_completion_prefetch( + self, + *, + container_execution: bts.ContainerExecution, + launched_container: launcher_interfaces.LaunchedContainer, + ) -> "_CompletionPrefetch | None": + """Returns and forgets the finished prefetch for this container execution, or starts one and returns None.""" + future = self._completion_prefetches.get(container_execution.id) + if future is None: + output_uris = [] + if ( + launched_container.status + == launcher_interfaces.ContainerStatus.SUCCEEDED + ): + output_uris = [ + output_artifact_info_dict["uri"] + for output_artifact_info_dict in ( + container_execution.output_artifact_data_map or {} + ).values() + ] + future = self._completion_prefetch_executor.submit( + _prefetch_completion, + launched_container=launched_container, + storage_provider=self._storage_provider, + output_uris=output_uris, + ) + future.add_done_callback( + functools.partial( + self._record_finished_completion_prefetch, container_execution.id + ) + ) + self._completion_prefetches[container_execution.id] = future + self._ended_launched_containers[container_execution.id] = launched_container + return None + if not future.done(): + return None + del self._completion_prefetches[container_execution.id] + self._ended_launched_containers.pop(container_execution.id, None) + try: + return future.result() + except Exception: + _logger.exception("Completion prefetch failed; processing without it.") + return _CompletionPrefetch() + + def _forget_completion_prefetch(self, *, container_execution_id: str) -> None: + """Forgets the prefetch of a container execution that left the running queue, cancelling it if it has not started.""" + self._ended_launched_containers.pop(container_execution_id, None) + future = self._completion_prefetches.pop(container_execution_id, None) + if future is not None: + future.cancel() + + def _record_finished_completion_prefetch( + self, + container_execution_id: str, + future: "concurrent.futures.Future[_CompletionPrefetch]", + ) -> None: + """Records that the container execution's prefetch is done, for the loop thread to move it to the front of the running queue.""" + if future.cancelled(): + return + with self._finished_completion_prefetch_ids_lock: + self._finished_completion_prefetch_ids.add(container_execution_id) + + def _move_finished_completion_prefetches_to_front( + self, *, session: orm.Session + ) -> None: + """Moves every pending or running container execution whose prefetch is done to the front of the running queue, so the next visit commits it.""" + with self._finished_completion_prefetch_ids_lock: + container_execution_ids = list(self._finished_completion_prefetch_ids) + self._finished_completion_prefetch_ids.clear() + if not container_execution_ids: + return + session.execute( + sql.update(bts.ContainerExecution) + .where( + bts.ContainerExecution.id.in_(container_execution_ids), + bts.ContainerExecution.status.in_( + ( + bts.ContainerExecutionStatus.PENDING, + bts.ContainerExecutionStatus.RUNNING, + ) + ), + ) + .values(last_processed_at=_FRONT_OF_RUNNING_QUEUE_LAST_PROCESSED_AT) + ) + session.commit() + + +_ENDED_CONTAINER_STATUSES = ( + launcher_interfaces.ContainerStatus.SUCCEEDED, + launcher_interfaces.ContainerStatus.FAILED, +) + + +@dataclasses.dataclass +class _CompletionPrefetch: + """The finished-container calls a prefetch completed; each call missing here is made inline instead.""" + + log_uploaded: bool = False + output_exists: dict[str, bool] = dataclasses.field(default_factory=dict) + output_data_infos: dict[str, storage_provider_interfaces.DataInfo] = ( + dataclasses.field(default_factory=dict) + ) + output_values: dict[str, str] = dataclasses.field(default_factory=dict) + + +def _prefetch_completion( + *, + launched_container: launcher_interfaces.LaunchedContainer, + storage_provider: storage_provider_interfaces.StorageProvider, + output_uris: list[str], +) -> _CompletionPrefetch: + """Makes the log upload and output reads that committing a finished container needs, recording each one that succeeds.""" + prefetch = _CompletionPrefetch() + with contextlib.suppress(Exception): + _retry(launched_container.upload_log) + prefetch.log_uploaded = True + + readers = {uri: storage_provider.make_uri(uri).get_reader() for uri in output_uris} + with contextlib.suppress(Exception): + for uri, reader in readers.items(): + if _retry(reader.exists): + prefetch.output_exists[uri] = True + if not all(prefetch.output_exists.get(uri) for uri in output_uris): + return prefetch + + with contextlib.suppress(Exception): + for uri, reader in readers.items(): + prefetch.output_data_infos[uri] = _retry(reader.get_info) + for uri, data_info in prefetch.output_data_infos.items(): + value = _maybe_get_small_artifact_value( + uri_reader=readers[uri], data_info=data_info + ) + if value is not None: + prefetch.output_values[uri] = value + return prefetch + + +def _prefetched_or_call( + prefetched: dict[str, _T], uri: str, call: typing.Callable[[], _T] +) -> _T: + """Returns the prefetched result for `uri`, or makes the call when the prefetch has none.""" + if uri in prefetched: + return prefetched[uri] + return call() + def _get_direct_downstream_executions( session: orm.Session, execution: bts.ExecutionNode diff --git a/tests/test_orchestrator_completion_prefetch.py b/tests/test_orchestrator_completion_prefetch.py new file mode 100644 index 0000000..390edfc --- /dev/null +++ b/tests/test_orchestrator_completion_prefetch.py @@ -0,0 +1,587 @@ +"""Tests for the completion prefetch. + +With the prefetch on, a finished container's log upload and output reads run on the prefetch workers, never on the loop +thread, and the container gets the same terminal outcome it gets with the prefetch off. +""" + +import concurrent.futures +import datetime +import threading +from unittest import mock + +import pytest +from cloud_pipelines.orchestration.storage_providers import ( + interfaces as storage_interfaces, +) +from sqlalchemy import orm, sql + +from cloud_pipelines_backend import api_server_sql +from cloud_pipelines_backend import backend_types_sql as bts +from cloud_pipelines_backend import component_structures as structures +from cloud_pipelines_backend import database_ops +from cloud_pipelines_backend import orchestrator_sql +from cloud_pipelines_backend.launchers import interfaces as launcher_interfaces + +_TS = datetime.datetime(2026, 1, 1, 12, 0, 0, tzinfo=datetime.timezone.utc) +_LAUNCHER_DATA = {"pod": "pod-0"} + + +@pytest.fixture(autouse=True) +def _no_retry_sleeps(): + """Patches out `_retry` sleeps, on the loop thread and the prefetch workers alike.""" + with mock.patch("time.sleep"): + yield + + +class _FakeStorage: + """A storage provider whose every output is a small file, recording which thread made each call.""" + + def __init__(self, *, exists: bool = True, error: Exception | None = None): + self.exists_result = exists + self.error = error + self.calls: list[tuple[str, int]] = [] + + def make_uri(self, uri: str) -> mock.MagicMock: + """An accessor whose reader answers for `uri`.""" + reader = mock.MagicMock() + reader.exists.side_effect = lambda: self._call("exists", self.exists_result) + reader.get_info.side_effect = lambda: self._call( + "get_info", + storage_interfaces.DataInfo( + total_size=5, is_dir=False, hashes={"md5": "abc"} + ), + ) + reader.download_as_bytes.side_effect = lambda: self._call( + "download_as_bytes", b"hello" + ) + accessor = mock.MagicMock() + accessor.get_reader.return_value = reader + return accessor + + def _call(self, name: str, result): + """Records the call and its thread, then returns `result` or raises the configured error.""" + self.calls.append((name, threading.get_ident())) + if self.error is not None: + raise self.error + return result + + +def _container_component( + *, inputs: list[str] | None = None, outputs: list[str] | None = None +) -> structures.ComponentSpec: + """A container component with the given input and output names.""" + return structures.ComponentSpec( + inputs=[structures.InputSpec(name=name) for name in (inputs or [])], + outputs=[structures.OutputSpec(name=name) for name in (outputs or [])], + implementation=structures.ContainerImplementation( + container=structures.ContainerSpec(image="python") + ), + ) + + +def _upstream_downstream_pipeline() -> structures.TaskSpec: + """`upstream` (produces `out`) feeding `downstream` (consumes `in`).""" + pipeline_spec = structures.ComponentSpec( + implementation=structures.GraphImplementation( + graph=structures.GraphSpec( + tasks={ + "upstream": structures.TaskSpec( + component_ref=structures.ComponentReference( + spec=_container_component(outputs=["out"]) + ), + ), + "downstream": structures.TaskSpec( + component_ref=structures.ComponentReference( + spec=_container_component(inputs=["in"]) + ), + arguments={ + "in": structures.TaskOutputArgument( + task_output=structures.TaskOutputReference( + task_id="upstream", output_name="out" + ) + ) + }, + ), + } + ) + ), + ) + return structures.TaskSpec( + component_ref=structures.ComponentReference(spec=pipeline_spec) + ) + + +def _launched_container( + status: launcher_interfaces.ContainerStatus, +) -> mock.MagicMock: + """A launched container in `status` that records the thread of each log upload.""" + container = mock.MagicMock( + status=status, + exit_code=0, + started_at=_TS, + ended_at=_TS, + launcher_error_message=None, + to_dict=lambda: dict(_LAUNCHER_DATA), + ) + container.upload_threads = [] + container.upload_log.side_effect = lambda: container.upload_threads.append( + threading.get_ident() + ) + return container + + +class _Scenario: + """One pipeline whose upstream container finishes as `ended_container`, with the completion prefetch on or off.""" + + def __init__( + self, + *, + prefetch: bool, + ended_container: mock.MagicMock, + storage: _FakeStorage, + ): + db_engine = database_ops.create_db_engine_and_migrate_db( + database_uri="sqlite://" + ) + self.session_factory = lambda: orm.Session(bind=db_engine) + api_server_sql.PipelineRunsApiService_Sql().create( + session=self.session_factory(), + root_task=_upstream_downstream_pipeline(), + created_by="user1", + ) + launcher = mock.MagicMock() + launcher.launch_container_task.side_effect = lambda *a, **kw: ( + _launched_container(launcher_interfaces.ContainerStatus.PENDING) + ) + launcher.deserialize_launched_container_from_dict.side_effect = ( + lambda data: _launched_container( + launcher_interfaces.ContainerStatus.PENDING + ) + ) + launcher.get_refreshed_launched_container_from_dict.side_effect = ( + lambda data: ended_container + ) + self.launcher = launcher + self.orchestrator = orchestrator_sql.OrchestratorService_Sql( + session_factory=self.session_factory, + launcher=launcher, + storage_provider=storage, + data_root_uri="file:///tmp/artifacts", + logs_root_uri="file:///tmp/logs", + _completion_prefetch_max_workers=1 if prefetch else 0, + ) + session = self.session_factory() + while self.orchestrator.internal_process_queued_executions_queue( + session=session + ): + pass + + def visit_running(self) -> None: + """Processes the running queue once.""" + self.orchestrator.internal_process_running_executions_queue( + session=self.session_factory() + ) + + def wait_for_prefetches(self) -> None: + """Blocks until every in-flight prefetch and its done callback have finished, then starts a fresh executor.""" + if self.orchestrator._completion_prefetch_executor is None: + return + self.orchestrator._completion_prefetch_executor.shutdown(wait=True) + self.orchestrator._completion_prefetch_executor = ( + concurrent.futures.ThreadPoolExecutor(max_workers=1) + ) + + def run_to_completion(self) -> None: + """Visits the ended container until it leaves PENDING, at most three times.""" + for _ in range(3): + self.visit_running() + if self.upstream_status() != bts.ContainerExecutionStatus.PENDING: + return + self.wait_for_prefetches() + + def node(self, task_id: str) -> bts.ExecutionNode: + """The execution node of `task_id`.""" + return self.session_factory().scalar( + sql.select(bts.ExecutionNode).where( + bts.ExecutionNode.task_id_in_parent_execution == task_id + ) + ) + + def upstream_status(self) -> bts.ContainerExecutionStatus: + """The upstream node's status.""" + return self.node("upstream").container_execution_status + + def upstream_last_processed_at(self) -> datetime.datetime: + """When the upstream container execution was last processed.""" + session = self.session_factory() + return session.get( + bts.ContainerExecution, self.node("upstream").container_execution_id + ).last_processed_at + + def set_upstream_last_processed_at(self, value: datetime.datetime) -> None: + """Sets when the upstream container execution was last processed.""" + with self.session_factory() as session: + session.get( + bts.ContainerExecution, + self.node("upstream").container_execution_id, + ).last_processed_at = value + session.commit() + + def outcome(self) -> dict: + """Everything the finished container's processing decides.""" + session = self.session_factory() + upstream = self.node("upstream") + container_execution = session.get( + bts.ContainerExecution, upstream.container_execution_id + ) + artifacts = [ + (data.total_size, data.is_dir, data.hash, data.value) + for data in session.scalars(sql.select(bts.ArtifactData)).all() + if data.uri + in [ + info["uri"] + for info in container_execution.output_artifact_data_map.values() + ] + ] + return { + "upstream": upstream.container_execution_status, + "downstream": self.node("downstream").container_execution_status, + "container": container_execution.status, + "error": (container_execution.extra_data or {}).get( + bts.CONTAINER_EXECUTION_EXTRA_DATA_ORCHESTRATION_ERROR_MESSAGE_KEY + ), + "artifacts": artifacts, + } + + +def _run(*, prefetch: bool, status, storage) -> _Scenario: + """Runs the scenario until the ended upstream container gets its terminal status.""" + scenario = _Scenario( + prefetch=prefetch, + ended_container=_launched_container(status), + storage=storage, + ) + scenario.run_to_completion() + return scenario + + +class TestPrefetchedCompletion: + def test_first_visit_starts_the_prefetch_and_keeps_the_status(self): + storage = _FakeStorage() + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=storage, + ) + with mock.patch.object( + scenario.orchestrator, "_completion_prefetch_executor" + ) as executor: + scenario.visit_running() + + executor.submit.assert_called_once() + assert scenario.upstream_status() == bts.ContainerExecutionStatus.PENDING + assert storage.calls == [] + + def test_terminal_status_lands_on_the_visit_after_the_prefetch(self): + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=_FakeStorage(), + ) + + scenario.visit_running() + assert scenario.upstream_status() == bts.ContainerExecutionStatus.PENDING + scenario.wait_for_prefetches() + scenario.visit_running() + + assert scenario.upstream_status() == bts.ContainerExecutionStatus.SUCCEEDED + assert scenario.orchestrator._completion_prefetches == {} + + def test_finished_prefetch_moves_the_execution_to_the_front_of_the_queue_on_the_loop_thread( + self, + ): + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=_FakeStorage(), + ) + + scenario.visit_running() + scenario.wait_for_prefetches() + assert scenario.upstream_last_processed_at().year != 1970 + scenario.orchestrator._move_finished_completion_prefetches_to_front( + session=scenario.session_factory() + ) + + assert scenario.upstream_last_processed_at().year == 1970 + + def test_output_that_was_missing_during_the_prefetch_is_read_again(self): + storage = _FakeStorage(exists=False) + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=storage, + ) + + scenario.visit_running() + scenario.wait_for_prefetches() + storage.exists_result = True + scenario.visit_running() + + assert scenario.upstream_status() == bts.ContainerExecutionStatus.SUCCEEDED + + def test_commits_the_ended_container_without_refreshing_it_again(self): + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=_FakeStorage(), + ) + scenario.visit_running() + scenario.wait_for_prefetches() + scenario.launcher.get_refreshed_launched_container_from_dict.side_effect = ( + RuntimeError("The pod is gone") + ) + + scenario.visit_running() + + assert scenario.upstream_status() == bts.ContainerExecutionStatus.SUCCEEDED + assert scenario.orchestrator._ended_launched_containers == {} + + def test_system_error_forgets_the_prefetch(self): + ended_container = _launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ) + scenario = _Scenario( + prefetch=True, ended_container=ended_container, storage=_FakeStorage() + ) + scenario.visit_running() + scenario.wait_for_prefetches() + + def _fail_to_serialize(): + raise RuntimeError("The launcher data could not be serialized") + + ended_container.to_dict = _fail_to_serialize + + scenario.visit_running() + + assert scenario.upstream_status() == bts.ContainerExecutionStatus.SYSTEM_ERROR + assert scenario.orchestrator._completion_prefetches == {} + assert scenario.orchestrator._ended_launched_containers == {} + + def test_pending_prefetch_visit_moves_the_execution_back_in_the_queue(self): + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=_FakeStorage(), + ) + scenario.orchestrator._completion_prefetches[ + scenario.node("upstream").container_execution_id + ] = mock.MagicMock(**{"done.return_value": False}) + scenario.set_upstream_last_processed_at( + datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) + ) + + scenario.visit_running() + + assert scenario.upstream_last_processed_at().year != 1970 + + def test_shutdown_stops_new_prefetches(self): + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=_FakeStorage(), + ) + + scenario.orchestrator.shutdown_completion_prefetches(timeout_seconds=0) + + with pytest.raises(RuntimeError): + scenario.orchestrator._completion_prefetch_executor.submit(lambda: None) + + def test_shutdown_reports_a_prefetch_still_running(self): + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=_FakeStorage(), + ) + release = threading.Event() + scenario.orchestrator._completion_prefetches["hung"] = ( + scenario.orchestrator._completion_prefetch_executor.submit(release.wait) + ) + + try: + assert not scenario.orchestrator.shutdown_completion_prefetches( + timeout_seconds=0.05 + ) + finally: + release.set() + + def test_shutdown_reports_finished_prefetches(self): + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=_FakeStorage(), + ) + scenario.visit_running() + assert scenario.orchestrator._completion_prefetches + + assert scenario.orchestrator.shutdown_completion_prefetches(timeout_seconds=5) + + def test_loop_thread_does_no_storage_or_log_io(self): + storage = _FakeStorage() + scenario = _run( + prefetch=True, + status=launcher_interfaces.ContainerStatus.SUCCEEDED, + storage=storage, + ) + ended_container = scenario.launcher.get_refreshed_launched_container_from_dict( + None + ) + loop_thread = threading.get_ident() + + assert sorted({name for name, _ in storage.calls}) == [ + "download_as_bytes", + "exists", + "get_info", + ] + assert all(thread != loop_thread for _, thread in storage.calls) + assert ended_container.upload_threads + assert loop_thread not in ended_container.upload_threads + + def test_refreshes_once_per_visit(self): + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.RUNNING + ), + storage=_FakeStorage(), + ) + + scenario.visit_running() + + assert ( + scenario.launcher.get_refreshed_launched_container_from_dict.call_count == 1 + ) + assert scenario.upstream_status() == bts.ContainerExecutionStatus.RUNNING + + +@pytest.mark.parametrize( + ("status", "storage_kwargs"), + [ + (launcher_interfaces.ContainerStatus.SUCCEEDED, {}), + (launcher_interfaces.ContainerStatus.SUCCEEDED, {"exists": False}), + (launcher_interfaces.ContainerStatus.FAILED, {}), + ( + launcher_interfaces.ContainerStatus.SUCCEEDED, + {"error": RuntimeError("storage is down")}, + ), + ], + ids=["succeeded", "missing-outputs", "failed", "storage-errors"], +) +def test_outcome_matches_the_orchestrator_without_the_prefetch(status, storage_kwargs): + without_prefetch = _run( + prefetch=False, + status=status, + storage=_FakeStorage(**storage_kwargs), + ) + with_prefetch = _run( + prefetch=True, + status=status, + storage=_FakeStorage(**storage_kwargs), + ) + + assert with_prefetch.outcome() == without_prefetch.outcome() + assert with_prefetch.upstream_status() != bts.ContainerExecutionStatus.PENDING + + +def test_failed_log_upload_is_retried_inline(): + container = _launched_container(launcher_interfaces.ContainerStatus.FAILED) + container.upload_log.side_effect = RuntimeError("pod log is gone") + scenario = _Scenario( + prefetch=True, + ended_container=container, + storage=_FakeStorage(), + ) + + scenario.run_to_completion() + + assert scenario.upstream_status() == bts.ContainerExecutionStatus.FAILED + assert container.upload_log.call_count == 10 + + +def test_cancelling_forgets_the_prefetch(): + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=_FakeStorage(), + ) + scenario.visit_running() + scenario.wait_for_prefetches() + with scenario.session_factory() as session: + upstream = scenario.node("upstream") + session.get(bts.ExecutionNode, upstream.id).extra_data = { + "desired_state": "TERMINATED" + } + session.commit() + + scenario.visit_running() + + assert scenario.upstream_status() == bts.ContainerExecutionStatus.CANCELLED + assert scenario.orchestrator._completion_prefetches == {} + + +def test_finished_prefetch_does_not_move_an_ended_execution(): + scenario = _Scenario( + prefetch=True, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=_FakeStorage(), + ) + scenario.visit_running() + scenario.wait_for_prefetches() + scenario.visit_running() + assert scenario.upstream_status() == bts.ContainerExecutionStatus.SUCCEEDED + scenario.orchestrator._finished_completion_prefetch_ids.add( + scenario.node("upstream").container_execution_id + ) + + scenario.orchestrator._move_finished_completion_prefetches_to_front( + session=scenario.session_factory() + ) + + assert scenario.upstream_last_processed_at().year != 1970 + + +def test_without_the_prefetch_the_first_visit_commits_the_status(): + scenario = _Scenario( + prefetch=False, + ended_container=_launched_container( + launcher_interfaces.ContainerStatus.SUCCEEDED + ), + storage=_FakeStorage(), + ) + + scenario.visit_running() + + assert scenario.upstream_status() == bts.ContainerExecutionStatus.SUCCEEDED + assert scenario.orchestrator.shutdown_completion_prefetches(timeout_seconds=0)