diff --git a/cloud_pipelines_backend/launchers/interfaces.py b/cloud_pipelines_backend/launchers/interfaces.py index 365ddaf..26d2949 100644 --- a/cloud_pipelines_backend/launchers/interfaces.py +++ b/cloud_pipelines_backend/launchers/interfaces.py @@ -60,6 +60,16 @@ class InputArgument: dynamic_data: str | dict[str, Any] | None = None +@dataclasses.dataclass(frozen=True, kw_only=True) +class ContainerExecutionChange: + """A launcher saw the container of a container execution it launched change.""" + + container_execution_id: str + + +ContainerExecutionChangeListener = typing.Callable[[ContainerExecutionChange], None] + + class ContainerTaskLauncher(typing.Generic[_TLaunchedContainer], abc.ABC): @abc.abstractmethod def launch_container_task( @@ -83,6 +93,12 @@ def get_refreshed_launched_container_from_dict( ) -> _TLaunchedContainer: raise NotImplementedError() + def subscribe_to_container_execution_changes( + self, *, listener: ContainerExecutionChangeListener + ) -> None: + """Calls `listener` whenever a container this launcher launched changes; does nothing when the launcher cannot tell.""" + del listener + class ContainerStatus(str, enum.Enum): PENDING = "PENDING" diff --git a/cloud_pipelines_backend/launchers/kubernetes_launchers.py b/cloud_pipelines_backend/launchers/kubernetes_launchers.py index a9d305e..3b4dec1 100644 --- a/cloud_pipelines_backend/launchers/kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/kubernetes_launchers.py @@ -217,6 +217,10 @@ def __init__( ) self._pod_postprocessor = pod_postprocessor self._create_volume_and_volume_mount = _create_volume_and_volume_mount + self._container_execution_change_listeners: tuple[ + interfaces.ContainerExecutionChangeListener, ... + ] = () + self._container_execution_change_listeners_lock = threading.Lock() try: k8s_client_lib.VersionApi(self._api_client).get_code( @@ -246,6 +250,7 @@ def __init__( # But Tangle has only recently started labeling it's pods/jobs, so if we add filtering too soon, this will become a breaking change. # label_selector=f"{_TANGLE_KUBERNETES_LABEL_KEY}=true", label_selector=None, + on_change=self._notify_container_execution_changed, ) self._pod_informer.start() except Exception: @@ -648,6 +653,10 @@ def get_output_path(output_name: str) -> str: service_account_name=pod_service_account, ) + pod_annotations = (pod_annotations or {}) | _container_execution_id_annotation( + annotations + ) + pod = k8s_client_lib.V1Pod( api_version="v1", kind="Pod", @@ -674,6 +683,32 @@ def _choose_namespace( return self._namespace return self._namespace(annotations=annotations) + def subscribe_to_container_execution_changes( + self, *, listener: interfaces.ContainerExecutionChangeListener + ) -> None: + """Calls `listener` whenever a pod or job this launcher launched changes.""" + with self._container_execution_change_listeners_lock: + self._container_execution_change_listeners = ( + *self._container_execution_change_listeners, + listener, + ) + + def _notify_container_execution_changed( + self, obj: k8s_client_lib.V1Pod | k8s_client_lib.V1Job + ) -> None: + """Tells every listener that the container execution behind `obj` changed, so one failing listener can't stop the rest.""" + container_execution_id = _get_container_execution_id(obj) + if not container_execution_id: + return + change = interfaces.ContainerExecutionChange( + container_execution_id=container_execution_id + ) + for listener in self._container_execution_change_listeners: + try: + listener(change) + except Exception: + _logger.exception("Container execution change listener failed.") + class _KubernetesPodLauncher( _KubernetesContainerLauncherBase, @@ -1346,6 +1381,7 @@ def __init__( # But Tangle has only recently started labeling it's pods/jobs, so if we add filtering too soon, this will become a breaking change. # label_selector=f"{_TANGLE_KUBERNETES_LABEL_KEY}=true", label_selector=None, + on_change=self._notify_container_execution_changed, ) self._job_informer.start() except Exception: @@ -1532,7 +1568,8 @@ def launch_container_task( name=explicit_job_name, namespace=namespace, # annotations=self._pod_annotations, - annotations=_DEFAULT_KUBERNETES_ANNOTATIONS, + annotations=_DEFAULT_KUBERNETES_ANNOTATIONS + | _container_execution_id_annotation(annotations), # ! Kubernetes gotcha: If Job labels are empty, Kubernetes copies the labels from the Job's Pod template (but not annotations). # https://github.com/kubernetes/kubernetes/blob/b1dc9d16763d2546b8464a09307471f3411ab391/pkg/apis/batch/v1/defaults.go#L49-L52 # So, changing code from no labels to some default labels stops that special behavior, causing pod template labels to not being propagated to Job itself. @@ -2329,6 +2366,13 @@ def get_refreshed_launched_container_from_dict( ) return launched_container.get_refreshed() + def subscribe_to_container_execution_changes( + self, *, listener: interfaces.ContainerExecutionChangeListener + ) -> None: + """Subscribes `listener` to the changes of both the pod and the job launcher.""" + self._pod_launcher.subscribe_to_container_execution_changes(listener=listener) + self._job_launcher.subscribe_to_container_execution_changes(listener=listener) + def deserialize_launched_container_from_dict( self, launched_container_dict: dict ) -> "LaunchedKubernetesContainer | LaunchedKubernetesJob": @@ -2401,6 +2445,37 @@ def __init__( ) +def _container_execution_id_annotation( + annotations: dict[str, Any] | None, +) -> dict[str, str]: + """The container execution id annotation from a task's annotations, to stamp on its pod or job; empty when absent.""" + container_execution_id = (annotations or {}).get( + common_annotations.CONTAINER_EXECUTION_ID_ANNOTATION_KEY + ) + if not container_execution_id: + return {} + return { + common_annotations.CONTAINER_EXECUTION_ID_ANNOTATION_KEY: container_execution_id + } + + +def _get_container_execution_id( + obj: k8s_client_lib.V1Pod | k8s_client_lib.V1Job, +) -> str | None: + """Reads the container execution id annotation off a pod or a job, falling back to a job's pod template.""" + metadatas = [obj.metadata] + if isinstance(obj, k8s_client_lib.V1Job) and obj.spec and obj.spec.template: + metadatas.append(obj.spec.template.metadata) + for metadata in metadatas: + if metadata and metadata.annotations: + container_execution_id = metadata.annotations.get( + common_annotations.CONTAINER_EXECUTION_ID_ANNOTATION_KEY + ) + if container_execution_id: + return container_execution_id + return None + + def _get_job_pod_completion_index(pod: k8s_client_lib.V1Pod) -> str | None: if not (pod.metadata and pod.metadata.annotations): return None @@ -2561,12 +2636,16 @@ def __init__( label_selector=None, field_selector=None, key_func=None, + on_change: ( + typing.Callable[[k8s_client_lib.V1Pod | k8s_client_lib.V1Job], None] | None + ) = None, ): self._list_func = list_func self._resync_period = resync_period self._label_selector = label_selector self._field_selector = field_selector self._key_func = key_func + self._on_change = on_change self._informers: dict[str, k8s_informer_lib.SharedInformer] = {} self._lock = threading.Lock() self._is_active = False @@ -2592,6 +2671,13 @@ def _get_or_create_informer( ) # Prefill the cache informer._initial_list() + if self._on_change: + for event_type in ( + k8s_informer_lib.ADDED, + k8s_informer_lib.MODIFIED, + k8s_informer_lib.DELETED, + ): + informer.add_event_handler(event_type, self._on_change) self._informers[namespace] = informer if self._is_active: informer.start() diff --git a/cloud_pipelines_backend/orchestrator_sql.py b/cloud_pipelines_backend/orchestrator_sql.py index 246a607..3f86cbf 100644 --- a/cloud_pipelines_backend/orchestrator_sql.py +++ b/cloud_pipelines_backend/orchestrator_sql.py @@ -4,6 +4,7 @@ import logging import itertools import os +import threading import time import traceback import typing @@ -36,6 +37,10 @@ DYNAMIC_DATA_SECRET_KEY = "secret" DYNAMIC_DATA_SECRET_NAME_KEY = "name" +_CHANGED_CONTAINER_EXECUTION_PRIORITY = datetime.timedelta(minutes=2) + +_MAX_CHANGED_CONTAINER_EXECUTION_IDS = 10_000 + class OrchestratorError(RuntimeError): pass @@ -94,6 +99,11 @@ def __init__( self._max_queue_batch_size = _max_queue_batch_size self._max_queue_batch_duration = _max_queue_batch_duration self._queued_execution_interceptor = queued_execution_interceptor + self._changed_container_execution_ids: set[str] = set() + self._changed_container_execution_ids_lock = threading.Lock() + self._launcher.subscribe_to_container_execution_changes( + listener=self._record_container_execution_change + ) def run_loop(self): while True: @@ -197,6 +207,7 @@ def internal_process_queued_executions_queue(self, session: orm.Session): return True def internal_process_running_executions_queue(self, session: orm.Session): + self._move_changed_container_executions_to_front(session=session) _logger.debug("Selecting running container execution to process.)") # Select only the ID to avoid loading large JSON columns into the sort buffer. @@ -341,6 +352,51 @@ def internal_process_running_executions_queue(self, session: orm.Session): ) return True + def _record_container_execution_change( + self, change: launcher_interfaces.ContainerExecutionChange + ) -> None: + """Records a container execution the launcher saw change, for the next running sweep to check first. + + Runs on the launcher's thread; past the cap new ids are dropped, so those executions just wait their normal turn. + """ + with self._changed_container_execution_ids_lock: + if ( + len(self._changed_container_execution_ids) + < _MAX_CHANGED_CONTAINER_EXECUTION_IDS + ): + self._changed_container_execution_ids.add(change.container_execution_id) + + def _move_changed_container_executions_to_front(self, session: orm.Session) -> None: + """Moves each pending or running execution the launcher saw change to `_CHANGED_CONTAINER_EXECUTION_PRIORITY` ago, unless it's already further ahead. + + Changes are served in the order recorded, an execution that keeps changing can't starve the rest, and a change + seen mid-refresh (such as the container finishing) moves that execution forward again. + """ + with self._changed_container_execution_ids_lock: + container_execution_ids = self._changed_container_execution_ids + self._changed_container_execution_ids = set() + if not container_execution_ids: + return + front_of_queue_last_processed_at = ( + _get_current_time() - _CHANGED_CONTAINER_EXECUTION_PRIORITY + ) + session.execute( + sql.update(bts.ContainerExecution) + .where( + bts.ContainerExecution.id.in_(container_execution_ids), + bts.ContainerExecution.status.in_( + ( + bts.ContainerExecutionStatus.PENDING, + bts.ContainerExecutionStatus.RUNNING, + ) + ), + bts.ContainerExecution.last_processed_at + > front_of_queue_last_processed_at, + ) + .values(last_processed_at=front_of_queue_last_processed_at) + ) + session.commit() + def internal_process_one_queued_execution( self, session: orm.Session, execution: bts.ExecutionNode ): diff --git a/tests/launchers/kubernetes/test_kubernetes_launchers_changed_container_executions.py b/tests/launchers/kubernetes/test_kubernetes_launchers_changed_container_executions.py new file mode 100644 index 0000000..8476882 --- /dev/null +++ b/tests/launchers/kubernetes/test_kubernetes_launchers_changed_container_executions.py @@ -0,0 +1,258 @@ +"""Tests for the Kubernetes launchers reporting which container executions' pods or jobs changed. + +The Kubernetes API calls are faked so that the tests run offline. +""" + +from __future__ import annotations + +import pathlib + +import pytest +from kubernetes import client as k8s_client_lib + +from cloud_pipelines_backend import component_structures as structures +from cloud_pipelines_backend.launchers import common_annotations +from cloud_pipelines_backend.launchers import interfaces +from cloud_pipelines_backend.launchers import kubernetes_launchers + +_CONTAINER_EXECUTION_ID_KEY = common_annotations.CONTAINER_EXECUTION_ID_ANNOTATION_KEY + + +class _FakeVersionApi: + def __init__(self, api_client=None): + pass + + def get_code(self, **kwargs): + return None + + +class _FakeCluster: + """Records the created pods and jobs.""" + + def __init__(self): + self.created_pods: list[k8s_client_lib.V1Pod] = [] + self.created_jobs: list[k8s_client_lib.V1Job] = [] + + def make_core_api(self): + cluster = self + + class _FakeCoreV1Api: + def __init__(self, api_client=None): + pass + + def create_namespaced_pod(self, namespace, body, **kwargs): + cluster.created_pods.append(body) + body.metadata.name = (body.metadata.generate_name or "") + "abcde" + body.metadata.namespace = namespace + return body + + def create_namespaced_service(self, namespace, body, **kwargs): + return body + + return _FakeCoreV1Api + + def make_batch_api(self): + cluster = self + + class _FakeBatchV1Api: + def __init__(self, api_client=None): + pass + + def create_namespaced_job(self, namespace, body, **kwargs): + cluster.created_jobs.append(body) + body.metadata.uid = "job-uid" + return body + + return _FakeBatchV1Api + + +class _FakeSharedInformer: + """Records the event handlers it is given and whether they were added after the cache was prefilled.""" + + def __init__(self, **kwargs): + self.handlers: list[tuple[str, object, bool]] = [] + self.listed = False + + def _initial_list(self): + self.listed = True + + def add_event_handler(self, event_type, handler): + self.handlers.append((event_type, handler, self.listed)) + + def start(self): + pass + + +@pytest.fixture +def cluster(monkeypatch) -> _FakeCluster: + cluster = _FakeCluster() + monkeypatch.setattr(k8s_client_lib, "VersionApi", _FakeVersionApi) + monkeypatch.setattr(k8s_client_lib, "CoreV1Api", cluster.make_core_api()) + monkeypatch.setattr(k8s_client_lib, "BatchV1Api", cluster.make_batch_api()) + + def _no_informer(**kwargs): + raise RuntimeError("Informers are disabled in tests.") + + monkeypatch.setattr(kubernetes_launchers, "MultiNamespaceInformer", _no_informer) + return cluster + + +def _make_launcher() -> ( + kubernetes_launchers.Local_Kubernetes_UsingHostPathStorage_KubernetesPodOrJobLauncher +): + return kubernetes_launchers.Local_Kubernetes_UsingHostPathStorage_KubernetesPodOrJobLauncher( + api_client=k8s_client_lib.ApiClient(), + ) + + +def _launch(launcher, tmp_path: pathlib.Path, annotations: dict[str, str]) -> None: + launcher.launch_container_task( + component_spec=structures.ComponentSpec( + name="test", + implementation=structures.ContainerImplementation( + container=structures.ContainerSpec(image="alpine") + ), + ), + input_arguments={}, + output_uris={}, + log_uri=str(tmp_path / "log.txt"), + annotations={_CONTAINER_EXECUTION_ID_KEY: "123", **annotations}, + ) + + +def test_launched_pod_carries_the_container_execution_id(cluster, tmp_path): + _launch(_make_launcher(), tmp_path, {}) + + [pod] = cluster.created_pods + assert pod.metadata.annotations[_CONTAINER_EXECUTION_ID_KEY] == "123" + + +def test_launched_job_and_its_pod_template_carry_the_container_execution_id( + cluster, tmp_path +): + _launch( + _make_launcher(), + tmp_path, + {kubernetes_launchers.RETRIES_MAX_RETRIES_ANNOTATION_KEY: "1"}, + ) + + [job] = cluster.created_jobs + assert job.metadata.annotations[_CONTAINER_EXECUTION_ID_KEY] == "123" + assert job.spec.template.metadata.annotations[_CONTAINER_EXECUTION_ID_KEY] == "123" + + +def test_launching_does_not_change_the_launcher_pod_annotations(cluster, tmp_path): + launcher = _make_launcher() + + _launch(launcher, tmp_path, {}) + + assert _CONTAINER_EXECUTION_ID_KEY not in launcher._pod_launcher._pod_annotations + + +def _pod(container_execution_id: str | None) -> k8s_client_lib.V1Pod: + annotations = ( + {_CONTAINER_EXECUTION_ID_KEY: container_execution_id} + if container_execution_id + else None + ) + return k8s_client_lib.V1Pod( + metadata=k8s_client_lib.V1ObjectMeta(annotations=annotations) + ) + + +def _change(container_execution_id: str) -> interfaces.ContainerExecutionChange: + return interfaces.ContainerExecutionChange( + container_execution_id=container_execution_id + ) + + +def test_tells_the_listener_about_each_changed_pod_and_job(cluster): + launcher = _make_launcher() + changes = [] + launcher.subscribe_to_container_execution_changes(listener=changes.append) + + launcher._pod_launcher._notify_container_execution_changed(_pod("pod-execution")) + launcher._job_launcher._notify_container_execution_changed( + k8s_client_lib.V1Job( + metadata=k8s_client_lib.V1ObjectMeta( + annotations={_CONTAINER_EXECUTION_ID_KEY: "job-execution"} + ) + ) + ) + + assert changes == [_change("pod-execution"), _change("job-execution")] + + +def test_reads_the_id_off_the_pod_template_of_a_job_without_one(cluster): + launcher = _make_launcher() + changes = [] + launcher.subscribe_to_container_execution_changes(listener=changes.append) + + launcher._job_launcher._notify_container_execution_changed( + k8s_client_lib.V1Job( + metadata=k8s_client_lib.V1ObjectMeta(), + spec=k8s_client_lib.V1JobSpec( + template=k8s_client_lib.V1PodTemplateSpec( + metadata=k8s_client_lib.V1ObjectMeta( + annotations={_CONTAINER_EXECUTION_ID_KEY: "job-execution"} + ) + ) + ), + ) + ) + + assert changes == [_change("job-execution")] + + +def test_ignores_objects_without_a_container_execution_id(cluster): + launcher = _make_launcher() + changes = [] + launcher.subscribe_to_container_execution_changes(listener=changes.append) + + launcher._pod_launcher._notify_container_execution_changed(_pod(None)) + + assert changes == [] + + +def test_every_listener_gets_every_change_even_when_one_fails(cluster): + launcher = _make_launcher() + changes = [] + + def _failing_listener(change): + raise RuntimeError("listener failed") + + launcher.subscribe_to_container_execution_changes(listener=_failing_listener) + launcher.subscribe_to_container_execution_changes(listener=changes.append) + + launcher._pod_launcher._notify_container_execution_changed(_pod("pod-execution")) + + assert changes == [_change("pod-execution")] + + +def test_informer_reports_changes_after_prefilling_its_cache(monkeypatch): + monkeypatch.setattr( + kubernetes_launchers.k8s_informer_lib, "SharedInformer", _FakeSharedInformer + ) + on_change = object() + + informer = kubernetes_launchers.MultiNamespaceInformer( + list_func=None, initial_namespaces=["default"], on_change=on_change + ) + + assert informer._informers["default"].handlers == [ + (kubernetes_launchers.k8s_informer_lib.ADDED, on_change, True), + (kubernetes_launchers.k8s_informer_lib.MODIFIED, on_change, True), + (kubernetes_launchers.k8s_informer_lib.DELETED, on_change, True), + ] + + +def test_informer_without_on_change_adds_no_handlers(monkeypatch): + monkeypatch.setattr( + kubernetes_launchers.k8s_informer_lib, "SharedInformer", _FakeSharedInformer + ) + + informer = kubernetes_launchers.MultiNamespaceInformer( + list_func=None, initial_namespaces=["default"] + ) + + assert informer._informers["default"].handlers == [] diff --git a/tests/test_orchestrator_changed_container_executions.py b/tests/test_orchestrator_changed_container_executions.py new file mode 100644 index 0000000..c3b9123 --- /dev/null +++ b/tests/test_orchestrator_changed_container_executions.py @@ -0,0 +1,254 @@ +"""Tests for refreshing the container executions the launcher saw change ahead of the round-robin.""" + +import datetime +from typing import Any, Callable +from unittest import mock + +from sqlalchemy import orm +from sqlalchemy import sql + +from cloud_pipelines_backend import api_server_sql +from cloud_pipelines_backend import backend_types_sql as bts +from cloud_pipelines_backend import component_structures as structures +from cloud_pipelines_backend import database_ops +from cloud_pipelines_backend import orchestrator_sql +from cloud_pipelines_backend.launchers import interfaces as launcher_interfaces + + +def _make_launched_container(launcher_data: dict[str, Any]) -> mock.MagicMock: + return mock.MagicMock( + status=launcher_interfaces.ContainerStatus.PENDING, + to_dict=lambda: dict(launcher_data), + ) + + +def _container_task(image: str) -> structures.TaskSpec: + return structures.TaskSpec( + component_ref=structures.ComponentReference( + spec=structures.ComponentSpec( + implementation=structures.ContainerImplementation( + container=structures.ContainerSpec(image=image) + ) + ) + ) + ) + + +def _make_orchestrator( + session_factory: Callable[[], orm.Session], launcher: mock.MagicMock +) -> orchestrator_sql.OrchestratorService_Sql: + return orchestrator_sql.OrchestratorService_Sql( + session_factory=session_factory, + launcher=launcher, + storage_provider=mock.MagicMock(), + data_root_uri="file:///tmp/artifacts", + logs_root_uri="file:///tmp/logs", + ) + + +def _create_launched_container_executions( + task_count: int, +) -> tuple[Callable[[], orm.Session], mock.MagicMock]: + """A pipeline run with ``task_count`` launched, PENDING container executions, and the launcher that launched them.""" + db_engine = database_ops.create_db_engine_and_migrate_db(database_uri="sqlite://") + session_factory = lambda: orm.Session(bind=db_engine) + api_server_sql.PipelineRunsApiService_Sql().create( + session=session_factory(), + root_task=structures.TaskSpec( + component_ref=structures.ComponentReference( + spec=structures.ComponentSpec( + implementation=structures.GraphImplementation( + graph=structures.GraphSpec( + tasks={ + f"task{i}": _container_task(image=f"python:3.{i}") + for i in range(task_count) + } + ) + ), + ) + ) + ), + created_by="user1", + ) + launch_count = iter(range(task_count)) + launcher = mock.MagicMock( + launch_container_task=mock.MagicMock( + side_effect=lambda *a, **kw: _make_launched_container( + {"pod": f"pod-{next(launch_count)}"} + ) + ), + deserialize_launched_container_from_dict=mock.MagicMock( + side_effect=_make_launched_container + ), + get_refreshed_launched_container_from_dict=mock.MagicMock( + side_effect=_make_launched_container + ), + ) + orchestrator = _make_orchestrator(session_factory, launcher) + session = session_factory() + while orchestrator.internal_process_queued_executions_queue(session=session): + pass + return session_factory, launcher + + +def _container_execution_ids_by_pod( + session_factory: Callable[[], orm.Session], +) -> dict[str, str]: + return { + container_execution.launcher_data["pod"]: container_execution.id + for container_execution in session_factory().scalars( + sql.select(bts.ContainerExecution) + ) + } + + +def _notify(launcher: mock.MagicMock, container_execution_id: str) -> None: + """Sends a change through the listener the newest orchestrator subscribed with.""" + listener = launcher.subscribe_to_container_execution_changes.call_args.kwargs[ + "listener" + ] + listener( + launcher_interfaces.ContainerExecutionChange( + container_execution_id=container_execution_id + ) + ) + + +def _refreshed_pods(launcher: mock.MagicMock) -> list[str]: + return [ + call.args[0]["pod"] + for call in launcher.get_refreshed_launched_container_from_dict.call_args_list + ] + + +def _set_last_processed_at( + session_factory: Callable[[], orm.Session], + container_execution_id: str, + last_processed_at: datetime.datetime, +) -> None: + with session_factory() as session: + session.get( + bts.ContainerExecution, container_execution_id + ).last_processed_at = last_processed_at + session.commit() + + +def test_subscribes_to_the_launcher_once(): + launcher = mock.MagicMock() + + _make_orchestrator(lambda: None, launcher) + + launcher.subscribe_to_container_execution_changes.assert_called_once() + + +def test_refreshes_a_changed_container_execution_first(): + session_factory, launcher = _create_launched_container_executions(task_count=3) + changed_id = _container_execution_ids_by_pod(session_factory)["pod-2"] + orchestrator = _make_orchestrator(session_factory, launcher) + launcher.get_refreshed_launched_container_from_dict.reset_mock() + _notify(launcher, changed_id) + + orchestrator.internal_process_running_executions_queue(session=session_factory()) + + assert _refreshed_pods(launcher) == ["pod-2"] + + +def test_leaves_an_ended_container_execution_out_of_the_running_queue(): + session_factory, launcher = _create_launched_container_executions(task_count=1) + container_execution_id = _container_execution_ids_by_pod(session_factory)["pod-0"] + with session_factory() as session: + container_execution = session.get( + bts.ContainerExecution, container_execution_id + ) + container_execution.status = bts.ContainerExecutionStatus.SUCCEEDED + last_processed_at = container_execution.last_processed_at + session.commit() + orchestrator = _make_orchestrator(session_factory, launcher) + _notify(launcher, container_execution_id) + + orchestrator.internal_process_running_executions_queue(session=session_factory()) + + assert ( + session_factory() + .get(bts.ContainerExecution, container_execution_id) + .last_processed_at + == last_processed_at + ) + launcher.get_refreshed_launched_container_from_dict.assert_not_called() + + +def test_an_execution_unrefreshed_longer_than_the_priority_goes_before_a_changed_one(): + session_factory, launcher = _create_launched_container_executions(task_count=2) + ids_by_pod = _container_execution_ids_by_pod(session_factory) + _set_last_processed_at( + session_factory, + ids_by_pod["pod-0"], + orchestrator_sql._get_current_time() - datetime.timedelta(minutes=10), + ) + orchestrator = _make_orchestrator(session_factory, launcher) + _notify(launcher, ids_by_pod["pod-1"]) + + orchestrator.internal_process_running_executions_queue(session=session_factory()) + + assert _refreshed_pods(launcher) == ["pod-0"] + + +def test_executions_that_keep_changing_do_not_starve_one_that_changed_once(): + session_factory, launcher = _create_launched_container_executions(task_count=3) + ids_by_pod = _container_execution_ids_by_pod(session_factory) + orchestrator = _make_orchestrator(session_factory, launcher) + _notify(launcher, ids_by_pod["pod-2"]) + + for _ in range(3): + _notify(launcher, ids_by_pod["pod-0"]) + _notify(launcher, ids_by_pod["pod-1"]) + orchestrator.internal_process_running_executions_queue( + session=session_factory() + ) + + assert sorted(_refreshed_pods(launcher)) == ["pod-0", "pod-1", "pod-2"] + + +def test_a_change_seen_during_a_refresh_moves_the_execution_forward_again(): + session_factory, launcher = _create_launched_container_executions(task_count=2) + ids_by_pod = _container_execution_ids_by_pod(session_factory) + orchestrator = _make_orchestrator(session_factory, launcher) + + def _refresh_and_change_again(launcher_data): + if launcher_data["pod"] == "pod-1": + _notify(launcher, ids_by_pod["pod-1"]) + return _make_launched_container(launcher_data) + + launcher.get_refreshed_launched_container_from_dict.side_effect = ( + _refresh_and_change_again + ) + _notify(launcher, ids_by_pod["pod-1"]) + + for _ in range(2): + orchestrator.internal_process_running_executions_queue( + session=session_factory() + ) + + assert _refreshed_pods(launcher) == ["pod-1", "pod-1"] + + +def test_keeps_at_most_the_maximum_number_of_changed_ids(monkeypatch): + monkeypatch.setattr(orchestrator_sql, "_MAX_CHANGED_CONTAINER_EXECUTION_IDS", 2) + launcher = mock.MagicMock() + orchestrator = _make_orchestrator(lambda: None, launcher) + + for container_execution_id in ("first", "second", "third"): + _notify(launcher, container_execution_id) + + assert orchestrator._changed_container_execution_ids == {"first", "second"} + + +def test_launchers_report_no_changes_by_default(): + class _Launcher(launcher_interfaces.ContainerTaskLauncher): + def launch_container_task(self, **kwargs): + raise NotImplementedError() + + assert ( + _Launcher().subscribe_to_container_execution_changes(listener=mock.MagicMock()) + is None + )