From c7b30da75f36cde6e8fc6700354de9ff758ed46b Mon Sep 17 00:00:00 2001 From: Morgan Wowk Date: Tue, 3 Nov 2026 11:02:00 -0700 Subject: [PATCH] feat(orchestrator): Refresh a container execution as soon as its pod or job changes The running queue visits container executions round-robin by last_processed_at, so a pod that finished waits for every other active container execution to be visited first. ContainerTaskLauncher gets subscribe_to_container_execution_changes(), which calls a listener with a ContainerExecutionChange whenever a container the launcher launched changes. It does nothing by default, so launchers that can't tell keep today's behaviour. Changes are hints: the orchestrator still does its normal refresh, and a dropped or repeated change only means that execution is checked at its normal turn, or once more. The Kubernetes launchers stamp the container execution id annotation on the pods and jobs they launch, and their pod and job informers send a change for each changed object to every listener. One listener failing doesn't stop the others. The pod-or-job launcher passes the subscription to both of the launchers it wraps. The orchestrator subscribes once, when it is created, and collects the ids in a set behind a lock, keeping at most 10,000. Before each pick from the running queue it swaps the set out and moves the executions that are still PENDING or RUNNING forward, in one UPDATE. Only the loop thread writes to the database. A moved execution goes back two minutes rather than to a fixed oldest time, and only if it isn't already further ahead. Every informer re-list (after a 410 Gone, a watch error or a resync) reports every cached pod and job as modified. With a fixed time those executions would keep taking the front and starve the executions of every other launcher. Now the front is served in the order changes were recorded, and an execution left unrefreshed for longer than two minutes still goes first. A change recorded while its execution is being refreshed, such as the container finishing, moves it forward again. Pods and jobs launched before this change carry no id and keep the round-robin. --- .../launchers/interfaces.py | 16 ++ .../launchers/kubernetes_launchers.py | 88 +++++- cloud_pipelines_backend/orchestrator_sql.py | 56 ++++ ..._launchers_changed_container_executions.py | 258 ++++++++++++++++++ ...chestrator_changed_container_executions.py | 254 +++++++++++++++++ 5 files changed, 671 insertions(+), 1 deletion(-) create mode 100644 tests/launchers/kubernetes/test_kubernetes_launchers_changed_container_executions.py create mode 100644 tests/test_orchestrator_changed_container_executions.py diff --git a/cloud_pipelines_backend/launchers/interfaces.py b/cloud_pipelines_backend/launchers/interfaces.py index 365ddafa..26d2949f 100644 --- a/cloud_pipelines_backend/launchers/interfaces.py +++ b/cloud_pipelines_backend/launchers/interfaces.py @@ -60,6 +60,16 @@ class InputArgument: dynamic_data: str | dict[str, Any] | None = None +@dataclasses.dataclass(frozen=True, kw_only=True) +class ContainerExecutionChange: + """A launcher saw the container of a container execution it launched change.""" + + container_execution_id: str + + +ContainerExecutionChangeListener = typing.Callable[[ContainerExecutionChange], None] + + class ContainerTaskLauncher(typing.Generic[_TLaunchedContainer], abc.ABC): @abc.abstractmethod def launch_container_task( @@ -83,6 +93,12 @@ def get_refreshed_launched_container_from_dict( ) -> _TLaunchedContainer: raise NotImplementedError() + def subscribe_to_container_execution_changes( + self, *, listener: ContainerExecutionChangeListener + ) -> None: + """Calls `listener` whenever a container this launcher launched changes; does nothing when the launcher cannot tell.""" + del listener + class ContainerStatus(str, enum.Enum): PENDING = "PENDING" diff --git a/cloud_pipelines_backend/launchers/kubernetes_launchers.py b/cloud_pipelines_backend/launchers/kubernetes_launchers.py index a9d305ef..3b4dec10 100644 --- a/cloud_pipelines_backend/launchers/kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/kubernetes_launchers.py @@ -217,6 +217,10 @@ def __init__( ) self._pod_postprocessor = pod_postprocessor self._create_volume_and_volume_mount = _create_volume_and_volume_mount + self._container_execution_change_listeners: tuple[ + interfaces.ContainerExecutionChangeListener, ... + ] = () + self._container_execution_change_listeners_lock = threading.Lock() try: k8s_client_lib.VersionApi(self._api_client).get_code( @@ -246,6 +250,7 @@ def __init__( # But Tangle has only recently started labeling it's pods/jobs, so if we add filtering too soon, this will become a breaking change. # label_selector=f"{_TANGLE_KUBERNETES_LABEL_KEY}=true", label_selector=None, + on_change=self._notify_container_execution_changed, ) self._pod_informer.start() except Exception: @@ -648,6 +653,10 @@ def get_output_path(output_name: str) -> str: service_account_name=pod_service_account, ) + pod_annotations = (pod_annotations or {}) | _container_execution_id_annotation( + annotations + ) + pod = k8s_client_lib.V1Pod( api_version="v1", kind="Pod", @@ -674,6 +683,32 @@ def _choose_namespace( return self._namespace return self._namespace(annotations=annotations) + def subscribe_to_container_execution_changes( + self, *, listener: interfaces.ContainerExecutionChangeListener + ) -> None: + """Calls `listener` whenever a pod or job this launcher launched changes.""" + with self._container_execution_change_listeners_lock: + self._container_execution_change_listeners = ( + *self._container_execution_change_listeners, + listener, + ) + + def _notify_container_execution_changed( + self, obj: k8s_client_lib.V1Pod | k8s_client_lib.V1Job + ) -> None: + """Tells every listener that the container execution behind `obj` changed, so one failing listener can't stop the rest.""" + container_execution_id = _get_container_execution_id(obj) + if not container_execution_id: + return + change = interfaces.ContainerExecutionChange( + container_execution_id=container_execution_id + ) + for listener in self._container_execution_change_listeners: + try: + listener(change) + except Exception: + _logger.exception("Container execution change listener failed.") + class _KubernetesPodLauncher( _KubernetesContainerLauncherBase, @@ -1346,6 +1381,7 @@ def __init__( # But Tangle has only recently started labeling it's pods/jobs, so if we add filtering too soon, this will become a breaking change. # label_selector=f"{_TANGLE_KUBERNETES_LABEL_KEY}=true", label_selector=None, + on_change=self._notify_container_execution_changed, ) self._job_informer.start() except Exception: @@ -1532,7 +1568,8 @@ def launch_container_task( name=explicit_job_name, namespace=namespace, # annotations=self._pod_annotations, - annotations=_DEFAULT_KUBERNETES_ANNOTATIONS, + annotations=_DEFAULT_KUBERNETES_ANNOTATIONS + | _container_execution_id_annotation(annotations), # ! Kubernetes gotcha: If Job labels are empty, Kubernetes copies the labels from the Job's Pod template (but not annotations). # https://github.com/kubernetes/kubernetes/blob/b1dc9d16763d2546b8464a09307471f3411ab391/pkg/apis/batch/v1/defaults.go#L49-L52 # So, changing code from no labels to some default labels stops that special behavior, causing pod template labels to not being propagated to Job itself. @@ -2329,6 +2366,13 @@ def get_refreshed_launched_container_from_dict( ) return launched_container.get_refreshed() + def subscribe_to_container_execution_changes( + self, *, listener: interfaces.ContainerExecutionChangeListener + ) -> None: + """Subscribes `listener` to the changes of both the pod and the job launcher.""" + self._pod_launcher.subscribe_to_container_execution_changes(listener=listener) + self._job_launcher.subscribe_to_container_execution_changes(listener=listener) + def deserialize_launched_container_from_dict( self, launched_container_dict: dict ) -> "LaunchedKubernetesContainer | LaunchedKubernetesJob": @@ -2401,6 +2445,37 @@ def __init__( ) +def _container_execution_id_annotation( + annotations: dict[str, Any] | None, +) -> dict[str, str]: + """The container execution id annotation from a task's annotations, to stamp on its pod or job; empty when absent.""" + container_execution_id = (annotations or {}).get( + common_annotations.CONTAINER_EXECUTION_ID_ANNOTATION_KEY + ) + if not container_execution_id: + return {} + return { + common_annotations.CONTAINER_EXECUTION_ID_ANNOTATION_KEY: container_execution_id + } + + +def _get_container_execution_id( + obj: k8s_client_lib.V1Pod | k8s_client_lib.V1Job, +) -> str | None: + """Reads the container execution id annotation off a pod or a job, falling back to a job's pod template.""" + metadatas = [obj.metadata] + if isinstance(obj, k8s_client_lib.V1Job) and obj.spec and obj.spec.template: + metadatas.append(obj.spec.template.metadata) + for metadata in metadatas: + if metadata and metadata.annotations: + container_execution_id = metadata.annotations.get( + common_annotations.CONTAINER_EXECUTION_ID_ANNOTATION_KEY + ) + if container_execution_id: + return container_execution_id + return None + + def _get_job_pod_completion_index(pod: k8s_client_lib.V1Pod) -> str | None: if not (pod.metadata and pod.metadata.annotations): return None @@ -2561,12 +2636,16 @@ def __init__( label_selector=None, field_selector=None, key_func=None, + on_change: ( + typing.Callable[[k8s_client_lib.V1Pod | k8s_client_lib.V1Job], None] | None + ) = None, ): self._list_func = list_func self._resync_period = resync_period self._label_selector = label_selector self._field_selector = field_selector self._key_func = key_func + self._on_change = on_change self._informers: dict[str, k8s_informer_lib.SharedInformer] = {} self._lock = threading.Lock() self._is_active = False @@ -2592,6 +2671,13 @@ def _get_or_create_informer( ) # Prefill the cache informer._initial_list() + if self._on_change: + for event_type in ( + k8s_informer_lib.ADDED, + k8s_informer_lib.MODIFIED, + k8s_informer_lib.DELETED, + ): + informer.add_event_handler(event_type, self._on_change) self._informers[namespace] = informer if self._is_active: informer.start() diff --git a/cloud_pipelines_backend/orchestrator_sql.py b/cloud_pipelines_backend/orchestrator_sql.py index 246a6077..3f86cbfb 100644 --- a/cloud_pipelines_backend/orchestrator_sql.py +++ b/cloud_pipelines_backend/orchestrator_sql.py @@ -4,6 +4,7 @@ import logging import itertools import os +import threading import time import traceback import typing @@ -36,6 +37,10 @@ DYNAMIC_DATA_SECRET_KEY = "secret" DYNAMIC_DATA_SECRET_NAME_KEY = "name" +_CHANGED_CONTAINER_EXECUTION_PRIORITY = datetime.timedelta(minutes=2) + +_MAX_CHANGED_CONTAINER_EXECUTION_IDS = 10_000 + class OrchestratorError(RuntimeError): pass @@ -94,6 +99,11 @@ def __init__( self._max_queue_batch_size = _max_queue_batch_size self._max_queue_batch_duration = _max_queue_batch_duration self._queued_execution_interceptor = queued_execution_interceptor + self._changed_container_execution_ids: set[str] = set() + self._changed_container_execution_ids_lock = threading.Lock() + self._launcher.subscribe_to_container_execution_changes( + listener=self._record_container_execution_change + ) def run_loop(self): while True: @@ -197,6 +207,7 @@ def internal_process_queued_executions_queue(self, session: orm.Session): return True def internal_process_running_executions_queue(self, session: orm.Session): + self._move_changed_container_executions_to_front(session=session) _logger.debug("Selecting running container execution to process.)") # Select only the ID to avoid loading large JSON columns into the sort buffer. @@ -341,6 +352,51 @@ def internal_process_running_executions_queue(self, session: orm.Session): ) return True + def _record_container_execution_change( + self, change: launcher_interfaces.ContainerExecutionChange + ) -> None: + """Records a container execution the launcher saw change, for the next running sweep to check first. + + Runs on the launcher's thread; past the cap new ids are dropped, so those executions just wait their normal turn. + """ + with self._changed_container_execution_ids_lock: + if ( + len(self._changed_container_execution_ids) + < _MAX_CHANGED_CONTAINER_EXECUTION_IDS + ): + self._changed_container_execution_ids.add(change.container_execution_id) + + def _move_changed_container_executions_to_front(self, session: orm.Session) -> None: + """Moves each pending or running execution the launcher saw change to `_CHANGED_CONTAINER_EXECUTION_PRIORITY` ago, unless it's already further ahead. + + Changes are served in the order recorded, an execution that keeps changing can't starve the rest, and a change + seen mid-refresh (such as the container finishing) moves that execution forward again. + """ + with self._changed_container_execution_ids_lock: + container_execution_ids = self._changed_container_execution_ids + self._changed_container_execution_ids = set() + if not container_execution_ids: + return + front_of_queue_last_processed_at = ( + _get_current_time() - _CHANGED_CONTAINER_EXECUTION_PRIORITY + ) + session.execute( + sql.update(bts.ContainerExecution) + .where( + bts.ContainerExecution.id.in_(container_execution_ids), + bts.ContainerExecution.status.in_( + ( + bts.ContainerExecutionStatus.PENDING, + bts.ContainerExecutionStatus.RUNNING, + ) + ), + bts.ContainerExecution.last_processed_at + > front_of_queue_last_processed_at, + ) + .values(last_processed_at=front_of_queue_last_processed_at) + ) + session.commit() + def internal_process_one_queued_execution( self, session: orm.Session, execution: bts.ExecutionNode ): diff --git a/tests/launchers/kubernetes/test_kubernetes_launchers_changed_container_executions.py b/tests/launchers/kubernetes/test_kubernetes_launchers_changed_container_executions.py new file mode 100644 index 00000000..84768822 --- /dev/null +++ b/tests/launchers/kubernetes/test_kubernetes_launchers_changed_container_executions.py @@ -0,0 +1,258 @@ +"""Tests for the Kubernetes launchers reporting which container executions' pods or jobs changed. + +The Kubernetes API calls are faked so that the tests run offline. +""" + +from __future__ import annotations + +import pathlib + +import pytest +from kubernetes import client as k8s_client_lib + +from cloud_pipelines_backend import component_structures as structures +from cloud_pipelines_backend.launchers import common_annotations +from cloud_pipelines_backend.launchers import interfaces +from cloud_pipelines_backend.launchers import kubernetes_launchers + +_CONTAINER_EXECUTION_ID_KEY = common_annotations.CONTAINER_EXECUTION_ID_ANNOTATION_KEY + + +class _FakeVersionApi: + def __init__(self, api_client=None): + pass + + def get_code(self, **kwargs): + return None + + +class _FakeCluster: + """Records the created pods and jobs.""" + + def __init__(self): + self.created_pods: list[k8s_client_lib.V1Pod] = [] + self.created_jobs: list[k8s_client_lib.V1Job] = [] + + def make_core_api(self): + cluster = self + + class _FakeCoreV1Api: + def __init__(self, api_client=None): + pass + + def create_namespaced_pod(self, namespace, body, **kwargs): + cluster.created_pods.append(body) + body.metadata.name = (body.metadata.generate_name or "") + "abcde" + body.metadata.namespace = namespace + return body + + def create_namespaced_service(self, namespace, body, **kwargs): + return body + + return _FakeCoreV1Api + + def make_batch_api(self): + cluster = self + + class _FakeBatchV1Api: + def __init__(self, api_client=None): + pass + + def create_namespaced_job(self, namespace, body, **kwargs): + cluster.created_jobs.append(body) + body.metadata.uid = "job-uid" + return body + + return _FakeBatchV1Api + + +class _FakeSharedInformer: + """Records the event handlers it is given and whether they were added after the cache was prefilled.""" + + def __init__(self, **kwargs): + self.handlers: list[tuple[str, object, bool]] = [] + self.listed = False + + def _initial_list(self): + self.listed = True + + def add_event_handler(self, event_type, handler): + self.handlers.append((event_type, handler, self.listed)) + + def start(self): + pass + + +@pytest.fixture +def cluster(monkeypatch) -> _FakeCluster: + cluster = _FakeCluster() + monkeypatch.setattr(k8s_client_lib, "VersionApi", _FakeVersionApi) + monkeypatch.setattr(k8s_client_lib, "CoreV1Api", cluster.make_core_api()) + monkeypatch.setattr(k8s_client_lib, "BatchV1Api", cluster.make_batch_api()) + + def _no_informer(**kwargs): + raise RuntimeError("Informers are disabled in tests.") + + monkeypatch.setattr(kubernetes_launchers, "MultiNamespaceInformer", _no_informer) + return cluster + + +def _make_launcher() -> ( + kubernetes_launchers.Local_Kubernetes_UsingHostPathStorage_KubernetesPodOrJobLauncher +): + return kubernetes_launchers.Local_Kubernetes_UsingHostPathStorage_KubernetesPodOrJobLauncher( + api_client=k8s_client_lib.ApiClient(), + ) + + +def _launch(launcher, tmp_path: pathlib.Path, annotations: dict[str, str]) -> None: + launcher.launch_container_task( + component_spec=structures.ComponentSpec( + name="test", + implementation=structures.ContainerImplementation( + container=structures.ContainerSpec(image="alpine") + ), + ), + input_arguments={}, + output_uris={}, + log_uri=str(tmp_path / "log.txt"), + annotations={_CONTAINER_EXECUTION_ID_KEY: "123", **annotations}, + ) + + +def test_launched_pod_carries_the_container_execution_id(cluster, tmp_path): + _launch(_make_launcher(), tmp_path, {}) + + [pod] = cluster.created_pods + assert pod.metadata.annotations[_CONTAINER_EXECUTION_ID_KEY] == "123" + + +def test_launched_job_and_its_pod_template_carry_the_container_execution_id( + cluster, tmp_path +): + _launch( + _make_launcher(), + tmp_path, + {kubernetes_launchers.RETRIES_MAX_RETRIES_ANNOTATION_KEY: "1"}, + ) + + [job] = cluster.created_jobs + assert job.metadata.annotations[_CONTAINER_EXECUTION_ID_KEY] == "123" + assert job.spec.template.metadata.annotations[_CONTAINER_EXECUTION_ID_KEY] == "123" + + +def test_launching_does_not_change_the_launcher_pod_annotations(cluster, tmp_path): + launcher = _make_launcher() + + _launch(launcher, tmp_path, {}) + + assert _CONTAINER_EXECUTION_ID_KEY not in launcher._pod_launcher._pod_annotations + + +def _pod(container_execution_id: str | None) -> k8s_client_lib.V1Pod: + annotations = ( + {_CONTAINER_EXECUTION_ID_KEY: container_execution_id} + if container_execution_id + else None + ) + return k8s_client_lib.V1Pod( + metadata=k8s_client_lib.V1ObjectMeta(annotations=annotations) + ) + + +def _change(container_execution_id: str) -> interfaces.ContainerExecutionChange: + return interfaces.ContainerExecutionChange( + container_execution_id=container_execution_id + ) + + +def test_tells_the_listener_about_each_changed_pod_and_job(cluster): + launcher = _make_launcher() + changes = [] + launcher.subscribe_to_container_execution_changes(listener=changes.append) + + launcher._pod_launcher._notify_container_execution_changed(_pod("pod-execution")) + launcher._job_launcher._notify_container_execution_changed( + k8s_client_lib.V1Job( + metadata=k8s_client_lib.V1ObjectMeta( + annotations={_CONTAINER_EXECUTION_ID_KEY: "job-execution"} + ) + ) + ) + + assert changes == [_change("pod-execution"), _change("job-execution")] + + +def test_reads_the_id_off_the_pod_template_of_a_job_without_one(cluster): + launcher = _make_launcher() + changes = [] + launcher.subscribe_to_container_execution_changes(listener=changes.append) + + launcher._job_launcher._notify_container_execution_changed( + k8s_client_lib.V1Job( + metadata=k8s_client_lib.V1ObjectMeta(), + spec=k8s_client_lib.V1JobSpec( + template=k8s_client_lib.V1PodTemplateSpec( + metadata=k8s_client_lib.V1ObjectMeta( + annotations={_CONTAINER_EXECUTION_ID_KEY: "job-execution"} + ) + ) + ), + ) + ) + + assert changes == [_change("job-execution")] + + +def test_ignores_objects_without_a_container_execution_id(cluster): + launcher = _make_launcher() + changes = [] + launcher.subscribe_to_container_execution_changes(listener=changes.append) + + launcher._pod_launcher._notify_container_execution_changed(_pod(None)) + + assert changes == [] + + +def test_every_listener_gets_every_change_even_when_one_fails(cluster): + launcher = _make_launcher() + changes = [] + + def _failing_listener(change): + raise RuntimeError("listener failed") + + launcher.subscribe_to_container_execution_changes(listener=_failing_listener) + launcher.subscribe_to_container_execution_changes(listener=changes.append) + + launcher._pod_launcher._notify_container_execution_changed(_pod("pod-execution")) + + assert changes == [_change("pod-execution")] + + +def test_informer_reports_changes_after_prefilling_its_cache(monkeypatch): + monkeypatch.setattr( + kubernetes_launchers.k8s_informer_lib, "SharedInformer", _FakeSharedInformer + ) + on_change = object() + + informer = kubernetes_launchers.MultiNamespaceInformer( + list_func=None, initial_namespaces=["default"], on_change=on_change + ) + + assert informer._informers["default"].handlers == [ + (kubernetes_launchers.k8s_informer_lib.ADDED, on_change, True), + (kubernetes_launchers.k8s_informer_lib.MODIFIED, on_change, True), + (kubernetes_launchers.k8s_informer_lib.DELETED, on_change, True), + ] + + +def test_informer_without_on_change_adds_no_handlers(monkeypatch): + monkeypatch.setattr( + kubernetes_launchers.k8s_informer_lib, "SharedInformer", _FakeSharedInformer + ) + + informer = kubernetes_launchers.MultiNamespaceInformer( + list_func=None, initial_namespaces=["default"] + ) + + assert informer._informers["default"].handlers == [] diff --git a/tests/test_orchestrator_changed_container_executions.py b/tests/test_orchestrator_changed_container_executions.py new file mode 100644 index 00000000..c3b9123f --- /dev/null +++ b/tests/test_orchestrator_changed_container_executions.py @@ -0,0 +1,254 @@ +"""Tests for refreshing the container executions the launcher saw change ahead of the round-robin.""" + +import datetime +from typing import Any, Callable +from unittest import mock + +from sqlalchemy import orm +from sqlalchemy import sql + +from cloud_pipelines_backend import api_server_sql +from cloud_pipelines_backend import backend_types_sql as bts +from cloud_pipelines_backend import component_structures as structures +from cloud_pipelines_backend import database_ops +from cloud_pipelines_backend import orchestrator_sql +from cloud_pipelines_backend.launchers import interfaces as launcher_interfaces + + +def _make_launched_container(launcher_data: dict[str, Any]) -> mock.MagicMock: + return mock.MagicMock( + status=launcher_interfaces.ContainerStatus.PENDING, + to_dict=lambda: dict(launcher_data), + ) + + +def _container_task(image: str) -> structures.TaskSpec: + return structures.TaskSpec( + component_ref=structures.ComponentReference( + spec=structures.ComponentSpec( + implementation=structures.ContainerImplementation( + container=structures.ContainerSpec(image=image) + ) + ) + ) + ) + + +def _make_orchestrator( + session_factory: Callable[[], orm.Session], launcher: mock.MagicMock +) -> orchestrator_sql.OrchestratorService_Sql: + return orchestrator_sql.OrchestratorService_Sql( + session_factory=session_factory, + launcher=launcher, + storage_provider=mock.MagicMock(), + data_root_uri="file:///tmp/artifacts", + logs_root_uri="file:///tmp/logs", + ) + + +def _create_launched_container_executions( + task_count: int, +) -> tuple[Callable[[], orm.Session], mock.MagicMock]: + """A pipeline run with ``task_count`` launched, PENDING container executions, and the launcher that launched them.""" + db_engine = database_ops.create_db_engine_and_migrate_db(database_uri="sqlite://") + session_factory = lambda: orm.Session(bind=db_engine) + api_server_sql.PipelineRunsApiService_Sql().create( + session=session_factory(), + root_task=structures.TaskSpec( + component_ref=structures.ComponentReference( + spec=structures.ComponentSpec( + implementation=structures.GraphImplementation( + graph=structures.GraphSpec( + tasks={ + f"task{i}": _container_task(image=f"python:3.{i}") + for i in range(task_count) + } + ) + ), + ) + ) + ), + created_by="user1", + ) + launch_count = iter(range(task_count)) + launcher = mock.MagicMock( + launch_container_task=mock.MagicMock( + side_effect=lambda *a, **kw: _make_launched_container( + {"pod": f"pod-{next(launch_count)}"} + ) + ), + deserialize_launched_container_from_dict=mock.MagicMock( + side_effect=_make_launched_container + ), + get_refreshed_launched_container_from_dict=mock.MagicMock( + side_effect=_make_launched_container + ), + ) + orchestrator = _make_orchestrator(session_factory, launcher) + session = session_factory() + while orchestrator.internal_process_queued_executions_queue(session=session): + pass + return session_factory, launcher + + +def _container_execution_ids_by_pod( + session_factory: Callable[[], orm.Session], +) -> dict[str, str]: + return { + container_execution.launcher_data["pod"]: container_execution.id + for container_execution in session_factory().scalars( + sql.select(bts.ContainerExecution) + ) + } + + +def _notify(launcher: mock.MagicMock, container_execution_id: str) -> None: + """Sends a change through the listener the newest orchestrator subscribed with.""" + listener = launcher.subscribe_to_container_execution_changes.call_args.kwargs[ + "listener" + ] + listener( + launcher_interfaces.ContainerExecutionChange( + container_execution_id=container_execution_id + ) + ) + + +def _refreshed_pods(launcher: mock.MagicMock) -> list[str]: + return [ + call.args[0]["pod"] + for call in launcher.get_refreshed_launched_container_from_dict.call_args_list + ] + + +def _set_last_processed_at( + session_factory: Callable[[], orm.Session], + container_execution_id: str, + last_processed_at: datetime.datetime, +) -> None: + with session_factory() as session: + session.get( + bts.ContainerExecution, container_execution_id + ).last_processed_at = last_processed_at + session.commit() + + +def test_subscribes_to_the_launcher_once(): + launcher = mock.MagicMock() + + _make_orchestrator(lambda: None, launcher) + + launcher.subscribe_to_container_execution_changes.assert_called_once() + + +def test_refreshes_a_changed_container_execution_first(): + session_factory, launcher = _create_launched_container_executions(task_count=3) + changed_id = _container_execution_ids_by_pod(session_factory)["pod-2"] + orchestrator = _make_orchestrator(session_factory, launcher) + launcher.get_refreshed_launched_container_from_dict.reset_mock() + _notify(launcher, changed_id) + + orchestrator.internal_process_running_executions_queue(session=session_factory()) + + assert _refreshed_pods(launcher) == ["pod-2"] + + +def test_leaves_an_ended_container_execution_out_of_the_running_queue(): + session_factory, launcher = _create_launched_container_executions(task_count=1) + container_execution_id = _container_execution_ids_by_pod(session_factory)["pod-0"] + with session_factory() as session: + container_execution = session.get( + bts.ContainerExecution, container_execution_id + ) + container_execution.status = bts.ContainerExecutionStatus.SUCCEEDED + last_processed_at = container_execution.last_processed_at + session.commit() + orchestrator = _make_orchestrator(session_factory, launcher) + _notify(launcher, container_execution_id) + + orchestrator.internal_process_running_executions_queue(session=session_factory()) + + assert ( + session_factory() + .get(bts.ContainerExecution, container_execution_id) + .last_processed_at + == last_processed_at + ) + launcher.get_refreshed_launched_container_from_dict.assert_not_called() + + +def test_an_execution_unrefreshed_longer_than_the_priority_goes_before_a_changed_one(): + session_factory, launcher = _create_launched_container_executions(task_count=2) + ids_by_pod = _container_execution_ids_by_pod(session_factory) + _set_last_processed_at( + session_factory, + ids_by_pod["pod-0"], + orchestrator_sql._get_current_time() - datetime.timedelta(minutes=10), + ) + orchestrator = _make_orchestrator(session_factory, launcher) + _notify(launcher, ids_by_pod["pod-1"]) + + orchestrator.internal_process_running_executions_queue(session=session_factory()) + + assert _refreshed_pods(launcher) == ["pod-0"] + + +def test_executions_that_keep_changing_do_not_starve_one_that_changed_once(): + session_factory, launcher = _create_launched_container_executions(task_count=3) + ids_by_pod = _container_execution_ids_by_pod(session_factory) + orchestrator = _make_orchestrator(session_factory, launcher) + _notify(launcher, ids_by_pod["pod-2"]) + + for _ in range(3): + _notify(launcher, ids_by_pod["pod-0"]) + _notify(launcher, ids_by_pod["pod-1"]) + orchestrator.internal_process_running_executions_queue( + session=session_factory() + ) + + assert sorted(_refreshed_pods(launcher)) == ["pod-0", "pod-1", "pod-2"] + + +def test_a_change_seen_during_a_refresh_moves_the_execution_forward_again(): + session_factory, launcher = _create_launched_container_executions(task_count=2) + ids_by_pod = _container_execution_ids_by_pod(session_factory) + orchestrator = _make_orchestrator(session_factory, launcher) + + def _refresh_and_change_again(launcher_data): + if launcher_data["pod"] == "pod-1": + _notify(launcher, ids_by_pod["pod-1"]) + return _make_launched_container(launcher_data) + + launcher.get_refreshed_launched_container_from_dict.side_effect = ( + _refresh_and_change_again + ) + _notify(launcher, ids_by_pod["pod-1"]) + + for _ in range(2): + orchestrator.internal_process_running_executions_queue( + session=session_factory() + ) + + assert _refreshed_pods(launcher) == ["pod-1", "pod-1"] + + +def test_keeps_at_most_the_maximum_number_of_changed_ids(monkeypatch): + monkeypatch.setattr(orchestrator_sql, "_MAX_CHANGED_CONTAINER_EXECUTION_IDS", 2) + launcher = mock.MagicMock() + orchestrator = _make_orchestrator(lambda: None, launcher) + + for container_execution_id in ("first", "second", "third"): + _notify(launcher, container_execution_id) + + assert orchestrator._changed_container_execution_ids == {"first", "second"} + + +def test_launchers_report_no_changes_by_default(): + class _Launcher(launcher_interfaces.ContainerTaskLauncher): + def launch_container_task(self, **kwargs): + raise NotImplementedError() + + assert ( + _Launcher().subscribe_to_container_execution_changes(listener=mock.MagicMock()) + is None + )