Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions cloud_pipelines_backend/launchers/interfaces.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,16 @@ class InputArgument:
dynamic_data: str | dict[str, Any] | None = None


@dataclasses.dataclass(frozen=True, kw_only=True)
class ContainerExecutionChange:
"""A launcher saw the container of a container execution it launched change."""

container_execution_id: str


ContainerExecutionChangeListener = typing.Callable[[ContainerExecutionChange], None]


class ContainerTaskLauncher(typing.Generic[_TLaunchedContainer], abc.ABC):
@abc.abstractmethod
def launch_container_task(
Expand All @@ -83,6 +93,12 @@ def get_refreshed_launched_container_from_dict(
) -> _TLaunchedContainer:
raise NotImplementedError()

def subscribe_to_container_execution_changes(
self, *, listener: ContainerExecutionChangeListener
) -> None:
"""Calls `listener` whenever a container this launcher launched changes; does nothing when the launcher cannot tell."""
del listener


class ContainerStatus(str, enum.Enum):
PENDING = "PENDING"
Expand Down
88 changes: 87 additions & 1 deletion cloud_pipelines_backend/launchers/kubernetes_launchers.py
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,10 @@ def __init__(
)
self._pod_postprocessor = pod_postprocessor
self._create_volume_and_volume_mount = _create_volume_and_volume_mount
self._container_execution_change_listeners: tuple[
interfaces.ContainerExecutionChangeListener, ...
] = ()
self._container_execution_change_listeners_lock = threading.Lock()

try:
k8s_client_lib.VersionApi(self._api_client).get_code(
Expand Down Expand Up @@ -246,6 +250,7 @@ def __init__(
# But Tangle has only recently started labeling it's pods/jobs, so if we add filtering too soon, this will become a breaking change.
# label_selector=f"{_TANGLE_KUBERNETES_LABEL_KEY}=true",
label_selector=None,
on_change=self._notify_container_execution_changed,
)
self._pod_informer.start()
except Exception:
Expand Down Expand Up @@ -648,6 +653,10 @@ def get_output_path(output_name: str) -> str:
service_account_name=pod_service_account,
)

pod_annotations = (pod_annotations or {}) | _container_execution_id_annotation(
annotations
)

pod = k8s_client_lib.V1Pod(
api_version="v1",
kind="Pod",
Expand All @@ -674,6 +683,32 @@ def _choose_namespace(
return self._namespace
return self._namespace(annotations=annotations)

def subscribe_to_container_execution_changes(
self, *, listener: interfaces.ContainerExecutionChangeListener
) -> None:
"""Calls `listener` whenever a pod or job this launcher launched changes."""
with self._container_execution_change_listeners_lock:
self._container_execution_change_listeners = (
*self._container_execution_change_listeners,
listener,
)

def _notify_container_execution_changed(
self, obj: k8s_client_lib.V1Pod | k8s_client_lib.V1Job
) -> None:
"""Tells every listener that the container execution behind `obj` changed, so one failing listener can't stop the rest."""
container_execution_id = _get_container_execution_id(obj)
if not container_execution_id:
return
change = interfaces.ContainerExecutionChange(
container_execution_id=container_execution_id
)
for listener in self._container_execution_change_listeners:
try:
listener(change)
except Exception:
_logger.exception("Container execution change listener failed.")


class _KubernetesPodLauncher(
_KubernetesContainerLauncherBase,
Expand Down Expand Up @@ -1346,6 +1381,7 @@ def __init__(
# But Tangle has only recently started labeling it's pods/jobs, so if we add filtering too soon, this will become a breaking change.
# label_selector=f"{_TANGLE_KUBERNETES_LABEL_KEY}=true",
label_selector=None,
on_change=self._notify_container_execution_changed,
)
self._job_informer.start()
except Exception:
Expand Down Expand Up @@ -1532,7 +1568,8 @@ def launch_container_task(
name=explicit_job_name,
namespace=namespace,
# annotations=self._pod_annotations,
annotations=_DEFAULT_KUBERNETES_ANNOTATIONS,
annotations=_DEFAULT_KUBERNETES_ANNOTATIONS
| _container_execution_id_annotation(annotations),
# ! Kubernetes gotcha: If Job labels are empty, Kubernetes copies the labels from the Job's Pod template (but not annotations).
# https://github.com/kubernetes/kubernetes/blob/b1dc9d16763d2546b8464a09307471f3411ab391/pkg/apis/batch/v1/defaults.go#L49-L52
# So, changing code from no labels to some default labels stops that special behavior, causing pod template labels to not being propagated to Job itself.
Expand Down Expand Up @@ -2329,6 +2366,13 @@ def get_refreshed_launched_container_from_dict(
)
return launched_container.get_refreshed()

def subscribe_to_container_execution_changes(
self, *, listener: interfaces.ContainerExecutionChangeListener
) -> None:
"""Subscribes `listener` to the changes of both the pod and the job launcher."""
self._pod_launcher.subscribe_to_container_execution_changes(listener=listener)
self._job_launcher.subscribe_to_container_execution_changes(listener=listener)

def deserialize_launched_container_from_dict(
self, launched_container_dict: dict
) -> "LaunchedKubernetesContainer | LaunchedKubernetesJob":
Expand Down Expand Up @@ -2401,6 +2445,37 @@ def __init__(
)


def _container_execution_id_annotation(
annotations: dict[str, Any] | None,
) -> dict[str, str]:
"""The container execution id annotation from a task's annotations, to stamp on its pod or job; empty when absent."""
container_execution_id = (annotations or {}).get(
common_annotations.CONTAINER_EXECUTION_ID_ANNOTATION_KEY
)
if not container_execution_id:
return {}
return {
common_annotations.CONTAINER_EXECUTION_ID_ANNOTATION_KEY: container_execution_id
}


def _get_container_execution_id(
obj: k8s_client_lib.V1Pod | k8s_client_lib.V1Job,
) -> str | None:
"""Reads the container execution id annotation off a pod or a job, falling back to a job's pod template."""
metadatas = [obj.metadata]
if isinstance(obj, k8s_client_lib.V1Job) and obj.spec and obj.spec.template:
metadatas.append(obj.spec.template.metadata)
for metadata in metadatas:
if metadata and metadata.annotations:
container_execution_id = metadata.annotations.get(
common_annotations.CONTAINER_EXECUTION_ID_ANNOTATION_KEY
)
if container_execution_id:
return container_execution_id
return None


def _get_job_pod_completion_index(pod: k8s_client_lib.V1Pod) -> str | None:
if not (pod.metadata and pod.metadata.annotations):
return None
Expand Down Expand Up @@ -2561,12 +2636,16 @@ def __init__(
label_selector=None,
field_selector=None,
key_func=None,
on_change: (
typing.Callable[[k8s_client_lib.V1Pod | k8s_client_lib.V1Job], None] | None
) = None,
):
self._list_func = list_func
self._resync_period = resync_period
self._label_selector = label_selector
self._field_selector = field_selector
self._key_func = key_func
self._on_change = on_change
self._informers: dict[str, k8s_informer_lib.SharedInformer] = {}
self._lock = threading.Lock()
self._is_active = False
Expand All @@ -2592,6 +2671,13 @@ def _get_or_create_informer(
)
# Prefill the cache
informer._initial_list()
if self._on_change:
for event_type in (
k8s_informer_lib.ADDED,
k8s_informer_lib.MODIFIED,
k8s_informer_lib.DELETED,
):
informer.add_event_handler(event_type, self._on_change)
self._informers[namespace] = informer
if self._is_active:
informer.start()
Expand Down
56 changes: 56 additions & 0 deletions cloud_pipelines_backend/orchestrator_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import logging
import itertools
import os
import threading
import time
import traceback
import typing
Expand Down Expand Up @@ -36,6 +37,10 @@
DYNAMIC_DATA_SECRET_KEY = "secret"
DYNAMIC_DATA_SECRET_NAME_KEY = "name"

_CHANGED_CONTAINER_EXECUTION_PRIORITY = datetime.timedelta(minutes=2)

_MAX_CHANGED_CONTAINER_EXECUTION_IDS = 10_000


class OrchestratorError(RuntimeError):
pass
Expand Down Expand Up @@ -94,6 +99,11 @@ def __init__(
self._max_queue_batch_size = _max_queue_batch_size
self._max_queue_batch_duration = _max_queue_batch_duration
self._queued_execution_interceptor = queued_execution_interceptor
self._changed_container_execution_ids: set[str] = set()
self._changed_container_execution_ids_lock = threading.Lock()
self._launcher.subscribe_to_container_execution_changes(
listener=self._record_container_execution_change
)

def run_loop(self):
while True:
Expand Down Expand Up @@ -197,6 +207,7 @@ def internal_process_queued_executions_queue(self, session: orm.Session):
return True

def internal_process_running_executions_queue(self, session: orm.Session):
self._move_changed_container_executions_to_front(session=session)
_logger.debug("Selecting running container execution to process.)")

# Select only the ID to avoid loading large JSON columns into the sort buffer.
Expand Down Expand Up @@ -341,6 +352,51 @@ def internal_process_running_executions_queue(self, session: orm.Session):
)
return True

def _record_container_execution_change(
self, change: launcher_interfaces.ContainerExecutionChange
) -> None:
"""Records a container execution the launcher saw change, for the next running sweep to check first.

Runs on the launcher's thread; past the cap new ids are dropped, so those executions just wait their normal turn.
"""
with self._changed_container_execution_ids_lock:
if (
len(self._changed_container_execution_ids)
< _MAX_CHANGED_CONTAINER_EXECUTION_IDS
):
self._changed_container_execution_ids.add(change.container_execution_id)

def _move_changed_container_executions_to_front(self, session: orm.Session) -> None:
"""Moves each pending or running execution the launcher saw change to `_CHANGED_CONTAINER_EXECUTION_PRIORITY` ago, unless it's already further ahead.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This might behave differently from expectations in certain cases. For example if the queue is long and a normal full running execution queue sweep takes more than _CHANGED_CONTAINER_EXECUTION_PRIORITY, then the recently changed executions will be processed with arbitrary delay.

This seems to be the natural conflict between processing the "normal" queue and the listener-based queue. You can make the problem fixable by making _CHANGED_CONTAINER_EXECUTION_PRIORITY configurable.

Non-blocking.


Changes are served in the order recorded, an execution that keeps changing can't starve the rest, and a change

@Ark-kun Ark-kun Nov 17, 2026 •

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

an execution that keeps changing can't starve the rest

Clever idea. A bit non-obvious to prove, but I've verified it's correct.

seen mid-refresh (such as the container finishing) moves that execution forward again.
"""
with self._changed_container_execution_ids_lock:
container_execution_ids = self._changed_container_execution_ids
self._changed_container_execution_ids = set()
if not container_execution_ids:
return
front_of_queue_last_processed_at = (
_get_current_time() - _CHANGED_CONTAINER_EXECUTION_PRIORITY

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

datetime.now can be non-monotonic. Fortunately UTC timezone does not experience daylight saving time-travel. But time can still travel back when clock is adjusted.

Unlikely to be a big problem.

)
session.execute(
sql.update(bts.ContainerExecution)
.where(
bts.ContainerExecution.id.in_(container_execution_ids),
bts.ContainerExecution.status.in_(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Clever. This fixes a small race condition (caused by multi-threaded informers). Maybe we can leave a short comment about this.

The race condition is as follows:

  • Informer catches pod X Revision A
  • Orchestrator pops the list of changed IDs which includes X. The launcher's list is now empty.
  • Informer catches pod X Revision B (Succeeded). It's added to the list of changed executions.
  • Orchestrator refreshes the execution and sees revision B (Succeeded) updates the DB with new state of X.
  • Orchestrator pops the list of changed IDs which includes X again. Orchestrator overwrites X's last_processed_at.

The case is really minor (incorrect last_processed_at is not a big deal), but I'm glad you avoided this issue.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤖 AI-assisted: Kept in c7b30da, and the case is now in the _move_changed_container_executions_to_front docstring. test_a_change_seen_during_a_refresh_moves_the_execution_forward_again covers it.

(
bts.ContainerExecutionStatus.PENDING,
bts.ContainerExecutionStatus.RUNNING,
)
),
bts.ContainerExecution.last_processed_at
> front_of_queue_last_processed_at,
)
.values(last_processed_at=front_of_queue_last_processed_at)
)
session.commit()

def internal_process_one_queued_execution(
self, session: orm.Session, execution: bts.ExecutionNode
):
Expand Down
Loading
Loading