Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions cloud_pipelines_backend/api_server_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,10 @@

T = typing.TypeVar("T")

_CANCELLED_CONTAINER_EXECUTION_LAST_PROCESSED_AT = datetime.datetime(
1970, 1, 1, tzinfo=datetime.timezone.utc
)


class ApiServiceError(RuntimeError):
pass
Expand Down Expand Up @@ -243,6 +247,10 @@ def terminate(
if execution_node.extra_data is None:
execution_node.extra_data = {}
execution_node.extra_data["desired_state"] = "TERMINATED"
if execution_node.container_execution is not None:
execution_node.container_execution.last_processed_at = (
_CANCELLED_CONTAINER_EXECUTION_LAST_PROCESSED_AT
)
session.commit()

# Note: This method must be last to not shadow the "list" type
Expand Down
26 changes: 8 additions & 18 deletions cloud_pipelines_backend/orchestrator_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ def __init__(
_max_queue_batch_size: int = 1,
_max_queue_batch_duration: datetime.timedelta = datetime.timedelta(),
_running_execution_refresh_interval: datetime.timedelta = datetime.timedelta(
seconds=30
seconds=60
),
queued_execution_interceptor: QueuedExecutionInterceptor | None = None,
):
Expand All @@ -97,7 +97,6 @@ def __init__(
self._max_queue_batch_size = _max_queue_batch_size
self._max_queue_batch_duration = _max_queue_batch_duration
self._running_execution_refresh_interval = _running_execution_refresh_interval
self._running_executions_queue_overdue = False
self._queued_execution_interceptor = queued_execution_interceptor

def run_loop(self):
Expand All @@ -113,16 +112,13 @@ def run_loop(self):

def process_each_queue_once(self) -> bool:
"""Processes each queue once; returns whether the next sweep should start without waiting."""
self._running_executions_queue_overdue = False
queued_queue_has_items = self._process_queue_batch(
self.internal_process_queued_executions_queue
)
running_queue_has_items = self._process_queue_batch(
self.internal_process_running_executions_queue
)
return queued_queue_has_items or (
running_queue_has_items and self._running_executions_queue_overdue
)
return queued_queue_has_items or running_queue_has_items

def _process_queue_batch(
self, queue_handler: typing.Callable[..., typing.Any]
Expand Down Expand Up @@ -157,14 +153,6 @@ def _process_queue_batch(
break
return queue_has_items

def _is_overdue(self, *, last_processed_at: datetime.datetime | None) -> bool:
"""Whether a container execution last refreshed at `last_processed_at` is due for another refresh."""
return (
last_processed_at is None
or last_processed_at
< _get_current_time() - self._running_execution_refresh_interval
)

def internal_process_queued_executions_queue(self, session: orm.Session):
_logger.debug("Selecting queued execution to process.)")

Expand Down Expand Up @@ -237,7 +225,12 @@ def internal_process_running_executions_queue(self, session: orm.Session):
bts.ContainerExecutionStatus.PENDING,
bts.ContainerExecutionStatus.RUNNING,
)
)
),
sql.or_(
bts.ContainerExecution.last_processed_at.is_(None),
bts.ContainerExecution.last_processed_at
< _get_current_time() - self._running_execution_refresh_interval,
),
)
.order_by(bts.ContainerExecution.last_processed_at)
.limit(1)
Expand All @@ -256,9 +249,6 @@ def internal_process_running_executions_queue(self, session: orm.Session):
return False

self._running_executions_queue_idle = False
self._running_executions_queue_overdue = self._is_overdue(
last_processed_at=running_container_execution.last_processed_at
)

execution_node_ids = list(
session.scalars(
Expand Down
2 changes: 2 additions & 0 deletions tests/test_container_execution_refresh_retries.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Tests for the refresh-failure budget in the running-executions queue."""

import datetime
from typing import Any, Callable
from unittest import mock

Expand Down Expand Up @@ -124,6 +125,7 @@ def _make_orchestrator(
data_root_uri="file:///tmp/artifacts",
logs_root_uri="file:///tmp/logs",
_max_container_execution_refresh_error_retries=max_failures,
_running_execution_refresh_interval=datetime.timedelta(0),
)


Expand Down
1 change: 1 addition & 0 deletions tests/test_orchestrator_failed_log_upload.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ def test_failed_execution_survives_log_upload_failure() -> None:
storage_provider=mock.MagicMock(),
data_root_uri="file:///tmp/artifacts",
logs_root_uri="file:///tmp/logs",
_running_execution_refresh_interval=datetime.timedelta(0),
)

# Launch the queued executions: `upstream` becomes PENDING, `downstream` stays
Expand Down
1 change: 1 addition & 0 deletions tests/test_orchestrator_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ def _make_orchestrator(
data_root_uri="file:///tmp/artifacts",
logs_root_uri="file:///tmp/logs",
queued_execution_interceptor=queued_execution_interceptor,
_running_execution_refresh_interval=datetime.timedelta(0),
)


Expand Down
59 changes: 59 additions & 0 deletions tests/test_orchestrator_sweep_pacing.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,3 +157,62 @@ def test_sweeps_again_without_waiting_when_a_batch_raises_after_finding_work():
side_effect=[True, True, RuntimeError("The database was unavailable")],
):
assert orchestrator.process_each_queue_once() is True


def test_skips_a_container_execution_refreshed_within_the_interval():
session_factory = _create_session_factory()
_create_pipeline_run(session_factory)
orchestrator = _make_orchestrator(session_factory)
orchestrator.process_each_queue_once()
_set_last_processed_at(
session_factory,
datetime.datetime.now(tz=datetime.timezone.utc)
- datetime.timedelta(seconds=59),
)

assert (
orchestrator.internal_process_running_executions_queue(
session=session_factory()
)
is False
)
orchestrator._launcher.get_refreshed_launched_container_from_dict.assert_not_called()


def test_refreshes_a_container_execution_once_the_interval_passed():
session_factory = _create_session_factory()
_create_pipeline_run(session_factory)
orchestrator = _make_orchestrator(session_factory)
orchestrator.process_each_queue_once()
_set_last_processed_at(
session_factory,
datetime.datetime.now(tz=datetime.timezone.utc)
- datetime.timedelta(seconds=61),
)

assert (
orchestrator.internal_process_running_executions_queue(
session=session_factory()
)
is True
)
orchestrator._launcher.get_refreshed_launched_container_from_dict.assert_called_once()


def test_cancelling_a_run_refreshes_its_container_execution_without_waiting():
session_factory = _create_session_factory()
_create_pipeline_run(session_factory)
orchestrator = _make_orchestrator(session_factory)
orchestrator.process_each_queue_once()
with session_factory() as session:
pipeline_run_id = session.scalar(sql.select(bts.PipelineRun.id))
api_server_sql.PipelineRunsApiService_Sql().terminate(
session=session, id=pipeline_run_id, skip_user_check=True
)

assert (
orchestrator.internal_process_running_executions_queue(
session=session_factory()
)
is True
)
Loading