From 29cb7706c95aa630af69983019e9e75496397af4 Mon Sep 17 00:00:00 2001 From: ghirish-shopify Date: Tue, 17 Nov 2026 16:00:59 -0500 Subject: [PATCH] Recover native worker attachment after a database disconnect --- cloud_pipelines_backend/orchestrator_sql.py | 162 ++++++++++--- tests/test_orchestrator_sql.py | 241 +++++++++++++++++++- 2 files changed, 374 insertions(+), 29 deletions(-) diff --git a/cloud_pipelines_backend/orchestrator_sql.py b/cloud_pipelines_backend/orchestrator_sql.py index 33f22f7..e346f24 100644 --- a/cloud_pipelines_backend/orchestrator_sql.py +++ b/cloud_pipelines_backend/orchestrator_sql.py @@ -1,5 +1,6 @@ import copy import json +import hashlib import datetime import logging import itertools @@ -786,6 +787,21 @@ def generate_execution_log_uri( container_execution_uuid ) + # Snapshot ORM inputs before the remote launch. A connection can disappear + # during submit; reading expired ORM values afterwards would lose the handle + # before attachment recovery even starts. + execution_id = execution.id + input_data = { + name: dict( + id=data.id, + is_dir=data.is_dir, + total_size=data.total_size, + hash=data.hash, + ) + for name, data in input_artifact_data.items() + } + session.rollback() + try: launched_container: launcher_interfaces.LaunchedContainer = ( self._launcher.launch_container_task( @@ -800,6 +816,9 @@ def generate_execution_log_uri( if launched_container.status not in ( launcher_interfaces.ContainerStatus.PENDING, launcher_interfaces.ContainerStatus.RUNNING, + launcher_interfaces.ContainerStatus.SUCCEEDED, + launcher_interfaces.ContainerStatus.FAILED, + launcher_interfaces.ContainerStatus.ERROR, ): raise OrchestratorError( f"Unexpected status of just launched container: {launched_container.status=}, {launched_container=}" @@ -832,42 +851,127 @@ def generate_execution_log_uri( current_time = _get_current_time() - container_execution = bts.ContainerExecution( - status=bts.ContainerExecutionStatus(launched_container.status), + # Terminal handles still need one ordinary polling pass to validate outputs, + # record artifacts, and unblock (or skip) descendants. + status = ( + bts.ContainerExecutionStatus.SYSTEM_ERROR + if launched_container.status == launcher_interfaces.ContainerStatus.ERROR + else bts.ContainerExecutionStatus(launched_container.status) + ) + if status not in ( + bts.ContainerExecutionStatus.PENDING, + bts.ContainerExecutionStatus.RUNNING, + ): + status = bts.ContainerExecutionStatus.PENDING + launcher_data = launched_container.to_dict() + attachment = dict( + id=container_execution_uuid, + status=status, last_processed_at=current_time, created_at=current_time, - launcher_data=launched_container.to_dict(), - # Updating the map with uploaded value URIs and downloaded values. + launcher_data=launcher_data, + extra_data={ + "initial_launcher_data_sha256": hashlib.sha256( + json.dumps( + launcher_data, sort_keys=True, separators=(",", ":") + ).encode() + ).hexdigest() + }, input_artifact_data_map={ - input_name: dict( - id=artifact_data.id, - is_dir=artifact_data.is_dir, - total_size=artifact_data.total_size, - hash=artifact_data.hash, - # The value and URI might have been updated during launching. - value=input_arguments[input_name].value, - uri=input_arguments[input_name].uri, + name: dict( + **data, + value=input_arguments[name].value, + uri=input_arguments[name].uri, ) - for input_name, artifact_data in input_artifact_data.items() + for name, data in input_data.items() }, output_artifact_data_map={ - output_name: dict( - uri=uri, - ) - for output_name, uri in output_artifact_uris.items() + name: dict(uri=uri) for name, uri in output_artifact_uris.items() }, log_uri=log_uri, ) - container_execution.id = container_execution_uuid - - # Need to commit/rollback before explicitly beginning non-nested transaction. - session.rollback() - with session.begin(): - session.add(container_execution) - execution.container_execution = container_execution - execution.container_execution_cache_key = cache_key - execution.container_execution_status = container_execution.status - # TODO: Maybe add artifact value and URI to input ArtifactData. + self._attach_launched_container( + execution_id=execution_id, cache_key=cache_key, attachment=attachment + ) + session.expire_all() + + def _attach_launched_container( + self, *, execution_id: str, cache_key: str, attachment: dict[str, Any] + ) -> None: + """Retry only DB attachment, never submit, after a lost connection. + + A commit may have succeeded before its response was lost. A fresh session + verifies the exact attachment before treating that case as success. This + bounded retry does not provide process-crash recovery of an unrecorded handle. + """ + for attempt in range(3): + try: + with self._session_factory() as attach_session, attach_session.begin(): + node = attach_session.scalar( + sql.select(bts.ExecutionNode) + .where(bts.ExecutionNode.id == execution_id) + .with_for_update() + ) + if node is None: + raise OrchestratorError( + "Execution disappeared before native worker attachment" + ) + existing = attach_session.get( + bts.ContainerExecution, attachment["id"] + ) + if existing is not None: + same = ( + node.container_execution_id == existing.id + and node.container_execution_cache_key == cache_key + and existing.log_uri == attachment["log_uri"] + and existing.input_artifact_data_map + == attachment["input_artifact_data_map"] + and { + name: {"uri": data.get("uri")} + for name, data in ( + existing.output_artifact_data_map or {} + ).items() + } + == attachment["output_artifact_data_map"] + and (existing.extra_data or {}).get( + "initial_launcher_data_sha256" + ) + == attachment["extra_data"]["initial_launcher_data_sha256"] + ) + if not same: + raise OrchestratorError( + "Native worker attachment conflicts with persisted execution" + ) + return + if ( + node.container_execution_id is not None + or node.container_execution_status + != bts.ContainerExecutionStatus.QUEUED + ): + raise OrchestratorError( + "Execution changed before native worker attachment" + ) + container = bts.ContainerExecution( + **{ + key: value + for key, value in attachment.items() + if key != "id" + } + ) + container.id = attachment["id"] + attach_session.add(container) + node.container_execution = container + node.container_execution_cache_key = cache_key + node.container_execution_status = container.status + return + except sql.exc.DBAPIError as error: + if not error.connection_invalidated or attempt == 2: + raise + _logger.warning( + "Lost DB connection attaching native worker; retrying attachment only (attempt %s)", + attempt + 1, + ) + time.sleep((0.25, 1.0)[attempt]) def internal_process_one_running_execution( self, session: orm.Session, container_execution: bts.ContainerExecution @@ -882,7 +986,9 @@ def internal_process_one_running_execution( launched_container: launcher_interfaces.LaunchedContainer = ( self._launcher.deserialize_launched_container_from_dict(launcher_data) ) - previous_status = launched_container.status + # The database tracks which transitions have been processed. The launcher + # handle may already be terminal when submit reconciliation attaches it. + previous_status = container_execution.status if previous_status not in ( bts.ContainerExecutionStatus.PENDING, bts.ContainerExecutionStatus.RUNNING, diff --git a/tests/test_orchestrator_sql.py b/tests/test_orchestrator_sql.py index 9710468..0895ad7 100644 --- a/tests/test_orchestrator_sql.py +++ b/tests/test_orchestrator_sql.py @@ -5,7 +5,7 @@ from unittest import mock import pytest -from sqlalchemy import orm +from sqlalchemy import orm, event, exc from sqlalchemy import sql from cloud_pipelines_backend import api_server_sql @@ -1077,3 +1077,242 @@ def test_other_launcher_errors_are_still_system_errors(self) -> None: app_metrics.execution_system_errors.add.assert_called_once_with(1) bugsnag.notify.assert_called_once() app_metrics.execution_launch_rejections.add.assert_not_called() + + +class TestNativeWorkerAttachment: + @pytest.mark.parametrize("already_completed", [False, True]) + @pytest.mark.parametrize("failure_point", ["read", "before_commit", "after_commit"]) + def test_disconnect_retries_attachment_without_submitting_again( + self, failure_point, already_completed + ): + session_factory = _create_session_factory() + root_task = _make_graph_task_spec( + tasks={ + "upstream": structures.TaskSpec( + component_ref=structures.ComponentReference( + spec=_make_container_component(outputs=["out"]) + ) + ), + "downstream": structures.TaskSpec( + component_ref=structures.ComponentReference( + spec=_make_container_component(inputs=["in"]) + ), + arguments={"in": _output_argument("upstream", "out")}, + ), + } + ) + _create_pipeline_run(session_factory, root_task) + launch = _make_launched_container_mock() + if already_completed: + launch.return_value.status = launcher_interfaces.ContainerStatus.SUCCEEDED + orchestrator = _make_running_orchestrator(session_factory, launch) + failed = False + attempts = 0 + + def attachment_session(): + nonlocal attempts + attempts += 1 + session = session_factory() + + def disconnect(*args): + nonlocal failed + if not failed: + failed = True + raise exc.OperationalError( + "test", + {}, + OSError("connection lost"), + connection_invalidated=True, + ) + + event_name = "do_orm_execute" if failure_point == "read" else failure_point + event.listen(session, event_name, disconnect) + return session + + orchestrator._session_factory = attachment_session + with mock.patch.object(orchestrator_sql.time, "sleep"): + orchestrator.internal_process_queued_executions_queue( + session=session_factory() + ) + assert failed + assert attempts == 2 + launch.assert_called_once() + with session_factory() as session: + upstream = _get_execution_node(session, "upstream") + assert ( + upstream.container_execution_status + == bts.ContainerExecutionStatus.PENDING + ) + assert upstream.container_execution.launcher_data == {"foo": "bar"} + assert len(session.scalars(sql.select(bts.ContainerExecution)).all()) == 1 + assert ( + _get_execution_node(session, "downstream").container_execution_status + == bts.ContainerExecutionStatus.WAITING_FOR_UPSTREAM + ) + self._complete(orchestrator, session_factory) + with session_factory() as session: + assert ( + _get_execution_node(session, "upstream").container_execution_status + == bts.ContainerExecutionStatus.SUCCEEDED + ) + assert ( + _get_execution_node(session, "downstream").container_execution_status + == bts.ContainerExecutionStatus.QUEUED + ) + assert ( + session.scalar( + sql.select(bts.ArtifactData).where( + bts.ArtifactData.hash == "test=hash" + ) + ) + is not None + ) + + @staticmethod + def _complete(orchestrator, session_factory): + handle = mock.MagicMock( + status=launcher_interfaces.ContainerStatus.SUCCEEDED, + exit_code=0, + started_at=None, + ended_at=datetime.datetime.now(datetime.timezone.utc), + to_dict=lambda: {"foo": "bar"}, + ) + orchestrator._launcher.get_refreshed_launched_container_from_dict.return_value = ( + handle + ) + reader = ( + orchestrator._storage_provider.make_uri.return_value.get_reader.return_value + ) + reader.exists.return_value = True + reader.get_info.return_value = mock.MagicMock( + is_dir=False, total_size=2, hashes={"test": "hash"} + ) + reader.download_as_bytes.return_value = b"ok" + orchestrator.internal_process_running_executions_queue( + session=session_factory() + ) + + @pytest.mark.parametrize( + "status", + [ + launcher_interfaces.ContainerStatus.SUCCEEDED, + launcher_interfaces.ContainerStatus.FAILED, + launcher_interfaces.ContainerStatus.ERROR, + ], + ) + def test_already_terminal_handle_still_runs_normal_completion(self, status): + session_factory = _create_session_factory() + _create_pipeline_run(session_factory, _single_task_pipeline()) + launch = _make_launched_container_mock() + launch.return_value.status = status + launch.return_value.exit_code = ( + 0 if status == launcher_interfaces.ContainerStatus.SUCCEEDED else 1 + ) + launch.return_value.started_at = None + launch.return_value.ended_at = datetime.datetime.now(datetime.timezone.utc) + launch.return_value.launcher_error_message = None + orchestrator = _make_running_orchestrator(session_factory, launch) + orchestrator.internal_process_queued_executions_queue(session=session_factory()) + with session_factory() as session: + assert ( + session.scalar(sql.select(bts.ContainerExecution)).status + == bts.ContainerExecutionStatus.PENDING + ) + orchestrator.internal_process_running_executions_queue( + session=session_factory() + ) + with session_factory() as session: + assert session.scalar(sql.select(bts.ContainerExecution)).status == ( + bts.ContainerExecutionStatus.SYSTEM_ERROR + if status == launcher_interfaces.ContainerStatus.ERROR + else bts.ContainerExecutionStatus(status) + ) + launch.assert_called_once() + + @pytest.mark.parametrize("invalidated, expected_attempts", [(True, 3), (False, 1)]) + def test_attachment_retries_are_bounded_and_only_for_disconnects( + self, invalidated, expected_attempts + ): + session_factory = _create_session_factory() + _create_pipeline_run(session_factory, _single_task_pipeline()) + launch = _make_launched_container_mock() + orchestrator = _make_orchestrator( + session_factory=session_factory, launched_container_mock=launch + ) + factory = mock.MagicMock( + side_effect=exc.OperationalError( + "test", {}, OSError("lost"), connection_invalidated=invalidated + ) + ) + orchestrator._session_factory = factory + with mock.patch.object(orchestrator_sql.time, "sleep"): + orchestrator.internal_process_queued_executions_queue( + session=session_factory() + ) + assert factory.call_count == expected_attempts + launch.assert_called_once() + with session_factory() as session: + assert ( + _get_execution_node(session, "task").container_execution_status + == bts.ContainerExecutionStatus.SYSTEM_ERROR + ) + + def test_cancel_during_submit_survives_attachment(self): + session_factory = _create_session_factory() + _create_pipeline_run(session_factory, _single_task_pipeline()) + launch = _make_launched_container_mock() + + def cancel(**kwargs): + with session_factory() as session: + _request_termination(session=session, task_id="task") + return launch.return_value + + launch.side_effect = cancel + orchestrator = _make_running_orchestrator(session_factory, launch) + orchestrator.internal_process_queued_executions_queue(session=session_factory()) + orchestrator.internal_process_running_executions_queue( + session=session_factory() + ) + launch.return_value.terminate.assert_called_once() + with session_factory() as session: + assert ( + _get_execution_node(session, "task").container_execution_status + == bts.ContainerExecutionStatus.CANCELLED + ) + + def test_attachment_conflict_cannot_replace_an_existing_worker(self): + session_factory = _create_session_factory() + _create_pipeline_run(session_factory, _single_task_pipeline()) + launch = _make_launched_container_mock() + orchestrator = _make_orchestrator( + session_factory=session_factory, launched_container_mock=launch + ) + original = orchestrator._attach_launched_container + saved = {} + + def capture(**kwargs): + saved.update(kwargs) + original(**kwargs) + + with mock.patch.object( + orchestrator, "_attach_launched_container", side_effect=capture + ): + orchestrator.internal_process_queued_executions_queue( + session=session_factory() + ) + original(**saved) # A lost success acknowledgement is idempotent. + conflicting = dict( + saved["attachment"], + extra_data={"initial_launcher_data_sha256": "different-worker"}, + ) + with pytest.raises(orchestrator_sql.OrchestratorError, match="conflicts"): + original(**{**saved, "attachment": conflicting}) + with pytest.raises(orchestrator_sql.OrchestratorError, match="changed"): + original( + **{**saved, "attachment": dict(saved["attachment"], id="different-id")} + ) + with session_factory() as session: + containers = session.scalars(sql.select(bts.ContainerExecution)).all() + assert len(containers) == 1 + assert containers[0].id == saved["attachment"]["id"] + assert containers[0].launcher_data == {"foo": "bar"}