diff --git a/google/cloud/managed_spark_connect/_cell_monitoring.py b/google/cloud/managed_spark_connect/_cell_monitoring.py new file mode 100644 index 00000000..9b061e9c --- /dev/null +++ b/google/cloud/managed_spark_connect/_cell_monitoring.py @@ -0,0 +1,191 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Thin lifecycle bridge to `managed_spark_cell_monitoring.spark_connect`.""" + +import logging +import sys +import threading +from typing import Any, Callable, List, Optional + +logger = logging.getLogger(__name__) + + +def classify_exception_outcome(exc: BaseException) -> str: + """Classifies an `ExecutePlan` exception into `CANCELLED` or `FAILED`.""" + if isinstance(exc, (KeyboardInterrupt, GeneratorExit)): + return "CANCELLED" + + seen: set[int] = set() + cur: Optional[BaseException] = exc + while cur is not None and id(cur) not in seen: + seen.add(id(cur)) + code_fn = getattr(cur, "code", None) + if callable(code_fn): + try: + code = code_fn() + if getattr(code, "name", None) == "CANCELLED": + return "CANCELLED" + except Exception: + pass + grpc_code = getattr(cur, "grpc_status_code", None) + if getattr(grpc_code, "name", None) == "CANCELLED": + return "CANCELLED" + err_cls_fn = getattr(cur, "getErrorClass", None) + if callable(err_cls_fn): + try: + err_cls = err_cls_fn() or "" + if "CANCELED" in err_cls or "CANCELLED" in err_cls: + return "CANCELLED" + except Exception: + pass + msg = str(cur) + if "OPERATION_CANCELED" in msg or "JOB_CANCELLED" in msg: + return "CANCELLED" + cur = cur.__cause__ or cur.__context__ + + return "FAILED" + + +class SessionCellMonitorBridge: + """Lazily delegates operation hooks to `managed-spark-cell-monitoring`. + + When `managed_spark_cell_monitoring.spark_connect` is not loaded in the + current Python process, all methods are fast no-ops with zero imports or + background threads. + """ + + def __init__( + self, + client: Any, + spark_ui_url_getter: Optional[Callable[[], str]] = None, + on_fallback: Optional[Callable[[List[str]], None]] = None, + ) -> None: + self._client = client + self._spark_ui_url_getter = spark_ui_url_getter + self._on_fallback = on_fallback + self._lock = threading.Lock() + self._poller: Any = None + self._closed = False + + @property + def is_supported(self) -> bool: + poller = self._poller + if poller is None: + return True + return bool(getattr(poller, "is_supported", True)) + + def _resolve_poller(self) -> Any: + poller = self._poller + if poller is not None or self._closed: + return poller + mod = sys.modules.get("managed_spark_cell_monitoring.spark_connect") + if mod is None: + return None + factory = getattr(mod, "create_session_poller", None) + if not callable(factory): + return None + with self._lock: + if self._closed: + return None + if self._poller is None: + try: + self._poller = factory( + self._client, + spark_ui_url_getter=self._spark_ui_url_getter, + on_fallback=self._on_fallback, + ) + except Exception: + logger.debug( + "Failed creating Spark Connect cell monitor poller", + exc_info=True, + ) + return None + return self._poller + + def begin_operation(self, operation_id: Optional[str]) -> bool: + if not operation_id or self._closed: + return False + poller = self._resolve_poller() + if poller is None: + return False + try: + return bool(poller.begin_operation(operation_id)) + except Exception: + logger.debug( + "Error in cell monitor begin_operation(%s)", + operation_id, + exc_info=True, + ) + return False + + def finish_operation( + self, + operation_id: Optional[str], + outcome: str = "SUCCEEDED", + ) -> None: + if not operation_id: + return + poller = self._poller + if poller is None: + return + try: + poller.finish_operation(operation_id, outcome=outcome) + except TypeError: + poller.finish_operation(operation_id) + except Exception: + logger.debug( + "Error in cell monitor finish_operation(%s)", + operation_id, + exc_info=True, + ) + + def note_spark_tasks_seen(self, operation_id: Optional[str]) -> None: + poller = self._poller + if poller is None or not operation_id: + return + try: + poller.note_spark_tasks_seen(operation_id) + except Exception: + logger.debug( + "Error recording Spark tasks for %s", + operation_id, + exc_info=True, + ) + + def should_suppress_progress_bar(self, operation_id: Optional[str]) -> bool: + poller = self._poller + if poller is None or not operation_id or self._closed: + return False + try: + return bool(poller.should_suppress_progress_bar(operation_id)) + except Exception: + return False + + def close(self) -> None: + with self._lock: + if self._closed: + return + self._closed = True + poller = self._poller + self._poller = None + self._spark_ui_url_getter = None + self._on_fallback = None + if poller is not None: + try: + poller.close() + except Exception: + logger.debug( + "Error closing Spark Connect cell monitor poller", + exc_info=True, + ) diff --git a/google/cloud/managed_spark_connect/_ipython.py b/google/cloud/managed_spark_connect/_ipython.py index c1539bc9..e42fff75 100644 --- a/google/cloud/managed_spark_connect/_ipython.py +++ b/google/cloud/managed_spark_connect/_ipython.py @@ -38,6 +38,10 @@ "google.cloud.managed_spark_magics", (("line", "dpip"), ("line", "sparksql"), ("cell", "sparksql")), ), + ( + "managed_spark_cell_monitoring.spark_connect", + (("line", "cellmonitor"),), + ), ) @@ -76,7 +80,7 @@ class ManagedSparkConnect(Configurable): enable_extras = Bool( help=( "Whether to automatically load the installed interactive extras " - "(explore_dataframe, %dpip, %%sparksql)." + "(explore_dataframe, %dpip, %%sparksql, %cellmonitor)." ), ).tag(config=True) diff --git a/google/cloud/managed_spark_connect/client/core.py b/google/cloud/managed_spark_connect/client/core.py index ebc8c705..f3762bb9 100644 --- a/google/cloud/managed_spark_connect/client/core.py +++ b/google/cloud/managed_spark_connect/client/core.py @@ -29,7 +29,11 @@ logger = logging.getLogger(__name__) -_UNGUARDED_METHOD_SUFFIXES = ("/ReleaseExecute", "/ReleaseSession") +_UNGUARDED_METHOD_SUFFIXES = ( + "/ReleaseExecute", + "/ReleaseSession", + "/GetStatus", +) _GUARD_MODE: contextvars.ContextVar[str] = contextvars.ContextVar( "msc_guard_mode", default="direct" ) diff --git a/google/cloud/managed_spark_connect/session.py b/google/cloud/managed_spark_connect/session.py index 61f5256d..82ee0838 100644 --- a/google/cloud/managed_spark_connect/session.py +++ b/google/cloud/managed_spark_connect/session.py @@ -29,6 +29,7 @@ import threading import time import uuid +import weakref import tqdm from packaging import version from types import MethodType @@ -101,6 +102,7 @@ TerminateSessionRequest, ) from google.cloud.dataproc_v1.types import sessions +from google.cloud.managed_spark_connect import _cell_monitoring from google.cloud.managed_spark_connect import _logging from google.cloud.managed_spark_connect import _runtimes from google.cloud.managed_spark_connect import environment @@ -2046,6 +2048,17 @@ def __init__( self._show_operation_links: Optional[bool] = show_operation_links self._show_progress_bar: Optional[bool] = show_progress_bar self._progress_bar_leave: Optional[bool] = progress_bar_leave + weak_ui_url = weakref.WeakMethod(self._get_session_spark_ui_url) + weak_fallback = weakref.WeakMethod(self._on_cell_monitor_fallback) + self._cell_monitor_poller = _cell_monitoring.SessionCellMonitorBridge( + self.client, + spark_ui_url_getter=lambda: ( + fn() if (fn := weak_ui_url()) is not None else "" + ), + on_fallback=lambda op_ids: ( + fn(op_ids) if (fn := weak_fallback()) is not None else None + ), + ) # The _execute_and_fetch hooks stack in install order, innermost # first: the position-lost replay (so the hooks around it see one # call, with the caller's request, however many attempts it takes), @@ -2073,17 +2086,50 @@ def _execute_plan_request_with_metadata(*args, **kwargs): def execute_hook(inner): def _execute(req, *args, **kwargs): + monitored = False if not self._sql_lazy_transformation(req): - self._display_operation_link(req.operation_id) - return inner(req, *args, **kwargs) + monitored = self._cell_monitor_poller.begin_operation( + req.operation_id + ) + if not monitored: + self._display_operation_link(req.operation_id) + outcome = "SUCCEEDED" + try: + return inner(req, *args, **kwargs) + except BaseException as exc: + outcome = _cell_monitoring.classify_exception_outcome(exc) + raise + finally: + if monitored: + self._cell_monitor_poller.finish_operation( + req.operation_id, outcome=outcome + ) return _execute def execute_and_fetch_as_iterator_hook(inner): def _execute_and_fetch_as_iterator(req, *args, **kwargs): + monitored = False if not self._sql_lazy_transformation(req): - self._display_operation_link(req.operation_id) - return inner(req, *args, **kwargs) + monitored = self._cell_monitor_poller.begin_operation( + req.operation_id + ) + if not monitored: + self._display_operation_link(req.operation_id) + try: + iterator = inner(req, *args, **kwargs) + except BaseException as exc: + if monitored: + self._cell_monitor_poller.finish_operation( + req.operation_id, + outcome=_cell_monitoring.classify_exception_outcome( + exc + ), + ) + raise + if not monitored: + return iterator + return self._wrap_monitored_iterator(iterator, req.operation_id) return _execute_and_fetch_as_iterator @@ -2165,11 +2211,6 @@ def handler( if operation_id is None: return - # Don't build / render progress bar for non-interactive (despite - # Ipython or non-IPython) or when disabled, unless tests forced the bar on. - if not self._should_show_progress_bar(): - return - total_tasks = 0 completed_tasks = 0 @@ -2177,6 +2218,21 @@ def handler( total_tasks += stage.num_tasks completed_tasks += stage.num_completed_tasks + poller = self.__dict__.get("_cell_monitor_poller") + if poller is not None: + if total_tasks > 0: + poller.note_spark_tasks_seen(operation_id) + if poller.should_suppress_progress_bar(operation_id): + self._close_progress_bar( + operation_id, reason="cell_monitor" + ) + return + + # Don't build / render progress bar for non-interactive (despite + # Ipython or non-IPython) or when disabled, unless tests forced the bar on. + if not self._should_show_progress_bar(): + return + # Don't show progress bar till we receive some tasks if total_tasks == 0: return @@ -2322,6 +2378,36 @@ def _sql_lazy_transformation(req): except AttributeError: return False + def _wrap_monitored_iterator(self, iterator, operation_id: str): + outcome = "CANCELLED" + try: + yield from iterator + outcome = "SUCCEEDED" + except BaseException as exc: + outcome = _cell_monitoring.classify_exception_outcome(exc) + raise + finally: + poller = self.__dict__.get("_cell_monitor_poller") + if poller is not None: + poller.finish_operation(operation_id, outcome=outcome) + + def _get_session_spark_ui_url(self) -> str: + if ( + self._region is not None + and self._active_s8s_session_id is not None + and self._project_id is not None + ): + return ( + f"{_MANAGED_SPARK_SESSIONS_BASE_URL}/{self._region}/" + f"{self._active_s8s_session_id}/sparkApplications/applications" + f"?project={self._project_id}" + ) + return "" + + def _on_cell_monitor_fallback(self, operation_ids: List[str]) -> None: + for op_id in operation_ids: + self._display_operation_link(op_id) + def _repr_html_(self) -> str: if not self._active_s8s_session_id: return """ @@ -2549,6 +2635,9 @@ def _close_client(self) -> None: logger.debug( "Failed to close the StreamingQueryManager", exc_info=True ) + poller = self.__dict__.get("_cell_monitor_poller") + if poller is not None: + poller.close() try: client.close() except Exception: diff --git a/tests/unit/test_cell_monitoring.py b/tests/unit/test_cell_monitoring.py new file mode 100644 index 00000000..3c7df532 --- /dev/null +++ b/tests/unit/test_cell_monitoring.py @@ -0,0 +1,341 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for thin Spark Connect cell-monitoring lifecycle integration.""" + +import gc +import io +import sys +import types +from typing import Any, List +import unittest +from unittest import mock +import weakref + +import grpc +from pyspark.sql.connect.shell.progress import StageInfo + +from google.cloud.managed_spark_connect import _cell_monitoring +from google.cloud.managed_spark_connect.client.core import ( + _UNGUARDED_METHOD_SUFFIXES, +) +from google.cloud.managed_spark_connect.session import ManagedSparkSession + + +class _FakeRpcError(grpc.RpcError): + + def __init__(self, status_code: grpc.StatusCode, details: str = ""): + super().__init__(details) + self._code = status_code + self._details = details + + def code(self) -> grpc.StatusCode: + return self._code + + def details(self) -> str: + return self._details + + +class _FakePoller: + + def __init__( + self, + client: Any, + spark_ui_url_getter=None, + on_fallback=None, + **_kwargs, + ): + self.client = client + self.spark_ui_url_getter = spark_ui_url_getter + self.on_fallback = on_fallback + self.active_ops: set[str] = set() + self.begun: List[str] = [] + self.finished: List[tuple[str, str]] = [] + self.spark_tasks_seen: List[str] = [] + self.closed = False + self.is_supported = True + + def begin_operation(self, operation_id: str) -> bool: + if not self.is_supported or self.closed: + return False + self.begun.append(operation_id) + self.active_ops.add(operation_id) + return True + + def finish_operation( + self, operation_id: str, outcome: str = "SUCCEEDED" + ) -> None: + self.active_ops.discard(operation_id) + self.finished.append((operation_id, outcome)) + + def note_spark_tasks_seen(self, operation_id: str) -> None: + self.spark_tasks_seen.append(operation_id) + + def should_suppress_progress_bar(self, operation_id: str) -> bool: + return operation_id in self.active_ops and self.is_supported + + def close(self) -> None: + self.closed = True + self.active_ops.clear() + self.spark_ui_url_getter = None + self.on_fallback = None + + +class TestCellMonitoringBridgeUnit(unittest.TestCase): + + def test_get_status_is_unguarded_method_suffix(self): + self.assertTrue( + "/spark.connect.SparkConnectService/GetStatus".endswith( + _UNGUARDED_METHOD_SUFFIXES + ) + ) + + def test_classify_exception_outcome(self): + self.assertEqual( + _cell_monitoring.classify_exception_outcome(KeyboardInterrupt()), + "CANCELLED", + ) + self.assertEqual( + _cell_monitoring.classify_exception_outcome(GeneratorExit()), + "CANCELLED", + ) + self.assertEqual( + _cell_monitoring.classify_exception_outcome( + _FakeRpcError(grpc.StatusCode.CANCELLED, "aborted") + ), + "CANCELLED", + ) + + wrapped = RuntimeError("SparkConnectGrpcException(OPERATION_CANCELED)") + self.assertEqual( + _cell_monitoring.classify_exception_outcome(wrapped), + "CANCELLED", + ) + self.assertEqual( + _cell_monitoring.classify_exception_outcome( + ValueError("Division by zero") + ), + "FAILED", + ) + + def test_noop_when_cell_monitoring_extension_not_loaded(self): + with mock.patch.dict( + sys.modules, {"managed_spark_cell_monitoring.spark_connect": None} + ): + bridge = _cell_monitoring.SessionCellMonitorBridge(mock.MagicMock()) + self.assertFalse(bridge.begin_operation("op-1")) + self.assertFalse(bridge.should_suppress_progress_bar("op-1")) + bridge.note_spark_tasks_seen("op-1") + bridge.finish_operation("op-1", outcome="SUCCEEDED") + bridge.close() + + +class TestSessionAndExtrasIntegration(unittest.TestCase): + + def _create_local_session(self) -> ManagedSparkSession: + with mock.patch( + "pyspark.sql.connect.session.SparkSession._start_connect_server" + ): + session = ManagedSparkSession("sc://localhost:15002") + session._register_progress_execution_handler() + return session + + def test_session_hooks_outcomes_progress_suppression_fallback_and_gc(self): + created_pollers: List[_FakePoller] = [] + fake_mod = types.ModuleType( + "managed_spark_cell_monitoring.spark_connect" + ) + + def _factory(client, **kwargs): + p = _FakePoller(client, **kwargs) + created_pollers.append(p) + return p + + fake_mod.create_session_poller = _factory # type: ignore[attr-defined] + + with mock.patch.dict( + sys.modules, + {"managed_spark_cell_monitoring.spark_connect": fake_mod}, + ): + ManagedSparkSession._project_id = "proj-1" + ManagedSparkSession._region = "us-central1" + ManagedSparkSession._active_s8s_session_id = "sc-sess-1" + + bars_opened: List[Any] = [] + ManagedSparkSession._set_tqdm_factory( + lambda **kw: bars_opened.append(kw) + or mock.MagicMock(n=0, total=4) + ) + self.addCleanup(ManagedSparkSession._set_tqdm_factory, None) + + active_sess: List[ManagedSparkSession] = [] + yielded_during_active: List[bool] = [] + stream_mode = "ok" + + def _fake_stream(r, *a, **kw): + s = active_sess[0] + for h in s.client._progress_handlers: + h([StageInfo(1, 4, 2, 0, False)], 2, r.operation_id, False) + yielded_during_active.append( + s._cell_monitor_poller.should_suppress_progress_bar( + r.operation_id + ) + ) + yield "batch-1" + if stream_mode == "fail": + raise RuntimeError("task failed") + if stream_mode == "cancel": + raise KeyboardInterrupt() + yield "batch-2" + + with mock.patch.dict( + "os.environ", + {"MANAGED_SPARK_CONNECT_FORCE_PROGRESS_BAR": "1"}, + ): + with mock.patch( + "pyspark.sql.connect.client.core.SparkConnectClient._execute_and_fetch_as_iterator", + side_effect=_fake_stream, + ): + s2 = self._create_local_session() + active_sess.append(s2) + req1 = s2.client._execute_plan_request_with_metadata() + + with mock.patch.object( + s2, "_display_operation_link" + ) as mock_link: + # 1. Normal exhaustion -> outcome="SUCCEEDED" + it = s2.client._execute_and_fetch_as_iterator(req1, {}) + self.assertEqual(next(it), "batch-1") + self.assertEqual(yielded_during_active, [True]) + self.assertEqual(len(created_pollers), 1) + poller = created_pollers[0] + self.assertEqual(poller.finished, []) + self.assertEqual( + poller.spark_ui_url_getter(), + "https://console.cloud.google.com/dataproc/interactive/us-central1/sc-sess-1/sparkApplications/applications?project=proj-1", + ) + self.assertEqual(next(it), "batch-2") + with self.assertRaises(StopIteration): + next(it) + mock_link.assert_not_called() + self.assertEqual( + poller.finished, [(req1.operation_id, "SUCCEEDED")] + ) + self.assertEqual( + poller.spark_tasks_seen, [req1.operation_id] + ) + + # 2. Mid-stream exception -> outcome="FAILED" + stream_mode = "fail" + req2 = s2.client._execute_plan_request_with_metadata() + it2 = s2.client._execute_and_fetch_as_iterator(req2, {}) + self.assertEqual(next(it2), "batch-1") + with self.assertRaises(RuntimeError): + next(it2) + self.assertEqual( + poller.finished[-1], (req2.operation_id, "FAILED") + ) + + # 3. KeyboardInterrupt -> outcome="CANCELLED" + stream_mode = "cancel" + req3 = s2.client._execute_plan_request_with_metadata() + it3 = s2.client._execute_and_fetch_as_iterator(req3, {}) + self.assertEqual(next(it3), "batch-1") + with self.assertRaises(KeyboardInterrupt): + next(it3) + self.assertEqual( + poller.finished[-1], + (req3.operation_id, "CANCELLED"), + ) + + # 4. Fallback callback displays suppressed operation link + poller.on_fallback(["op-fallback"]) + mock_link.assert_called_once_with("op-fallback") + + self.assertEqual(bars_opened, []) + + # Verify _close_client stops poller and leaves zero refcycles pinning session + active_sess.clear() + del it, it2, it3 + ManagedSparkSession._forget_server_session(s2) + weak_s2 = weakref.ref(s2) + s2._close_client() + self.assertTrue(poller.closed) + del s2 + gc.collect() + self.assertIsNone(weak_s2()) + + def test_ipython_extras_loads_and_unloads_cellmonitor_extension(self): + from IPython.core.interactiveshell import InteractiveShell + from google.cloud.managed_spark_connect import _ipython + + fake_ext = types.ModuleType( + "managed_spark_cell_monitoring.spark_connect" + ) + unloaded: List[Any] = [] + + def _load_ext(ip): + def cellmonitor(line=""): + return line + + cellmonitor.__module__ = ( + "managed_spark_cell_monitoring.spark_connect" + ) + ip.register_magic_function( + cellmonitor, magic_kind="line", magic_name="cellmonitor" + ) + + def _unload_ext(ip): + unloaded.append(ip) + + fake_ext.load_ipython_extension = _load_ext # type: ignore[attr-defined] + fake_ext.unload_ipython_extension = _unload_ext # type: ignore[attr-defined] + + ip = InteractiveShell.instance() + self.addCleanup(InteractiveShell.clear_instance) + self.addCleanup(_ipython._SHELL_STATES.clear) + _ipython._SHELL_STATES.clear() + ip.magics_manager.magics.get("line", {}).pop("cellmonitor", None) + + spec = types.SimpleNamespace( + name=fake_ext.__name__, _initializing=False + ) + fake_ext.__spec__ = spec # type: ignore[attr-defined] + + stdout_buf = io.StringIO() + with ( + mock.patch.dict( + sys.modules, + { + "managed_spark_cell_monitoring": types.ModuleType( + "managed_spark_cell_monitoring" + ), + "managed_spark_cell_monitoring.spark_connect": fake_ext, + "google.colabsqlviz": None, + "google.cloud.managed_spark_magics": None, + }, + ), + mock.patch("IPython.get_ipython", return_value=ip), + mock.patch("sys.stdout", stdout_buf), + ): + _ipython._init_extras() + self.assertIn("%cellmonitor", stdout_buf.getvalue()) + self.assertIn("cellmonitor", ip.magics_manager.magics["line"]) + + _ipython._get_or_create_config(ip).enable_extras = False + self.assertEqual(unloaded, [ip]) + self.assertNotIn("cellmonitor", ip.magics_manager.magics["line"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_init.py b/tests/unit/test_init.py index 78be3cba..edaa5de9 100644 --- a/tests/unit/test_init.py +++ b/tests/unit/test_init.py @@ -305,6 +305,15 @@ def setUp(self): env_patcher = mock.patch.dict(os.environ) env_patcher.start() self.addCleanup(env_patcher.stop) + # Isolate optional cell-monitoring extension so unit assertions on + # managed_spark_magics remain deterministic whether or not + # managed-spark-cell-monitoring is also installed in the venv. + mod_patcher = mock.patch.dict( + sys.modules, + {"managed_spark_cell_monitoring.spark_connect": None}, + ) + mod_patcher.start() + self.addCleanup(mod_patcher.stop) os.environ.pop(_ipython.ENV_ENABLE_EXTRAS, None) _ipython._SHELL_STATES.clear() self.addCleanup(_ipython._SHELL_STATES.clear)