Repository navigation
feat: integrate ManagedSparkSession with Spark Connect cell monitoring #237
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,191 @@ | ||
| # Copyright 2026 Google LLC | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| """Thin lifecycle bridge to `managed_spark_cell_monitoring.spark_connect`.""" | ||
|
|
||
| import logging | ||
| import sys | ||
| import threading | ||
| from typing import Any, Callable, List, Optional | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
|
|
||
|
|
||
| def classify_exception_outcome(exc: BaseException) -> str: | ||
| """Classifies an `ExecutePlan` exception into `CANCELLED` or `FAILED`.""" | ||
| if isinstance(exc, (KeyboardInterrupt, GeneratorExit)): | ||
| return "CANCELLED" | ||
|
|
||
| seen: set[int] = set() | ||
| cur: Optional[BaseException] = exc | ||
| while cur is not None and id(cur) not in seen: | ||
| seen.add(id(cur)) | ||
| code_fn = getattr(cur, "code", None) | ||
| if callable(code_fn): | ||
| try: | ||
| code = code_fn() | ||
| if getattr(code, "name", None) == "CANCELLED": | ||
| return "CANCELLED" | ||
| except Exception: | ||
| pass | ||
| grpc_code = getattr(cur, "grpc_status_code", None) | ||
| if getattr(grpc_code, "name", None) == "CANCELLED": | ||
| return "CANCELLED" | ||
| err_cls_fn = getattr(cur, "getErrorClass", None) | ||
| if callable(err_cls_fn): | ||
| try: | ||
| err_cls = err_cls_fn() or "" | ||
| if "CANCELED" in err_cls or "CANCELLED" in err_cls: | ||
| return "CANCELLED" | ||
| except Exception: | ||
| pass | ||
| msg = str(cur) | ||
| if "OPERATION_CANCELED" in msg or "JOB_CANCELLED" in msg: | ||
| return "CANCELLED" | ||
| cur = cur.__cause__ or cur.__context__ | ||
|
|
||
| return "FAILED" | ||
|
|
||
|
|
||
| class SessionCellMonitorBridge: | ||
| """Lazily delegates operation hooks to `managed-spark-cell-monitoring`. | ||
|
|
||
| When `managed_spark_cell_monitoring.spark_connect` is not loaded in the | ||
| current Python process, all methods are fast no-ops with zero imports or | ||
| background threads. | ||
| """ | ||
|
|
||
| def __init__( | ||
| self, | ||
| client: Any, | ||
| spark_ui_url_getter: Optional[Callable[[], str]] = None, | ||
| on_fallback: Optional[Callable[[List[str]], None]] = None, | ||
| ) -> None: | ||
| self._client = client | ||
| self._spark_ui_url_getter = spark_ui_url_getter | ||
| self._on_fallback = on_fallback | ||
| self._lock = threading.Lock() | ||
| self._poller: Any = None | ||
| self._closed = False | ||
|
|
||
| @property | ||
| def is_supported(self) -> bool: | ||
| poller = self._poller | ||
| if poller is None: | ||
| return True | ||
| return bool(getattr(poller, "is_supported", True)) | ||
|
|
||
| def _resolve_poller(self) -> Any: | ||
| poller = self._poller | ||
| if poller is not None or self._closed: | ||
| return poller | ||
| mod = sys.modules.get("managed_spark_cell_monitoring.spark_connect") | ||
| if mod is None: | ||
| return None | ||
| factory = getattr(mod, "create_session_poller", None) | ||
| if not callable(factory): | ||
| return None | ||
| with self._lock: | ||
| if self._closed: | ||
| return None | ||
| if self._poller is None: | ||
| try: | ||
| self._poller = factory( | ||
| self._client, | ||
| spark_ui_url_getter=self._spark_ui_url_getter, | ||
| on_fallback=self._on_fallback, | ||
| ) | ||
| except Exception: | ||
| logger.debug( | ||
| "Failed creating Spark Connect cell monitor poller", | ||
| exc_info=True, | ||
| ) | ||
| return None | ||
| return self._poller | ||
|
|
||
| def begin_operation(self, operation_id: Optional[str]) -> bool: | ||
| if not operation_id or self._closed: | ||
| return False | ||
| poller = self._resolve_poller() | ||
| if poller is None: | ||
| return False | ||
| try: | ||
| return bool(poller.begin_operation(operation_id)) | ||
| except Exception: | ||
| logger.debug( | ||
| "Error in cell monitor begin_operation(%s)", | ||
| operation_id, | ||
| exc_info=True, | ||
| ) | ||
| return False | ||
|
|
||
| def finish_operation( | ||
| self, | ||
| operation_id: Optional[str], | ||
| outcome: str = "SUCCEEDED", | ||
| ) -> None: | ||
| if not operation_id: | ||
| return | ||
| poller = self._poller | ||
| if poller is None: | ||
| return | ||
| try: | ||
| poller.finish_operation(operation_id, outcome=outcome) | ||
| except TypeError: | ||
| poller.finish_operation(operation_id) | ||
| except Exception: | ||
| logger.debug( | ||
| "Error in cell monitor finish_operation(%s)", | ||
| operation_id, | ||
| exc_info=True, | ||
| ) | ||
|
|
||
| def note_spark_tasks_seen(self, operation_id: Optional[str]) -> None: | ||
| poller = self._poller | ||
| if poller is None or not operation_id: | ||
| return | ||
| try: | ||
| poller.note_spark_tasks_seen(operation_id) | ||
| except Exception: | ||
| logger.debug( | ||
| "Error recording Spark tasks for %s", | ||
| operation_id, | ||
| exc_info=True, | ||
| ) | ||
|
|
||
| def should_suppress_progress_bar(self, operation_id: Optional[str]) -> bool: | ||
| poller = self._poller | ||
| if poller is None or not operation_id or self._closed: | ||
| return False | ||
| try: | ||
| return bool(poller.should_suppress_progress_bar(operation_id)) | ||
| except Exception: | ||
| return False | ||
|
|
||
| def close(self) -> None: | ||
| with self._lock: | ||
| if self._closed: | ||
| return | ||
| self._closed = True | ||
| poller = self._poller | ||
| self._poller = None | ||
| self._spark_ui_url_getter = None | ||
| self._on_fallback = None | ||
| if poller is not None: | ||
| try: | ||
| poller.close() | ||
| except Exception: | ||
| logger.debug( | ||
| "Error closing Spark Connect cell monitor poller", | ||
| exc_info=True, | ||
| ) | ||
| Original file line number | Diff line number | Diff line change | ||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -29,6 +29,7 @@ | |||||||||||||
| import threading | ||||||||||||||
| import time | ||||||||||||||
| import uuid | ||||||||||||||
| import weakref | ||||||||||||||
| import tqdm | ||||||||||||||
| from packaging import version | ||||||||||||||
| from types import MethodType | ||||||||||||||
|
|
@@ -101,6 +102,7 @@ | |||||||||||||
| TerminateSessionRequest, | ||||||||||||||
| ) | ||||||||||||||
| from google.cloud.dataproc_v1.types import sessions | ||||||||||||||
| from google.cloud.managed_spark_connect import _cell_monitoring | ||||||||||||||
| from google.cloud.managed_spark_connect import _logging | ||||||||||||||
| from google.cloud.managed_spark_connect import _runtimes | ||||||||||||||
| from google.cloud.managed_spark_connect import environment | ||||||||||||||
|
|
@@ -2046,6 +2048,17 @@ def __init__( | |||||||||||||
| self._show_operation_links: Optional[bool] = show_operation_links | ||||||||||||||
| self._show_progress_bar: Optional[bool] = show_progress_bar | ||||||||||||||
| self._progress_bar_leave: Optional[bool] = progress_bar_leave | ||||||||||||||
| weak_ui_url = weakref.WeakMethod(self._get_session_spark_ui_url) | ||||||||||||||
| weak_fallback = weakref.WeakMethod(self._on_cell_monitor_fallback) | ||||||||||||||
| self._cell_monitor_poller = _cell_monitoring.SessionCellMonitorBridge( | ||||||||||||||
| self.client, | ||||||||||||||
| spark_ui_url_getter=lambda: ( | ||||||||||||||
| fn() if (fn := weak_ui_url()) is not None else "" | ||||||||||||||
| ), | ||||||||||||||
| on_fallback=lambda op_ids: ( | ||||||||||||||
| fn(op_ids) if (fn := weak_fallback()) is not None else None | ||||||||||||||
| ), | ||||||||||||||
| ) | ||||||||||||||
| # The _execute_and_fetch hooks stack in install order, innermost | ||||||||||||||
| # first: the position-lost replay (so the hooks around it see one | ||||||||||||||
| # call, with the caller's request, however many attempts it takes), | ||||||||||||||
|
|
@@ -2073,17 +2086,50 @@ def _execute_plan_request_with_metadata(*args, **kwargs): | |||||||||||||
|
|
||||||||||||||
| def execute_hook(inner): | ||||||||||||||
| def _execute(req, *args, **kwargs): | ||||||||||||||
| monitored = False | ||||||||||||||
| if not self._sql_lazy_transformation(req): | ||||||||||||||
| self._display_operation_link(req.operation_id) | ||||||||||||||
| return inner(req, *args, **kwargs) | ||||||||||||||
| monitored = self._cell_monitor_poller.begin_operation( | ||||||||||||||
| req.operation_id | ||||||||||||||
| ) | ||||||||||||||
| if not monitored: | ||||||||||||||
| self._display_operation_link(req.operation_id) | ||||||||||||||
| outcome = "SUCCEEDED" | ||||||||||||||
| try: | ||||||||||||||
| return inner(req, *args, **kwargs) | ||||||||||||||
| except BaseException as exc: | ||||||||||||||
| outcome = _cell_monitoring.classify_exception_outcome(exc) | ||||||||||||||
| raise | ||||||||||||||
| finally: | ||||||||||||||
| if monitored: | ||||||||||||||
| self._cell_monitor_poller.finish_operation( | ||||||||||||||
| req.operation_id, outcome=outcome | ||||||||||||||
| ) | ||||||||||||||
|
|
||||||||||||||
| return _execute | ||||||||||||||
|
Comment on lines
2087
to
2108
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. In def execute_hook(inner):
def _execute(req, *args, **kwargs):
monitored = False
if not self._sql_lazy_transformation(req):
monitored = self._cell_monitor_poller.begin_operation(
req.operation_id
)
if not monitored:
self._display_operation_link(req.operation_id)
outcome = "SUCCEEDED"
try:
return inner(req, *args, **kwargs)
except BaseException as exc:
if monitored:
outcome = _cell_monitoring.classify_exception_outcome(exc)
raise
finally:
if monitored:
self._cell_monitor_poller.finish_operation(
req.operation_id, outcome=outcome
)
return _execute |
||||||||||||||
|
|
||||||||||||||
| def execute_and_fetch_as_iterator_hook(inner): | ||||||||||||||
| def _execute_and_fetch_as_iterator(req, *args, **kwargs): | ||||||||||||||
| monitored = False | ||||||||||||||
| if not self._sql_lazy_transformation(req): | ||||||||||||||
| self._display_operation_link(req.operation_id) | ||||||||||||||
| return inner(req, *args, **kwargs) | ||||||||||||||
| monitored = self._cell_monitor_poller.begin_operation( | ||||||||||||||
| req.operation_id | ||||||||||||||
| ) | ||||||||||||||
| if not monitored: | ||||||||||||||
| self._display_operation_link(req.operation_id) | ||||||||||||||
| try: | ||||||||||||||
| iterator = inner(req, *args, **kwargs) | ||||||||||||||
| except BaseException as exc: | ||||||||||||||
| if monitored: | ||||||||||||||
| self._cell_monitor_poller.finish_operation( | ||||||||||||||
| req.operation_id, | ||||||||||||||
| outcome=_cell_monitoring.classify_exception_outcome( | ||||||||||||||
| exc | ||||||||||||||
| ), | ||||||||||||||
| ) | ||||||||||||||
| raise | ||||||||||||||
| if not monitored: | ||||||||||||||
| return iterator | ||||||||||||||
| return self._wrap_monitored_iterator(iterator, req.operation_id) | ||||||||||||||
|
|
||||||||||||||
| return _execute_and_fetch_as_iterator | ||||||||||||||
|
|
||||||||||||||
|
|
@@ -2165,18 +2211,28 @@ def handler( | |||||||||||||
| if operation_id is None: | ||||||||||||||
| return | ||||||||||||||
|
|
||||||||||||||
| # Don't build / render progress bar for non-interactive (despite | ||||||||||||||
| # Ipython or non-IPython) or when disabled, unless tests forced the bar on. | ||||||||||||||
| if not self._should_show_progress_bar(): | ||||||||||||||
| return | ||||||||||||||
|
|
||||||||||||||
| total_tasks = 0 | ||||||||||||||
| completed_tasks = 0 | ||||||||||||||
|
|
||||||||||||||
| for stage in stages or []: | ||||||||||||||
| total_tasks += stage.num_tasks | ||||||||||||||
| completed_tasks += stage.num_completed_tasks | ||||||||||||||
|
|
||||||||||||||
| poller = self.__dict__.get("_cell_monitor_poller") | ||||||||||||||
| if poller is not None: | ||||||||||||||
| if total_tasks > 0: | ||||||||||||||
| poller.note_spark_tasks_seen(operation_id) | ||||||||||||||
| if poller.should_suppress_progress_bar(operation_id): | ||||||||||||||
| self._close_progress_bar( | ||||||||||||||
| operation_id, reason="cell_monitor" | ||||||||||||||
| ) | ||||||||||||||
| return | ||||||||||||||
|
|
||||||||||||||
| # Don't build / render progress bar for non-interactive (despite | ||||||||||||||
| # Ipython or non-IPython) or when disabled, unless tests forced the bar on. | ||||||||||||||
| if not self._should_show_progress_bar(): | ||||||||||||||
| return | ||||||||||||||
|
|
||||||||||||||
| # Don't show progress bar till we receive some tasks | ||||||||||||||
| if total_tasks == 0: | ||||||||||||||
| return | ||||||||||||||
|
|
@@ -2322,6 +2378,36 @@ def _sql_lazy_transformation(req): | |||||||||||||
| except AttributeError: | ||||||||||||||
| return False | ||||||||||||||
|
|
||||||||||||||
| def _wrap_monitored_iterator(self, iterator, operation_id: str): | ||||||||||||||
| outcome = "CANCELLED" | ||||||||||||||
| try: | ||||||||||||||
| yield from iterator | ||||||||||||||
| outcome = "SUCCEEDED" | ||||||||||||||
| except BaseException as exc: | ||||||||||||||
| outcome = _cell_monitoring.classify_exception_outcome(exc) | ||||||||||||||
| raise | ||||||||||||||
| finally: | ||||||||||||||
| poller = self.__dict__.get("_cell_monitor_poller") | ||||||||||||||
| if poller is not None: | ||||||||||||||
| poller.finish_operation(operation_id, outcome=outcome) | ||||||||||||||
|
|
||||||||||||||
| def _get_session_spark_ui_url(self) -> str: | ||||||||||||||
| if ( | ||||||||||||||
| self._region is not None | ||||||||||||||
| and self._active_s8s_session_id is not None | ||||||||||||||
| and self._project_id is not None | ||||||||||||||
| ): | ||||||||||||||
| return ( | ||||||||||||||
| f"{_MANAGED_SPARK_SESSIONS_BASE_URL}/{self._region}/" | ||||||||||||||
| f"{self._active_s8s_session_id}/sparkApplications/applications" | ||||||||||||||
| f"?project={self._project_id}" | ||||||||||||||
| ) | ||||||||||||||
| return "" | ||||||||||||||
|
|
||||||||||||||
| def _on_cell_monitor_fallback(self, operation_ids: List[str]) -> None: | ||||||||||||||
| for op_id in operation_ids: | ||||||||||||||
| self._display_operation_link(op_id) | ||||||||||||||
|
|
||||||||||||||
| def _repr_html_(self) -> str: | ||||||||||||||
| if not self._active_s8s_session_id: | ||||||||||||||
| return """ | ||||||||||||||
|
|
@@ -2549,6 +2635,9 @@ def _close_client(self) -> None: | |||||||||||||
| logger.debug( | ||||||||||||||
| "Failed to close the StreamingQueryManager", exc_info=True | ||||||||||||||
| ) | ||||||||||||||
| poller = self.__dict__.get("_cell_monitor_poller") | ||||||||||||||
| if poller is not None: | ||||||||||||||
| poller.close() | ||||||||||||||
|
Comment on lines
+2638
to
+2640
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. While closing the poller in
Suggested change
|
||||||||||||||
| try: | ||||||||||||||
| client.close() | ||||||||||||||
| except Exception: | ||||||||||||||
|
|
||||||||||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The exception classification logic in
classify_exception_outcomeaccesses various attributes and methods (likecode(),grpc_status_code,getErrorClass(), andstr(cur)) on the exception chain. If any of these custom exceptions have buggy implementations or descriptors that raise exceptions,classify_exception_outcomewill raise an unhandled exception. This will propagate and mask the original exception, which is a critical robustness issue. Wrapping the entire classification logic in atry...except BaseException:block ensures that the original exception is never masked and"FAILED"is safely returned as a fallback.