Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
191 changes: 191 additions & 0 deletions google/cloud/managed_spark_connect/_cell_monitoring.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Thin lifecycle bridge to `managed_spark_cell_monitoring.spark_connect`."""

import logging
import sys
import threading
from typing import Any, Callable, List, Optional

logger = logging.getLogger(__name__)


def classify_exception_outcome(exc: BaseException) -> str:
"""Classifies an `ExecutePlan` exception into `CANCELLED` or `FAILED`."""
if isinstance(exc, (KeyboardInterrupt, GeneratorExit)):
return "CANCELLED"

seen: set[int] = set()
cur: Optional[BaseException] = exc
while cur is not None and id(cur) not in seen:
seen.add(id(cur))
code_fn = getattr(cur, "code", None)
if callable(code_fn):
try:
code = code_fn()
if getattr(code, "name", None) == "CANCELLED":
return "CANCELLED"
except Exception:
pass
grpc_code = getattr(cur, "grpc_status_code", None)
if getattr(grpc_code, "name", None) == "CANCELLED":
return "CANCELLED"
err_cls_fn = getattr(cur, "getErrorClass", None)
if callable(err_cls_fn):
try:
err_cls = err_cls_fn() or ""
if "CANCELED" in err_cls or "CANCELLED" in err_cls:
return "CANCELLED"
except Exception:
pass
msg = str(cur)
if "OPERATION_CANCELED" in msg or "JOB_CANCELLED" in msg:
return "CANCELLED"
cur = cur.__cause__ or cur.__context__

return "FAILED"
Comment on lines +24 to +57

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

The exception classification logic in classify_exception_outcome accesses various attributes and methods (like code(), grpc_status_code, getErrorClass(), and str(cur)) on the exception chain. If any of these custom exceptions have buggy implementations or descriptors that raise exceptions, classify_exception_outcome will raise an unhandled exception. This will propagate and mask the original exception, which is a critical robustness issue. Wrapping the entire classification logic in a try...except BaseException: block ensures that the original exception is never masked and "FAILED" is safely returned as a fallback.

def classify_exception_outcome(exc: BaseException) -> str:
    """Classifies an `ExecutePlan` exception into `CANCELLED` or `FAILED`."""
    try:
        if isinstance(exc, (KeyboardInterrupt, GeneratorExit)):
            return "CANCELLED"

        seen: set[int] = set()
        cur: Optional[BaseException] = exc
        while cur is not None and id(cur) not in seen:
            seen.add(id(cur))
            code_fn = getattr(cur, "code", None)
            if callable(code_fn):
                try:
                    code = code_fn()
                    if getattr(code, "name", None) == "CANCELLED":
                        return "CANCELLED"
                except Exception:
                    pass
            try:
                grpc_code = getattr(cur, "grpc_status_code", None)
                if getattr(grpc_code, "name", None) == "CANCELLED":
                    return "CANCELLED"
            except Exception:
                pass
            err_cls_fn = getattr(cur, "getErrorClass", None)
            if callable(err_cls_fn):
                try:
                    err_cls = err_cls_fn() or ""
                    if "CANCELED" in err_cls or "CANCELLED" in err_cls:
                        return "CANCELLED"
                except Exception:
                    pass
            try:
                msg = str(cur)
                if "OPERATION_CANCELED" in msg or "JOB_CANCELLED" in msg:
                    return "CANCELLED"
            except Exception:
                pass
            try:
                cur = cur.__cause__ or cur.__context__
            except Exception:
                break
    except BaseException:
        pass

    return "FAILED"



class SessionCellMonitorBridge:
"""Lazily delegates operation hooks to `managed-spark-cell-monitoring`.

When `managed_spark_cell_monitoring.spark_connect` is not loaded in the
current Python process, all methods are fast no-ops with zero imports or
background threads.
"""

def __init__(
self,
client: Any,
spark_ui_url_getter: Optional[Callable[[], str]] = None,
on_fallback: Optional[Callable[[List[str]], None]] = None,
) -> None:
self._client = client
self._spark_ui_url_getter = spark_ui_url_getter
self._on_fallback = on_fallback
self._lock = threading.Lock()
self._poller: Any = None
self._closed = False

@property
def is_supported(self) -> bool:
poller = self._poller
if poller is None:
return True
return bool(getattr(poller, "is_supported", True))

def _resolve_poller(self) -> Any:
poller = self._poller
if poller is not None or self._closed:
return poller
mod = sys.modules.get("managed_spark_cell_monitoring.spark_connect")
if mod is None:
return None
factory = getattr(mod, "create_session_poller", None)
if not callable(factory):
return None
with self._lock:
if self._closed:
return None
if self._poller is None:
try:
self._poller = factory(
self._client,
spark_ui_url_getter=self._spark_ui_url_getter,
on_fallback=self._on_fallback,
)
except Exception:
logger.debug(
"Failed creating Spark Connect cell monitor poller",
exc_info=True,
)
return None
return self._poller

def begin_operation(self, operation_id: Optional[str]) -> bool:
if not operation_id or self._closed:
return False
poller = self._resolve_poller()
if poller is None:
return False
try:
return bool(poller.begin_operation(operation_id))
except Exception:
logger.debug(
"Error in cell monitor begin_operation(%s)",
operation_id,
exc_info=True,
)
return False

def finish_operation(
self,
operation_id: Optional[str],
outcome: str = "SUCCEEDED",
) -> None:
if not operation_id:
return
poller = self._poller
if poller is None:
return
try:
poller.finish_operation(operation_id, outcome=outcome)
except TypeError:
poller.finish_operation(operation_id)
except Exception:
logger.debug(
"Error in cell monitor finish_operation(%s)",
operation_id,
exc_info=True,
)

def note_spark_tasks_seen(self, operation_id: Optional[str]) -> None:
poller = self._poller
if poller is None or not operation_id:
return
try:
poller.note_spark_tasks_seen(operation_id)
except Exception:
logger.debug(
"Error recording Spark tasks for %s",
operation_id,
exc_info=True,
)

def should_suppress_progress_bar(self, operation_id: Optional[str]) -> bool:
poller = self._poller
if poller is None or not operation_id or self._closed:
return False
try:
return bool(poller.should_suppress_progress_bar(operation_id))
except Exception:
return False

def close(self) -> None:
with self._lock:
if self._closed:
return
self._closed = True
poller = self._poller
self._poller = None
self._spark_ui_url_getter = None
self._on_fallback = None
if poller is not None:
try:
poller.close()
except Exception:
logger.debug(
"Error closing Spark Connect cell monitor poller",
exc_info=True,
)
6 changes: 5 additions & 1 deletion google/cloud/managed_spark_connect/_ipython.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@
"google.cloud.managed_spark_magics",
(("line", "dpip"), ("line", "sparksql"), ("cell", "sparksql")),
),
(
"managed_spark_cell_monitoring.spark_connect",
(("line", "cellmonitor"),),
),
)


Expand Down Expand Up @@ -76,7 +80,7 @@ class ManagedSparkConnect(Configurable):
enable_extras = Bool(
help=(
"Whether to automatically load the installed interactive extras "
"(explore_dataframe, %dpip, %%sparksql)."
"(explore_dataframe, %dpip, %%sparksql, %cellmonitor)."
),
).tag(config=True)

Expand Down
6 changes: 5 additions & 1 deletion google/cloud/managed_spark_connect/client/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,11 @@

logger = logging.getLogger(__name__)

_UNGUARDED_METHOD_SUFFIXES = ("/ReleaseExecute", "/ReleaseSession")
_UNGUARDED_METHOD_SUFFIXES = (
"/ReleaseExecute",
"/ReleaseSession",
"/GetStatus",
)
_GUARD_MODE: contextvars.ContextVar[str] = contextvars.ContextVar(
"msc_guard_mode", default="direct"
)
Expand Down
107 changes: 98 additions & 9 deletions google/cloud/managed_spark_connect/session.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import threading
import time
import uuid
import weakref
import tqdm
from packaging import version
from types import MethodType
Expand Down Expand Up @@ -101,6 +102,7 @@
TerminateSessionRequest,
)
from google.cloud.dataproc_v1.types import sessions
from google.cloud.managed_spark_connect import _cell_monitoring
from google.cloud.managed_spark_connect import _logging
from google.cloud.managed_spark_connect import _runtimes
from google.cloud.managed_spark_connect import environment
Expand Down Expand Up @@ -2046,6 +2048,17 @@ def __init__(
self._show_operation_links: Optional[bool] = show_operation_links
self._show_progress_bar: Optional[bool] = show_progress_bar
self._progress_bar_leave: Optional[bool] = progress_bar_leave
weak_ui_url = weakref.WeakMethod(self._get_session_spark_ui_url)
weak_fallback = weakref.WeakMethod(self._on_cell_monitor_fallback)
self._cell_monitor_poller = _cell_monitoring.SessionCellMonitorBridge(
self.client,
spark_ui_url_getter=lambda: (
fn() if (fn := weak_ui_url()) is not None else ""
),
on_fallback=lambda op_ids: (
fn(op_ids) if (fn := weak_fallback()) is not None else None
),
)
# The _execute_and_fetch hooks stack in install order, innermost
# first: the position-lost replay (so the hooks around it see one
# call, with the caller's request, however many attempts it takes),
Expand Down Expand Up @@ -2073,17 +2086,50 @@ def _execute_plan_request_with_metadata(*args, **kwargs):

def execute_hook(inner):
def _execute(req, *args, **kwargs):
monitored = False
if not self._sql_lazy_transformation(req):
self._display_operation_link(req.operation_id)
return inner(req, *args, **kwargs)
monitored = self._cell_monitor_poller.begin_operation(
req.operation_id
)
if not monitored:
self._display_operation_link(req.operation_id)
outcome = "SUCCEEDED"
try:
return inner(req, *args, **kwargs)
except BaseException as exc:
outcome = _cell_monitoring.classify_exception_outcome(exc)
raise
finally:
if monitored:
self._cell_monitor_poller.finish_operation(
req.operation_id, outcome=outcome
)

return _execute
Comment on lines 2087 to 2108

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

In _execute, _cell_monitoring.classify_exception_outcome(exc) is called unconditionally when an exception is raised, even if monitored is False. This is inefficient and introduces unnecessary risk of secondary exceptions masking the original error on unmonitored operations. We should only classify the exception if monitored is True, matching the pattern used in _execute_and_fetch_as_iterator.

        def execute_hook(inner):
            def _execute(req, *args, **kwargs):
                monitored = False
                if not self._sql_lazy_transformation(req):
                    monitored = self._cell_monitor_poller.begin_operation(
                        req.operation_id
                    )
                    if not monitored:
                        self._display_operation_link(req.operation_id)
                outcome = "SUCCEEDED"
                try:
                    return inner(req, *args, **kwargs)
                except BaseException as exc:
                    if monitored:
                        outcome = _cell_monitoring.classify_exception_outcome(exc)
                    raise
                finally:
                    if monitored:
                        self._cell_monitor_poller.finish_operation(
                            req.operation_id, outcome=outcome
                        )

            return _execute


def execute_and_fetch_as_iterator_hook(inner):
def _execute_and_fetch_as_iterator(req, *args, **kwargs):
monitored = False
if not self._sql_lazy_transformation(req):
self._display_operation_link(req.operation_id)
return inner(req, *args, **kwargs)
monitored = self._cell_monitor_poller.begin_operation(
req.operation_id
)
if not monitored:
self._display_operation_link(req.operation_id)
try:
iterator = inner(req, *args, **kwargs)
except BaseException as exc:
if monitored:
self._cell_monitor_poller.finish_operation(
req.operation_id,
outcome=_cell_monitoring.classify_exception_outcome(
exc
),
)
raise
if not monitored:
return iterator
return self._wrap_monitored_iterator(iterator, req.operation_id)

return _execute_and_fetch_as_iterator

Expand Down Expand Up @@ -2165,18 +2211,28 @@ def handler(
if operation_id is None:
return

# Don't build / render progress bar for non-interactive (despite
# Ipython or non-IPython) or when disabled, unless tests forced the bar on.
if not self._should_show_progress_bar():
return

total_tasks = 0
completed_tasks = 0

for stage in stages or []:
total_tasks += stage.num_tasks
completed_tasks += stage.num_completed_tasks

poller = self.__dict__.get("_cell_monitor_poller")
if poller is not None:
if total_tasks > 0:
poller.note_spark_tasks_seen(operation_id)
if poller.should_suppress_progress_bar(operation_id):
self._close_progress_bar(
operation_id, reason="cell_monitor"
)
return

# Don't build / render progress bar for non-interactive (despite
# Ipython or non-IPython) or when disabled, unless tests forced the bar on.
if not self._should_show_progress_bar():
return

# Don't show progress bar till we receive some tasks
if total_tasks == 0:
return
Expand Down Expand Up @@ -2322,6 +2378,36 @@ def _sql_lazy_transformation(req):
except AttributeError:
return False

def _wrap_monitored_iterator(self, iterator, operation_id: str):
outcome = "CANCELLED"
try:
yield from iterator
outcome = "SUCCEEDED"
except BaseException as exc:
outcome = _cell_monitoring.classify_exception_outcome(exc)
raise
finally:
poller = self.__dict__.get("_cell_monitor_poller")
if poller is not None:
poller.finish_operation(operation_id, outcome=outcome)

def _get_session_spark_ui_url(self) -> str:
if (
self._region is not None
and self._active_s8s_session_id is not None
and self._project_id is not None
):
return (
f"{_MANAGED_SPARK_SESSIONS_BASE_URL}/{self._region}/"
f"{self._active_s8s_session_id}/sparkApplications/applications"
f"?project={self._project_id}"
)
return ""

def _on_cell_monitor_fallback(self, operation_ids: List[str]) -> None:
for op_id in operation_ids:
self._display_operation_link(op_id)

def _repr_html_(self) -> str:
if not self._active_s8s_session_id:
return """
Expand Down Expand Up @@ -2549,6 +2635,9 @@ def _close_client(self) -> None:
logger.debug(
"Failed to close the StreamingQueryManager", exc_info=True
)
poller = self.__dict__.get("_cell_monitor_poller")
if poller is not None:
poller.close()
Comment on lines +2638 to +2640

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

While closing the poller in _close_client is correct, we should also ensure the poller is closed when the session is garbage collected (i.e., in __del__). Currently, __del__ only calls super().__del__() if _client_closed is False, which does not trigger _close_client or close the poller. This can lead to background thread/resource leaks if a session goes out of scope without being explicitly stopped. Additionally, using getattr(self, "_cell_monitor_poller", None) is more idiomatic than direct __dict__ access.

Suggested change
poller = self.__dict__.get("_cell_monitor_poller")
if poller is not None:
poller.close()
poller = getattr(self, "_cell_monitor_poller", None)
if poller is not None:
poller.close()

try:
client.close()
except Exception:
Expand Down
Loading
Loading