From e604681fd21e7230d4cc2aa8ab82be2c070329e6 Mon Sep 17 00:00:00 2001 From: Dave Borowitz Date: Mon, 16 Nov 2026 10:57:41 -0800 Subject: [PATCH] fix(resilience): cut silent streams before server detach timeout and clarify OPERATION_ABANDONED Long-running Spark Connect queries can fail during ReattachExecute with [INVALID_HANDLE.OPERATION_ABANDONED] when a backend tunnel or stream behind an open WebSocket stalls silently, or when an operation stays detached (due to a client disconnect or paused iterator consumption) longer than the server's spark.connect.execute.manager.detachedTimeout (5 minutes by default): - Lower default MANAGED_SPARK_CONNECT_WS_SILENCE_LIMIT_S from 300 to 150: Spark Connect sends progress updates while tasks run and rotates reattachable streams every 120 s (senderMaxStreamDuration=2m) even when idle. Because forward_bytes checks silence on each 60 s WebSocket recv() poll, the previous 300 s limit waited 300-360 s before cutting a silent stream, past Spark's 5-minute detachedTimeout. Lowering the default to 150 s (> /usr/local/google/home/dborowitz/.gemini/config/120 s stream rotation) cuts wedged streams within 150-210 s, leaving time to reconnect and reattach before a 5-minute server timeout expires. - Enrich INVALID_HANDLE.OPERATION_ABANDONED without re-running operations: add install_operation_abandoned_handler in client/reattach_patch.py and install it on every ManagedSparkSession client across _execute_and_fetch_as_iterator, _execute_and_fetch, and _execute. Never auto-replay abandoned operations; emit reattach.operation_abandoned (operation_id, plan_kind, command_kind); close open progress bars at actual completed task counts; isolate hook/enrichment errors so the original exception is never masked; and append Managed Spark recovery guidance onto the raised PySpark exception. - Preserve multi-paragraph messages in %sparksql / %%sparksql error formatting: split formatted Spark SQL errors into prose paragraphs prior to any SQL/plan context or JVM stack trace so appended Note paragraphs are rendered rather than truncated at the first blank line. --- README.md | 5 +- docs/instrumentation.md | 9 +- .../managed_spark_connect/client/proxy.py | 12 +- .../client/reattach_patch.py | 167 +++++- google/cloud/managed_spark_connect/session.py | 19 +- google/cloud/managed_spark_magics/sparksql.py | 40 +- .../managed_spark_magics/test_sparksql.py | 34 ++ tests/unit/test_proxy.py | 97 +++- tests/unit/test_reattach_patch.py | 531 +++++++++++++++++- 9 files changed, 880 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 05c7da59..0b9610e8 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ These environment variables change how the client behaves. | Variable | Default | Effect | |----------|---------|--------| | `MANAGED_SPARK_CONNECT_REPLAY_ON_POSITION_LOST` | `1` | When a read query (`collect`, `toPandas`, `count`, ...) fails with `INVALID_CURSOR.POSITION_NOT_AVAILABLE` because the server no longer buffers the result the client has to resume from, re-run it once under a new operation id and log a `WARNING`. Commands (writes, DDL) are never re-run, and neither are queries whose plan contains a UDF, UDTF, `mapInPandas`, `mapInArrow` or `applyInPandas`, or that come from `foreach` or `foreachPartition`, since re-running them re-runs any side effects of that code. UDFs only called by name (registered with `spark.udf.register` and called from SQL) aren't detected. Set to `0` to disable, e.g. for queries that are expensive or not deterministic, or that call side-effecting UDFs by name. Read when the session is created. | -| `MANAGED_SPARK_CONNECT_WS_SILENCE_LIMIT_S` | `300` | Cut the proxy WebSocket connection if a query's result stream (a reattachable `ExecutePlan` or `ReattachExecute`) has been waiting this many seconds for its next response and nothing at all was received, so a wedged stream fails and pyspark reattaches instead of hanging. By default the server ends these streams every 2 minutes even when it has nothing to send, so that much silence means the connection is broken. Other RPCs, such as a long `AnalyzePlan`, and result streams that nobody is reading from don't count. The check runs on each 60 s WebSocket receive poll, so the cut happens between this limit and the limit plus 60 s. Set to `0` to disable. | +| `MANAGED_SPARK_CONNECT_WS_SILENCE_LIMIT_S` | `150` | Cut the proxy WebSocket connection if a query's result stream (a reattachable `ExecutePlan` or `ReattachExecute`) has been waiting this many seconds for its next response and nothing at all was received, so a wedged stream behind an open WebSocket fails and pyspark reattaches instead of hanging or hitting `INVALID_HANDLE.OPERATION_ABANDONED`. By default the server sends progress updates while tasks run and ends these streams every 2 minutes (`spark.connect.execute.reattachable.senderMaxStreamDuration`) even when it has nothing to send; keep custom values above `senderMaxStreamDuration` so idle streams are not cut prematurely. Other RPCs, such as a long `AnalyzePlan`, and result streams that nobody is reading from don't count. The check runs on each 60 s WebSocket receive poll, so the cut happens between this limit and the limit plus 60 s (150–210 s by default, before Spark's 5-minute default `detachedTimeout`). Set to `0` to disable. | | `MANAGED_SPARK_CONNECT_LIVENESS_TTL_SECONDS` | unset | Before an RPC, the client checks that the Dataproc session is still alive. Once a check finds it `ACTIVE`, that answer is reused for this many seconds: unset or negative reuses it until an RPC fails, `0` checks before every RPC. A failed RPC always triggers a fresh check. The session only counts as gone when Dataproc reports it not found or in a terminal state; when the check itself fails (for example during a control-plane outage), the RPC proceeds. An RPC on a gone session raises `SessionNotActiveError` (a `RuntimeError`, from `google.cloud.managed_spark_connect.exceptions`) saying what Dataproc reported, e.g. `FAILED` and why. | | `MANAGED_SPARK_CONNECT_SESSION_START_RETRIES` | `1` | `getOrCreate()` makes a new session's first RPC itself, and if that fails and Dataproc reports the session gone (failed or terminated before it could be used), creates another session in its place, this many times, logging a `WARNING`. Only for generated session IDs: a session created with `sessionId()` raises `SessionNotActiveError` instead. `0` disables retrying. | | `GRPC_VERBOSITY` | `ERROR` | gRPC's own variable. If it is unset when this package is imported, and gRPC hasn't been imported yet, the package sets it to `ERROR` for the whole process to silence gRPC's informational fork messages. Set it yourself to override. | @@ -192,7 +192,8 @@ result conversions — as timed *spans*. It is off by default; opt in via Without instrumentation the session still hooks a few pyspark client methods on its own client (operation ids and links, progress bars, the -`INVALID_CURSOR.POSITION_NOT_AVAILABLE` replay; see +`INVALID_CURSOR.POSITION_NOT_AVAILABLE` replay, and +`INVALID_HANDLE.OPERATION_ABANDONED` error guidance; see [Configuration](#configuration-environment-variables)) and coalesces the `ReleaseExecute` RPCs of its result streams. diff --git a/docs/instrumentation.md b/docs/instrumentation.md index 71b6e0c2..358b9c85 100644 --- a/docs/instrumentation.md +++ b/docs/instrumentation.md @@ -74,8 +74,10 @@ JsonlFileSink(path, *, append=True) InMemorySink() LoggingSink(logger=None, `wrap_client=False` installs no instrumentation wrappers on the pyspark client (no `client.*` spans). The session's own client hooks are installed either way: they set operation ids, display operation links, close a query's progress -bar when it ends and re-run a read query once on -`INVALID_CURSOR.POSITION_NOT_AVAILABLE` (see `reattach.position_lost` below). +bar when it ends, re-run a read query once on +`INVALID_CURSOR.POSITION_NOT_AVAILABLE` (see `reattach.position_lost` below), +and enrich `INVALID_HANDLE.OPERATION_ABANDONED` errors without re-running the +operation (see `reattach.operation_abandoned` below). Every session also coalesces the `ReleaseExecute(until)` RPCs of its result streams, through a process-wide patch of pyspark's `ExecutePlanResponseReattachableIterator._release_until` that leaves the @@ -142,11 +144,12 @@ sessions; a session without an `Instrumentation` pays nothing for them. | `auth.token_refreshed` | The Application Default Credentials were actually refreshed while opening a websocket (inside the `auth.refresh` span) | `expiry_wall_ns` (epoch ns of the new token's expiry, `null` if unknown), `had_token` (a token existed before the refresh) | | `progress.bar_open` | The execution progress handler created a progress bar for an operation (first progress message with tasks that is not already final) | `operation_id`, `total_tasks` | | `progress.bar_update` | Every later progress message for an open bar, except the final one | `operation_id`, `completed_tasks`, `total_tasks` | -| `progress.bar_close` | The bar was closed and unregistered: by the handler on the final (`done=True`) message, or, if that never came, when the query's `_execute_and_fetch` returned or raised | `operation_id`, `completed_tasks`, `total_tasks`, `reason=done\|query_end\|replay` (`replay`: the query hit `INVALID_CURSOR.POSITION_NOT_AVAILABLE` and is re-run under a new operation id) | +| `progress.bar_close` | The bar was closed and unregistered: by the handler on the final (`done=True`) message, or, if that never came, when the execution hook (`_execute_and_fetch_as_iterator`, `_execute_and_fetch`, or `_execute`) returned or raised | `operation_id`, `completed_tasks`, `total_tasks`, `reason=done\|query_end\|replay` (`replay`: the query hit `INVALID_CURSOR.POSITION_NOT_AVAILABLE` and is re-run under a new operation id) | | `guard.result` | The session liveness guard ran (before an RPC, or after a failed one) and either had to ask the control plane or found the session inactive. A cached "active" verdict, the normal case for every RPC, emits nothing | `mode=pre_rpc\|on_error` (`direct` when called outside the channel), `cached` (answered from the liveness cache, no `GetSession` RPC), `active` (the verdict; `null` when the check raised), `error_type` (only when the check raised) | | `grpc.retry` | pyspark's retry loop is about to retry an RPC, after the policy's backoff. The failed RPC is the `grpc.*` span that ended just before, on the same `thread` | `attempt` (1 = first retry within that loop), `reason=error\|reattach` (`reattach`: the server ended the stream and pyspark reattaches), `grpc_code`, `error_type` | | `session.start_failed` | A session `getOrCreate()` just created never became usable: Dataproc reported it failed or terminated, either in the `CreateSession` result, while waiting for its endpoint, or after its first RPC failed (see `session.check_new`) | `detail` (what Dataproc reported, e.g. `Managed Spark Session sc-123 is FAILED: `), `attempt` (1 for the first session), `retry` (whether another session is created in its place: only for generated session IDs, up to `MANAGED_SPARK_CONNECT_SESSION_START_RETRIES` times) | | `reattach.position_lost` | A query's `_execute_and_fetch` (`collect`, `toPandas`, `count`, ...) failed with `INVALID_CURSOR.POSITION_NOT_AVAILABLE`: the server no longer buffers the response the client has to reattach from. A replay also logs a `WARNING` | `operation_id`, `replay` (whether the query is re-run once under a new operation id: only root (read) plans without inline UDFs or UDTFs, and not with `MANAGED_SPARK_CONNECT_REPLAY_ON_POSITION_LOST=0`; if the re-run fails the same way, a second event has its `operation_id` and `replay=false`), `new_operation_id` (only when `replay=true`) | +| `reattach.operation_abandoned` | An execution (`_execute_and_fetch_as_iterator`, `_execute_and_fetch`, or `_execute`) failed with `INVALID_HANDLE.OPERATION_ABANDONED`: the operation stayed detached (for example due to lost connectivity or paused iterator consumption) longer than `spark.connect.execute.manager.detachedTimeout` and was removed by the server. Never re-run automatically; recovery guidance is appended onto the raised PySpark exception | `operation_id`, `plan_kind=root\|command` (when present on the `ExecutePlanRequest`), `command_kind` (only when `plan_kind=command` and set, e.g. `sql_command`, `write_operation`) | Notes: `progress.*` events require the progress handler to be active, which it only is in interactive environments. diff --git a/google/cloud/managed_spark_connect/client/proxy.py b/google/cloud/managed_spark_connect/client/proxy.py index 3c2387f5..ef671f6d 100755 --- a/google/cloud/managed_spark_connect/client/proxy.py +++ b/google/cloud/managed_spark_connect/client/proxy.py @@ -56,8 +56,18 @@ # Cut a connection whose server stays silent this long while a result stream # is waiting for its next response (see forward_bytes). 0 disables. +# +# While WebSocket keepalive catches a dead first-hop peer in ~80 s, a backend +# tunnel or HTTP/2 stream behind an open WebSocket frontend can stall silently +# while PONGs keep flowing. Spark Connect sends progress updates while tasks +# run and rotates reattachable streams every 120 s (senderMaxStreamDuration=2m) +# even when idle, while detached operations are evicted after +# spark.connect.execute.manager.detachedTimeout (5m by default on unmodified +# servers). Because recv() polls every WS_RECV_TIMEOUT_S (60 s), 150 s (> 120 s +# stream rotation) typically cuts a silent stream within 150-210 s, leaving +# time to reconnect and send ReattachExecute before a 5m server timeout expires. _SILENCE_LIMIT_ENV = "MANAGED_SPARK_CONNECT_WS_SILENCE_LIMIT_S" -_DEFAULT_SILENCE_LIMIT_S = 300.0 +_DEFAULT_SILENCE_LIMIT_S = 150.0 def clear_credentials_cache(): diff --git a/google/cloud/managed_spark_connect/client/reattach_patch.py b/google/cloud/managed_spark_connect/client/reattach_patch.py index 723d0131..b910632c 100644 --- a/google/cloud/managed_spark_connect/client/reattach_patch.py +++ b/google/cloud/managed_spark_connect/client/reattach_patch.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Client-side recovery from ``INVALID_CURSOR.POSITION_NOT_AVAILABLE``. +"""Client-side reattach recovery, abandoned-handle guidance, and release coalescing. When a transport blip breaks an ``ExecutePlan`` stream, pyspark reattaches from the last response it returned. If the server no longer buffers that @@ -20,7 +20,7 @@ ``DefaultPolicy`` does not retry, and the whole query fails. :func:`install_replay_on_position_lost` hooks ``client._execute_and_fetch`` -``client._execute_and_fetch`` (``collect``, ``toPandas``, ``count``, ...): +(``collect``, ``toPandas``, ``count``, ...): when a root (read) plan fails that way, the query is re-run once under a fresh ``operation_id``. Command plans, and plans with inline user code (UDFs, ``mapInPandas``, ``foreach``, ...) whose side effects would run @@ -28,6 +28,17 @@ ``reattach.position_lost``, and a replay also logs a ``WARNING``. ``MANAGED_SPARK_CONNECT_REPLAY_ON_POSITION_LOST=0`` turns the replay off. +:func:`install_operation_abandoned_handler` hooks the client's execution +methods (``_execute_and_fetch_as_iterator``, ``_execute_and_fetch`` and +``_execute``) for ``[INVALID_HANDLE.OPERATION_ABANDONED]``, which the server +raises when an operation stays detached (for example because connectivity was +lost or an iterator consumer paused reading) longer than +``spark.connect.execute.manager.detachedTimeout`` and is removed by the +server. Operations are never re-run automatically after abandonment; instead +the hook emits ``reattach.operation_abandoned``, closes any open progress bar +for the operation, and appends actionable Managed Spark recovery guidance onto +the PySpark exception in place. + :func:`install_release_coalescing` makes the reattachable iterators of one client send at most one ``ReleaseExecute(until)`` at a time. pyspark sends one per response, each on its release thread pool, so a fast stream behind a @@ -56,13 +67,68 @@ ENV_REPLAY_ON_POSITION_LOST = "MANAGED_SPARK_CONNECT_REPLAY_ON_POSITION_LOST" POSITION_LOST_CONDITION = "INVALID_CURSOR.POSITION_NOT_AVAILABLE" +OPERATION_ABANDONED_CONDITION = "INVALID_HANDLE.OPERATION_ABANDONED" + +_OPERATION_ABANDONED_HANDLED_ATTR = "_msc_operation_abandoned_handled" +_OPERATION_ABANDONED_NOTE = ( + "Note: The client lost connectivity to (or paused reading results from) " + "the Managed Spark session for longer than " + "'spark.connect.execute.manager.detachedTimeout', so the server removed " + "the detached operation (which may or may not have completed). The " + "session is still usable; for write, DDL, or DML statements, check " + "whether changes were committed before re-running. To tolerate longer " + "client disconnects on new sessions, set " + ".config('spark.connect.execute.manager.detachedTimeout', '30m') (or a " + "longer duration) when creating the session." +) + + +def _error_condition(exc: BaseException) -> Optional[str]: + try: + cond = getattr(exc, "getCondition", None) or getattr( + exc, "getErrorClass", None + ) + except Exception: + return None + if isinstance(cond, str): + return cond or None + if callable(cond): + try: + value = cond() + except Exception: + return None + return value if isinstance(value, str) and value else None + return None def _is_position_not_available(exc: BaseException) -> bool: - cond = getattr(exc, "getCondition", None) or getattr( - exc, "getErrorClass", None - ) - return callable(cond) and cond() == POSITION_LOST_CONDITION + return _error_condition(exc) == POSITION_LOST_CONDITION + + +def _is_operation_abandoned(exc: BaseException) -> bool: + return _error_condition(exc) == OPERATION_ABANDONED_CONDITION + + +def _enrich_operation_abandoned_error(exc: BaseException) -> None: + """Append Managed Spark recovery guidance onto ``exc`` in place.""" + try: + msg = getattr(exc, "_message", None) + if isinstance(msg, str) and _OPERATION_ABANDONED_NOTE not in msg: + sep = "\n\n" if msg else "" + exc._message = f"{msg}{sep}{_OPERATION_ABANDONED_NOTE}" + if ( + isinstance(exc.args, tuple) + and len(exc.args) == 1 + and isinstance(exc.args[0], str) + and _OPERATION_ABANDONED_NOTE not in exc.args[0] + ): + exc.args = (exc._message,) + except Exception: + logger.debug( + "Failed to enrich %s exception message", + OPERATION_ABANDONED_CONDITION, + exc_info=True, + ) # Inline user code in a plan: Python, Scala and Java UDFs (including the @@ -197,6 +263,95 @@ def _execute_and_fetch(req, *args, **kwargs): install_client_hooks(client, {"_execute_and_fetch": replay_hook}) +def _record_and_enrich_abandoned( + exc: BaseException, + req: Any, + recorder: Any, + close_progress_bar: Optional[Callable[..., None]], +) -> None: + if not _is_operation_abandoned(exc): + return + if getattr(exc, _OPERATION_ABANDONED_HANDLED_ATTR, False): + return + try: + setattr(exc, _OPERATION_ABANDONED_HANDLED_ATTR, True) + except Exception: # pragma: no cover - defensive for custom exceptions + pass + operation_id = getattr(req, "operation_id", None) or "" + try: + attrs = {"operation_id": operation_id} + plan_kind, command_kind = _plan_kinds(req) + if plan_kind is not None: + attrs["plan_kind"] = plan_kind + if command_kind is not None: + attrs["command_kind"] = command_kind + recorder.event("reattach.operation_abandoned", **attrs) + except Exception: + logger.debug( + "Failed to record reattach.operation_abandoned event", + exc_info=True, + ) + _enrich_operation_abandoned_error(exc) + if close_progress_bar is not None: + try: + close_progress_bar(operation_id or None) + except Exception: + logger.debug( + "Failed to close progress bar for abandoned operation %s", + operation_id, + exc_info=True, + ) + + +def install_operation_abandoned_handler( + client: Any, + recorder: Any = NOOP_RECORDER, + close_progress_bar: Optional[Callable[..., None]] = None, +) -> None: + """Emit ``reattach.operation_abandoned`` and enrich abandoned-handle errors. + + Hooks whichever of ``_execute_and_fetch_as_iterator``, + ``_execute_and_fetch`` and ``_execute`` exist on ``client`` so all query, + SQL command (including DDL/DML) and ``DataFrameWriter`` paths are covered + without re-running the operation or duplicating telemetry/guidance when + ``_execute_and_fetch`` delegates to ``_execute_and_fetch_as_iterator``. + """ + + def make_sync_hook(inner): + def wrapped(req, *args, **kwargs): + try: + return inner(req, *args, **kwargs) + except Exception as exc: + _record_and_enrich_abandoned( + exc, req, recorder, close_progress_bar + ) + raise + + return wrapped + + def make_iterator_hook(inner): + def wrapped(req, *args, **kwargs): + try: + yield from inner(req, *args, **kwargs) + except Exception as exc: + _record_and_enrich_abandoned( + exc, req, recorder, close_progress_bar + ) + raise + + return wrapped + + hooks = {} + if hasattr(client, "_execute_and_fetch_as_iterator"): + hooks["_execute_and_fetch_as_iterator"] = make_iterator_hook + if hasattr(client, "_execute_and_fetch"): + hooks["_execute_and_fetch"] = make_sync_hook + if hasattr(client, "_execute"): + hooks["_execute"] = make_sync_hook + if hooks: + install_client_hooks(client, hooks) + + # Stubs of the clients install_release_coalescing() was called for. An # iterator coalesces only if it talks through one of them. _COALESCED_STUBS: "weakref.WeakSet[Any]" = weakref.WeakSet() diff --git a/google/cloud/managed_spark_connect/session.py b/google/cloud/managed_spark_connect/session.py index 61f5256d..6df8531d 100644 --- a/google/cloud/managed_spark_connect/session.py +++ b/google/cloud/managed_spark_connect/session.py @@ -69,6 +69,7 @@ get_default_credentials, ) from google.cloud.managed_spark_connect.client.reattach_patch import ( + install_operation_abandoned_handler, install_release_coalescing, install_replay_on_position_lost, ) @@ -2046,13 +2047,21 @@ def __init__( self._show_operation_links: Optional[bool] = show_operation_links self._show_progress_bar: Optional[bool] = show_progress_bar self._progress_bar_leave: Optional[bool] = progress_bar_leave - # The _execute_and_fetch hooks stack in install order, innermost - # first: the position-lost replay (so the hooks around it see one - # call, with the caller's request, however many attempts it takes), - # then the instrumentation wrappers, then the progress bar hook below. + # Client execution hooks stack in install order, innermost first: + # abandoned-operation enrichment (across _execute_and_fetch_as_iterator, + # _execute_and_fetch, and _execute), then the position-lost replay on + # _execute_and_fetch (so the hooks around the replay see one call, with + # the caller's request, however many attempts it takes), then the + # instrumentation wrappers, then the progress bar hook below. + recorder = get_recorder(instrumentation) + install_operation_abandoned_handler( + self.client, + recorder=recorder, + close_progress_bar=self._close_progress_bar, + ) install_replay_on_position_lost( self.client, - recorder=get_recorder(instrumentation), + recorder=recorder, close_progress_bar=self._close_progress_bar, ) if instrumentation is not None and instrumentation.wrap_client: diff --git a/google/cloud/managed_spark_magics/sparksql.py b/google/cloud/managed_spark_magics/sparksql.py index c2431522..aa494d8a 100644 --- a/google/cloud/managed_spark_magics/sparksql.py +++ b/google/cloud/managed_spark_magics/sparksql.py @@ -696,17 +696,18 @@ def _parse_position(exc: Any, sql: str) -> Optional[Tuple[int, int]]: # Where Spark's own message ends and the extras it appends begin: the JVM -# stack trace (classic, and Connect's str()), an "== SQL ==" excerpt, and on -# Connect the analyzed plan after "; line 2 pos 2;" or a parse error's -# " (line 1, pos 0)". -_MESSAGE_END_RE = re.compile( - r"\n\s*\n|\n== SQL|\nJVM stacktrace:|;\s*line \d+ pos \d+;?" +# stack trace (classic, and Connect's str()), an "== SQL ==" or +# "== DataFrame ==" excerpt, and on Connect the analyzed plan after +# "; line 2 pos 2;" or a parse error's " (line 1, pos 0)". +_MESSAGE_TRAILER_RE = re.compile( + r"\n== (?:SQL|DataFrame)|\nJVM stacktrace:|;\s*line \d+ pos \d+;?" r"| \(line \d+, pos \d+\)" ) +_STACKTRACE_PARA_RE = re.compile(r"(?:^|\n)\s*(?:at [\w.$]+\(|Caused by:)") -def _first_paragraph(exc: Any, condition: Optional[str]) -> str: - """Spark's message, without the context and JVM stack it appends.""" +def _message_paragraphs(exc: Any, condition: Optional[str]) -> List[str]: + """Prose paragraphs of Spark's message before any plan or JVM stack.""" text = None get_message = getattr(exc, "getMessage", None) if get_message is not None: @@ -716,10 +717,20 @@ def _first_paragraph(exc: Any, condition: Optional[str]) -> str: text = None if not text: text = str(exc) - text = _MESSAGE_END_RE.split(text.strip(), maxsplit=1)[0].rstrip() - if condition and not text.startswith("["): - text = f"[{condition}] {text}" - return text + body = _MESSAGE_TRAILER_RE.split(text.strip(), maxsplit=1)[0].rstrip() + paragraphs: List[str] = [] + for raw in re.split(r"\n\s*\n", body): + chunk = raw.strip() + if not chunk: + continue + if paragraphs and _STACKTRACE_PARA_RE.search(chunk): + break + paragraphs.append(chunk) + if not paragraphs: + paragraphs = [""] + if condition and not paragraphs[0].startswith("["): + paragraphs[0] = f"[{condition}] {paragraphs[0]}".rstrip() + return paragraphs def format_error( @@ -732,7 +743,7 @@ def format_error( ) -> str: """Formats a Spark SQL error from `statement` for display.""" condition = _condition(exc) - lines = [_first_paragraph(exc, condition)] + paragraphs = _message_paragraphs(exc, condition) sqlstate = None get_state = getattr(exc, "getSqlState", None) if get_state is not None: @@ -740,8 +751,9 @@ def format_error( sqlstate = get_state() except Exception: sqlstate = None - if sqlstate and f"SQLSTATE: {sqlstate}" not in lines[0]: - lines[0] += f" SQLSTATE: {sqlstate}" + if sqlstate and f"SQLSTATE: {sqlstate}" not in paragraphs[0]: + paragraphs[0] += f" SQLSTATE: {sqlstate}" + lines = ["\n\n".join(paragraphs)] where = [] if count > 1: diff --git a/tests/unit/managed_spark_magics/test_sparksql.py b/tests/unit/managed_spark_magics/test_sparksql.py index db157eae..2858b5e4 100644 --- a/tests/unit/managed_spark_magics/test_sparksql.py +++ b/tests/unit/managed_spark_magics/test_sparksql.py @@ -425,6 +425,40 @@ def test_drops_jvm_stacktrace(self): sparksql.format_error(exc, "SELECT x", stmt, 0, 1, True), "Bad." ) + def test_preserves_multiple_paragraphs_before_jvm_stacktrace(self): + stmts = sparksql.split_statements("SELECT 1;\nUPDATE t SET a = 1") + exc = FakeSparkError( + "[INVALID_HANDLE.OPERATION_ABANDONED] The handle abc is invalid." + " Operation was considered abandoned.\n\nNote: The client lost" + " connectivity to (or paused reading results from) the Managed" + " Spark session.\n\nAdditional context paragraph.\n\nJVM" + " stacktrace:\norg.apache.spark.SparkSQLException\n\tat" + " org.apache.spark.Foo.bar(Foo.scala:50)", + condition="INVALID_HANDLE.OPERATION_ABANDONED", + sqlstate="HY000", + ) + msg = sparksql.format_error( + exc, "SELECT 1;\nUPDATE t SET a = 1", stmts[1], 1, 2, True + ) + self.assertEqual( + msg, + "[INVALID_HANDLE.OPERATION_ABANDONED] The handle abc is invalid." + " Operation was considered abandoned. SQLSTATE: HY000\n\nNote: The" + " client lost connectivity to (or paused reading results from) the" + " Managed Spark session.\n\nAdditional context paragraph.\n" + "(statement 2 of 2; earlier statements already ran)", + ) + + def test_drops_headerless_java_stacktrace_paragraph(self): + (stmt,) = sparksql.split_statements("SELECT x") + exc = FakeSparkError( + "Bad.\n\norg.apache.spark.SparkException: Bad\n\tat" + " org.apache.spark.sql.Foo.bar(Foo.scala:12)" + ) + self.assertEqual( + sparksql.format_error(exc, "SELECT x", stmt, 0, 1, True), "Bad." + ) + def test_drops_connect_plan(self): (stmt,) = sparksql.split_statements("SELECT * FROM t") exc = FakeSparkError( diff --git a/tests/unit/test_proxy.py b/tests/unit/test_proxy.py index 6fbf8a49..3bf87860 100644 --- a/tests/unit/test_proxy.py +++ b/tests/unit/test_proxy.py @@ -843,7 +843,12 @@ def _forward_scripted_server_stream( clock = [0.0] monkeypatch.setattr(proxy.time, "monotonic", lambda: clock[0]) - monkeypatch.setenv("MANAGED_SPARK_CONNECT_WS_SILENCE_LIMIT_S", limit_s) + if limit_s is None: + monkeypatch.delenv( + "MANAGED_SPARK_CONNECT_WS_SILENCE_LIMIT_S", raising=False + ) + else: + monkeypatch.setenv("MANAGED_SPARK_CONNECT_WS_SILENCE_LIMIT_S", limit_s) remaining = list(steps) class ScriptedWebSocket: @@ -1058,6 +1063,96 @@ def test_silence_no_cut_without_waiting_since_source(monkeypatch): assert remaining == [] +def test_default_silence_limit_cuts_before_spark_detached_timeout(monkeypatch): + """Default limit (150 s + 60 s poll <= 210 s) cuts before 300 s timeout.""" + from google.cloud.managed_spark_connect.client import proxy + + assert proxy._DEFAULT_SILENCE_LIMIT_S > 120.0 + assert proxy._DEFAULT_SILENCE_LIMIT_S + proxy.WS_RECV_TIMEOUT_S < 300.0 + + p = _new_proxy() + w = _Waiter(p) + steps = [ + ("timeout", 60.0, w.begin), + ("timeout", 120.0), # 60 s silent: still below 150 s. + ("timeout", 180.0), # 120 s silent: still below 150 s. + ("timeout", 240.0), # 180 s silent (>= 150 s, < 300 s): cut! + ("timeout", 300.0), + ] + cuts, remaining = _forward_scripted_server_stream( + monkeypatch, steps, p, None + ) + assert cuts == [{"conn": 7, "silent_s": 180.0}] + assert remaining == [("timeout", 300.0)] + + +def test_default_silence_limit_preserves_silent_120s_rotation_and_cuts_mid_poll( + monkeypatch, +): + """Healthy 120 s idle rotation is preserved; worst-case mid-poll cuts < 300 s.""" + p = _new_proxy() + w = _Waiter(p) + # 1. A non-progress command stays completely silent for 120 s before its + # stream rotation frame arrives: must NOT cut under the 150 s default. + healthy_steps = [ + ("timeout", 0.0, w.begin), + ("timeout", 60.0), + ("timeout", 120.0), + ("frame", 120.1, w.end), + ] + cuts, remaining = _forward_scripted_server_stream( + monkeypatch, healthy_steps, p, None + ) + assert cuts == [] + assert remaining == [] + + # 2. Worst-case mid-poll alignment: consumer begins waiting 30.1 s into a + # 60 s poll, so poll 3 sees 149.9 s (< 150 s) and poll 4 cuts at 209.9 s + # (< 300 s server detachedTimeout). + p2 = _new_proxy() + w2 = _Waiter(p2) + stall_steps = [ + ("timeout", 30.1, w2.begin), + ("timeout", 60.0), # 29.9 s silent. + ("timeout", 120.0), # 89.9 s silent. + ("timeout", 180.0), # 149.9 s silent (< 150 s: no cut yet). + ("timeout", 240.0), # 209.9 s silent (>= 150 s, < 300 s: cut!). + ("timeout", 300.0), + ] + cuts2, remaining2 = _forward_scripted_server_stream( + monkeypatch, stall_steps, p2, None + ) + assert len(cuts2) == 1 + assert cuts2[0]["conn"] == 7 + assert cuts2[0]["silent_s"] == pytest.approx(209.9) + assert remaining2 == [("timeout", 300.0)] + + +def test_default_silence_limit_cuts_wedged_stream_rotation(monkeypatch): + """A lost 120 s stream rotation behind a live WS cuts before 5m eviction.""" + p = _new_proxy() + w = _Waiter(p) + # Tasks emit progress updates up to ~118 s, and the server rotates the + # stream at 120 s (starting its 300 s detachedTimeout clock at ~120 s). If + # the backend tunnel wedges while WS keepalive still answers pings, 60 s + # recv polls fire at 178 s, 238 s, and 298 s: the 180 s cut at t=298 s + # leaves ~122 s before the server's ~420 s abandonment sweep. + steps = [ + ("frame", 2.0, w.begin), + ("frame", 60.0), + ("frame", 118.0), + ("timeout", 178.0), # 60 s since last frame. + ("timeout", 238.0), # 120 s since last frame. + ("timeout", 298.0), # 180 s since last frame (>= 150 s): cut! + ("timeout", 358.0), + ] + cuts, remaining = _forward_scripted_server_stream( + monkeypatch, steps, p, None + ) + assert cuts == [{"conn": 7, "silent_s": 180.0}] + assert remaining == [("timeout", 358.0)] + + def test_forward_connection_gives_backend_socket_the_proxys_waits(): from google.cloud.managed_spark_connect.client import proxy diff --git a/tests/unit/test_reattach_patch.py b/tests/unit/test_reattach_patch.py index d808ef45..04f0c2e0 100644 --- a/tests/unit/test_reattach_patch.py +++ b/tests/unit/test_reattach_patch.py @@ -11,8 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Unit tests for the one-shot replay on INVALID_CURSOR.POSITION_NOT_AVAILABLE -and for ReleaseExecute coalescing.""" +"""Unit tests for reattach position-lost replay, abandoned-handle guidance, +and ReleaseExecute coalescing.""" import gc import logging @@ -38,7 +38,9 @@ from google.cloud.managed_spark_connect.client import reattach_patch from google.cloud.managed_spark_connect.client.reattach_patch import ( ENV_REPLAY_ON_POSITION_LOST, + OPERATION_ABANDONED_CONDITION, POSITION_LOST_CONDITION, + install_operation_abandoned_handler, install_release_coalescing, install_replay_on_position_lost, ) @@ -743,3 +745,528 @@ def test_release_failures_are_logged_as_warnings(caplog): _wait_idle(it) assert "ReleaseExecute failed with exception: boom." in caplog.messages + + +# -------------------------------------------------------------------------- +# INVALID_HANDLE.OPERATION_ABANDONED handling (no auto-replay + guidance) +# -------------------------------------------------------------------------- + + +def _operation_abandoned_error( + handle: str = "11111111-2222-4333-8444-555555555555", + *, + display_server_stacktrace: bool = False, +) -> UnknownException: + return UnknownException( + ( + f"[{OPERATION_ABANDONED_CONDITION}] The handle {handle} is invalid." + " Operation was considered abandoned because of inactivity and" + " removed. SQLSTATE: HY000" + ), + errorClass=OPERATION_ABANDONED_CONDITION, + messageParameters={"handle": handle}, + sql_state="HY000", + server_stacktrace=( + "org.apache.spark.SparkSQLException:" + " [INVALID_HANDLE.OPERATION_ABANDONED]\n\tat" + " org.apache.spark.sql.connect.execution" + ".SparkConnectPlanExecution.foo(File.scala:50)" + ), + display_server_stacktrace=display_server_stacktrace, + ) + + +def _dml_sql_command_request(op_id: str = "op-dml") -> proto.ExecutePlanRequest: + req = proto.ExecutePlanRequest(session_id="sess-1", operation_id=op_id) + req.plan.command.sql_command.sql = ( + "UPDATE analytics.order_items SET quantity = quantity + 1" + ) + return req + + +def _write_command_request(op_id: str = "op-write") -> proto.ExecutePlanRequest: + req = proto.ExecutePlanRequest(session_id="sess-1", operation_id=op_id) + req.plan.command.write_operation.mode = ( + proto.WriteOperation.SaveMode.SAVE_MODE_APPEND + ) + req.plan.command.write_operation.input.range.end = 5 + return req + + +class _FullExecutionClient: + """Fake SparkConnectClient with ``_execute_and_fetch_as_iterator``, + ``_execute_and_fetch`` (delegating to the iterator, like PySpark) and + ``_execute``.""" + + def __init__(self, fail_fn): + self._fail_fn = fail_fn + self.requests = [] + + def _execute_and_fetch_as_iterator(self, req, observations, progress=None): + self.requests.append(req) + yield {"started": True} + raise self._fail_fn(len(self.requests), req) + + def _execute_and_fetch(self, req, observations, self_destruct=False): + out = [] + for item in self._execute_and_fetch_as_iterator(req, observations): + out.append(item) + return ("table", None, [], [], {"items": out}) + + def _execute(self, req): + self.requests.append(req) + raise self._fail_fn(len(self.requests), req) + + +def test_operation_abandoned_root_plan_never_replays_and_preserves_exception(): + inst = Instrumentation([InMemorySink()], {"run_id": "r-abandon"}) + bars = _BarCloser() + client = _FullExecutionClient( + lambda _attempt, _req: _operation_abandoned_error( + display_server_stacktrace=True + ) + ) + install_operation_abandoned_handler( + client, recorder=inst.recorder, close_progress_bar=bars + ) + install_replay_on_position_lost( + client, recorder=inst.recorder, close_progress_bar=bars + ) + + with pytest.raises(UnknownException) as excinfo: + client._execute_and_fetch(_root_request("op-read-1"), {}) + + exc = excinfo.value + # Never replayed even though it is a pure root plan without UDFs. + assert [r.operation_id for r in client.requests] == ["op-read-1"] + assert _events(inst, "reattach.position_lost") == [] + assert exc.getCondition() == OPERATION_ABANDONED_CONDITION + assert exc.getSqlState() == "HY000" + assert exc.getMessageParameters() == { + "handle": "11111111-2222-4333-8444-555555555555" + } + assert "SparkConnectPlanExecution" in (exc.getStackTrace() or "") + + rendered = str(exc) + assert "11111111-2222-4333-8444-555555555555" in rendered + assert "Managed Spark session" in rendered + assert "paused reading results from" in rendered + assert "spark.connect.execute.manager.detachedTimeout" in rendered + assert "30m" in rendered + assert "Dataproc" not in rendered + # Must not match downstream inactive-session substring heuristics. + assert "not active" not in rendered.lower() + assert "session is closed" not in rendered.lower() + assert "session not found" not in rendered.lower() + assert "stopped" not in rendered.lower() + # Appended once on getMessage()/str() and args, above JVM stacktrace, + # and pickle round-trips without duplicating the note. + import pickle + + assert "Note: The client lost connectivity" in exc.getMessage() + assert "Note: The client lost connectivity" in exc.args[0] + unpickled = pickle.loads(pickle.dumps(exc)) + assert str(unpickled) == rendered + assert rendered.count("Note: The client lost connectivity") == 1 + assert rendered.index( + "Note: The client lost connectivity" + ) < rendered.index("JVM stacktrace:") + + (ev,) = _events(inst, "reattach.operation_abandoned") + assert ev["attrs"] == { + "operation_id": "op-read-1", + "plan_kind": "root", + "run_id": "r-abandon", + } + assert bars.calls == [("op-read-1", "query_end")] + + +def test_operation_abandoned_sql_dml_command_emits_once_and_warns_about_dml(): + """DML via _execute_and_fetch -> _execute_and_fetch_as_iterator.""" + inst = Instrumentation([InMemorySink()]) + bars = _BarCloser() + client = _FullExecutionClient( + lambda _attempt, _req: _operation_abandoned_error() + ) + install_operation_abandoned_handler( + client, recorder=inst.recorder, close_progress_bar=bars + ) + install_replay_on_position_lost( + client, recorder=inst.recorder, close_progress_bar=bars + ) + + with pytest.raises(UnknownException) as excinfo: + client._execute_and_fetch(_dml_sql_command_request("op-update"), {}) + + assert [r.operation_id for r in client.requests] == ["op-update"] + rendered = str(excinfo.value) + assert rendered.count("Note: The client lost connectivity") == 1 + assert "write, DDL, or DML" in rendered + (ev,) = _events(inst, "reattach.operation_abandoned") + assert ev["attrs"] == { + "operation_id": "op-update", + "plan_kind": "command", + "command_kind": "sql_command", + } + assert bars.calls == [("op-update", "query_end")] + + +def test_operation_abandoned_dataframe_write_execute_path(): + inst = Instrumentation([InMemorySink()]) + bars = _BarCloser() + client = _FullExecutionClient( + lambda _attempt, _req: _operation_abandoned_error() + ) + install_operation_abandoned_handler( + client, recorder=inst.recorder, close_progress_bar=bars + ) + + with pytest.raises(UnknownException) as excinfo: + client._execute(_write_command_request("op-save")) + + assert [r.operation_id for r in client.requests] == ["op-save"] + assert "Managed Spark session" in str(excinfo.value) + (ev,) = _events(inst, "reattach.operation_abandoned") + assert ev["attrs"] == { + "operation_id": "op-save", + "plan_kind": "command", + "command_kind": "write_operation", + } + assert bars.calls == [("op-save", "query_end")] + + +def test_operation_abandoned_direct_iterator_and_generator_close(): + """Direct iterator consumption (toLocalIterator) and early close().""" + inst = Instrumentation([InMemorySink()]) + bars = _BarCloser() + client = _FullExecutionClient( + lambda _attempt, _req: _operation_abandoned_error() + ) + install_operation_abandoned_handler( + client, recorder=inst.recorder, close_progress_bar=bars + ) + + # 1. Early consumer close() (GeneratorExit) passes through untouched. + gen_early = client._execute_and_fetch_as_iterator( + _root_request("op-early"), {} + ) + assert next(gen_early) == {"started": True} + gen_early.close() + assert _events(inst, "reattach.operation_abandoned") == [] + assert bars.calls == [] + + # 2. Full consumption hitting OPERATION_ABANDONED enriches & closes bar. + gen = client._execute_and_fetch_as_iterator(_root_request("op-iter"), {}) + assert next(gen) == {"started": True} + with pytest.raises(UnknownException) as excinfo: + next(gen) + assert "Note: The client lost connectivity" in str(excinfo.value) + (ev,) = _events(inst, "reattach.operation_abandoned") + assert ev["attrs"] == {"operation_id": "op-iter", "plan_kind": "root"} + assert bars.calls == [("op-iter", "query_end")] + + +def test_position_lost_replay_then_operation_abandoned_records_retry_op_id(): + """Attempt 1 POSITION_NOT_AVAILABLE replays; attempt 2 OPERATION_ABANDONED.""" + inst = Instrumentation([InMemorySink()]) + bars = _BarCloser() + + def fail_sequence(attempt, _req): + if attempt == 1: + return _position_lost_error() + return _operation_abandoned_error() + + client = _FullExecutionClient(fail_sequence) + install_operation_abandoned_handler( + client, recorder=inst.recorder, close_progress_bar=bars + ) + install_replay_on_position_lost( + client, recorder=inst.recorder, close_progress_bar=bars + ) + + with pytest.raises(UnknownException) as excinfo: + client._execute_and_fetch(_root_request("op-initial"), {}) + + assert len(client.requests) == 2 + retry_op_id = client.requests[1].operation_id + assert retry_op_id != "op-initial" + assert "Note: The client lost connectivity" in str(excinfo.value) + (pos_ev,) = _events(inst, "reattach.position_lost") + assert pos_ev["attrs"] == { + "operation_id": "op-initial", + "new_operation_id": retry_op_id, + "replay": True, + } + (aban_ev,) = _events(inst, "reattach.operation_abandoned") + assert aban_ev["attrs"] == { + "operation_id": retry_op_id, + "plan_kind": "root", + } + assert bars.calls == [ + ("op-initial", "replay"), + (retry_op_id, "query_end"), + (retry_op_id, "query_end"), + ] + + +def test_enrichment_and_condition_failures_never_mask_original_exception(): + """Broken getCondition, read-only _message, or failing bar close are safe.""" + + class BadConditionError(Exception): + + def getCondition(self): + raise RuntimeError("broken getCondition") + + class ReadOnlyMessageAbandonedError(Exception): + + def getCondition(self): + return OPERATION_ABANDONED_CONDITION + + @property + def _message(self): + return "immutable message" + + class StringAttrAbandonedError(Exception): + getCondition = OPERATION_ABANDONED_CONDITION + + def __init__(self, msg): + super().__init__(msg) + self._message = msg + + def raising_close(_op_id, reason="query_end"): + raise RuntimeError(f"bar close boom ({reason})") + + inst = Instrumentation([InMemorySink()]) + errors = [ + BadConditionError("raw-1"), + ReadOnlyMessageAbandonedError("raw-2"), + StringAttrAbandonedError("raw-3"), + ] + client = _FullExecutionClient(lambda attempt, _req: errors[attempt - 1]) + install_operation_abandoned_handler( + client, recorder=inst.recorder, close_progress_bar=raising_close + ) + install_replay_on_position_lost( + client, recorder=inst.recorder, close_progress_bar=raising_close + ) + + with pytest.raises(BadConditionError, match="raw-1"): + client._execute_and_fetch(_root_request("op-bad-cond"), {}) + assert _events(inst, "reattach.operation_abandoned") == [] + + with pytest.raises(ReadOnlyMessageAbandonedError, match="raw-2"): + client._execute_and_fetch(_root_request("op-ro-msg"), {}) + (ev,) = _events(inst, "reattach.operation_abandoned") + assert ev["attrs"] == {"operation_id": "op-ro-msg", "plan_kind": "root"} + + with pytest.raises(StringAttrAbandonedError) as excinfo: + client._execute_and_fetch(_root_request("op-str-attr"), {}) + assert "Note: The client lost connectivity" in excinfo.value._message + assert len(_events(inst, "reattach.operation_abandoned")) == 2 + + +def test_message_mentioning_operation_abandoned_without_condition_is_untouched(): + inst = Instrumentation([InMemorySink()]) + original_msg = f"Query literal '{OPERATION_ABANDONED_CONDITION}' failed" + client = _FakeClient( + lambda _attempt, _req: (_ for _ in ()).throw( + UnknownException(message=original_msg) + ) + ) + install_operation_abandoned_handler(client, recorder=inst.recorder) + + with pytest.raises(UnknownException) as excinfo: + client._execute_and_fetch(_root_request("op-lit"), {}) + + assert str(excinfo.value) == original_msg + assert _events(inst, "reattach.operation_abandoned") == [] + + +def test_uninstall_client_hooks_removes_operation_abandoned_handler(): + client = _FullExecutionClient( + lambda _attempt, _req: _operation_abandoned_error() + ) + install_operation_abandoned_handler(client) + assert "_execute_and_fetch_as_iterator" in client.__dict__ + assert "_execute_and_fetch" in client.__dict__ + assert "_execute" in client.__dict__ + + uninstall_client_hooks(client) + + assert "_execute_and_fetch_as_iterator" not in client.__dict__ + assert "_execute_and_fetch" not in client.__dict__ + assert "_execute" not in client.__dict__ + with pytest.raises(UnknownException) as excinfo: + client._execute_and_fetch(_root_request("op-raw"), {}) + assert "Note: The client lost connectivity" not in str(excinfo.value) + + +@pytest.mark.parametrize( + "jvm_classes", + [ + ["org.apache.spark.SparkSQLException"], + [ + "org.apache.spark.SparkSQLException", + "org.apache.spark.SparkException", + ], + ], +) +def test_operation_abandoned_end_to_end_reattach_after_stream_rotation( + monkeypatch, + jvm_classes, +): + """End-to-end client + reattach iterator hitting OPERATION_ABANDONED.""" + import json + + from google.rpc import error_details_pb2, status_pb2 + import grpc + from pyspark.errors.exceptions.connect import SparkConnectGrpcException + from pyspark.sql.connect.client.core import ConfigResult + from pyspark.sql.connect.proto import ConfigResponse + + for key in list(os.environ): + if key.startswith("MANAGED_SPARK_CONNECT_"): + monkeypatch.delenv(key) + monkeypatch.setenv(session_mod._FORCE_PROGRESS_BAR_ENV, "1") + + class FakeTqdm: + + def __init__(self, total=None, **kwargs): + self.total = total + self.n = 0 + self.closed = False + + def reset(self, total=None): + self.total = total + + def refresh(self): + pass + + def close(self): + self.closed = True + + class AbandonedRpcError(grpc.RpcError, grpc.Call): + """gRPC status carrying Spark ErrorInfo for OPERATION_ABANDONED.""" + + def __init__(self, handle: str): + super().__init__() + self._handle = handle + info = error_details_pb2.ErrorInfo( + reason="org.apache.spark.SparkSQLException", + domain="spark.apache.org", + metadata={ + "errorClass": OPERATION_ABANDONED_CONDITION, + "sqlState": "HY000", + "messageParameters": json.dumps({"handle": handle}), + "classes": json.dumps(jvm_classes), + }, + ) + st = status_pb2.Status( + code=grpc.StatusCode.INTERNAL.value[0], + message=( + f"[{OPERATION_ABANDONED_CONDITION}] The handle {handle} is" + " invalid. Operation was considered abandoned because of" + " inactivity and removed. SQLSTATE: HY000" + ), + ) + st.details.add().Pack(info) + self._status_bytes = st.SerializeToString() + + def code(self): + return grpc.StatusCode.INTERNAL + + def details(self): + return ( + f"[{OPERATION_ABANDONED_CONDITION}] The handle {self._handle}" + " is invalid. Operation was considered abandoned because of" + " inactivity and removed. SQLSTATE: HY000" + ) + + def trailing_metadata(self): + return (("grpc-status-details-bin", self._status_bytes),) + + class RotationThenAbandonedStub: + """Yields 2/4 progress then rotates; ReattachExecute fails abandoned.""" + + def __init__(self): + self.execute_calls = 0 + self.reattach_calls = 0 + + def ExecutePlan(self, req, metadata=None, **kwargs): + self.execute_calls += 1 + resp = proto.ExecutePlanResponse( + session_id=req.session_id, + operation_id=req.operation_id, + response_id="r-progress-1", + ) + stage = resp.execution_progress.stages.add() + stage.stage_id = 1 + stage.num_tasks = 4 + stage.num_completed_tasks = 2 + stage.done = False + resp.execution_progress.num_inflight_tasks = 2 + yield resp + # Stream ends cleanly without ResultComplete (120 s server rotation). + + def ReattachExecute(self, req, metadata=None, **kwargs): + self.reattach_calls += 1 + raise AbandonedRpcError(req.operation_id) + + def FetchErrorDetails(self, req, metadata=None, **kwargs): + return proto.FetchErrorDetailsResponse() + + def ReleaseExecute(self, req, metadata=None, **kwargs): + return proto.ReleaseExecuteResponse() + + ManagedSparkSession._set_tqdm_factory(FakeTqdm) + ManagedSparkSession._execution_progress_bar.clear() + try: + with mock.patch( + "pyspark.sql.connect.client.SparkConnectClient.config", + return_value=ConfigResult.fromProto(ConfigResponse()), + ): + inst = Instrumentation([InMemorySink()]) + attached = ManagedSparkSession( + connection="sc://localhost:15002", + instrumentation=inst, + ) + try: + attached._register_progress_execution_handler() + stub = RotationThenAbandonedStub() + attached.client._stub = stub + attached.client._session_id = "sess-local" + install_release_coalescing(attached.client) + + cmd = proto.Command() + cmd.sql_command.sql = ( + "UPDATE analytics.order_items SET status = 'SHIPPED'" + ) + with pytest.raises(SparkConnectGrpcException) as excinfo: + attached.client.execute_command(cmd) + + exc = excinfo.value + assert stub.execute_calls == 1 + assert stub.reattach_calls == 1 + assert exc.getCondition() == OPERATION_ABANDONED_CONDITION + assert exc.getSqlState() == "HY000" + assert "Managed Spark session" in str(exc) + assert "Note: The client lost connectivity" in exc.args[0] + assert "Dataproc" not in str(exc) + assert str(exc).count("Note: The client lost connectivity") == 1 + + (abandoned_ev,) = _events(inst, "reattach.operation_abandoned") + assert abandoned_ev["attrs"]["plan_kind"] == "command" + assert abandoned_ev["attrs"]["command_kind"] == "sql_command" + + # Progress bar closed at 2/4 tasks with reason="query_end", + # not falsely completed to 4/4 ("done") by Progress.__exit__. + assert ManagedSparkSession.progress_bars() == {} + (bar_close,) = _events(inst, "progress.bar_close") + assert bar_close["attrs"]["completed_tasks"] == 2 + assert bar_close["attrs"]["total_tasks"] == 4 + assert bar_close["attrs"]["reason"] == "query_end" + finally: + attached.stop() + finally: + ManagedSparkSession._set_tqdm_factory(None) + ManagedSparkSession._execution_progress_bar.clear()