From 7edc81f9f9a70c9654a01e9463c2b91973ead248 Mon Sep 17 00:00:00 2001 From: Dave Borowitz Date: Wed, 11 Nov 2026 07:48:11 -0800 Subject: [PATCH] fix(session): preserve 3.0+ runtimeProfile versions and warn when overriding pre-3.0 Previously, Builder._get_session_config() unconditionally populated session_config.runtime_config.version with ManagedSparkSession._DEFAULT_RUNTIME_VERSION whenever it was unset on the builder, silently overriding any runtime version configured on a Runtime Profile / SessionTemplate. Extract SessionTemplate (Runtime Profile) resolution and execution-default precedence into google/cloud/managed_spark_connect/_session_templates.py: - Represent runtime version resolution with an immutable TemplateRuntimeResolution value object so Builder carries no cross-method mutable state. - If the template specifies a supported 3.0+ runtime_config.version, leave session_config.runtime_config.version unset on CreateSessionRequest so the template's version is preserved, while running Python UDF and PySpark version compatibility checks against the template's version. - If the template specifies a pre-3.0 runtime_config.version (e.g. 2.3), emit a UserWarning and override with the client's default runtime version (and append the override note if CreateSession subsequently fails with InvalidArgument, such as on a 2.3-only staging_bucket field). - If the template is readable and its runtime_config.version is empty, populate the client's default runtime version silently so the server does not fall back to an unsupported pre-3.0 runtime. - If GetSessionTemplate fails (e.g. missing dataproc.sessionTemplates.get), leave runtime_config.version unset rather than risking overriding a 3.0+ template, and inspect GetSession immediately after create_session() returns (with post-wait fallback) so pre-3.0 or unversioned (server-default 2.2) resolved sessions fail fast before VM startup while 3.0+ sessions still run client compatibility checks. - Include 'dataproc-spark-connect~=0.9.0' downgrade guidance in pre-3.0 runtime compatibility errors and warnings, and add unit tests in tests/unit/test_session_templates.py plus Builder wiring and live integration tests. --- .../_session_templates.py | 420 ++++++++++++++++++ google/cloud/managed_spark_connect/session.py | 285 +++++------- tests/integration/test_session.py | 256 +++++++++++ tests/unit/test_init.py | 7 +- tests/unit/test_session.py | 284 +++++++++++- tests/unit/test_session_templates.py | 355 +++++++++++++++ 6 files changed, 1438 insertions(+), 169 deletions(-) create mode 100644 google/cloud/managed_spark_connect/_session_templates.py create mode 100644 tests/unit/test_session_templates.py diff --git a/google/cloud/managed_spark_connect/_session_templates.py b/google/cloud/managed_spark_connect/_session_templates.py new file mode 100644 index 0000000..f5f1e28 --- /dev/null +++ b/google/cloud/managed_spark_connect/_session_templates.py @@ -0,0 +1,420 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Runtime Profile (Dataproc ``SessionTemplate``) resolution and defaults. + +Handles merging a Managed Spark Runtime Profile (``Session.session_template``) +with client runtime-version and execution-environment defaults without +overwriting fields configured on the profile, including when the caller lacks +``dataproc.sessionTemplates.get`` permission. +""" + +from collections.abc import Callable, Mapping +import dataclasses +import logging +import os +import re +from typing import Any, Optional + +from google.cloud.dataproc_v1 import ( + AuthenticationConfig, + GetSessionTemplateRequest, + Session, + SessionTemplate, +) +from google.cloud.managed_spark_connect.exceptions import ( + ManagedSparkConnectException, +) +from packaging import version + +logger = logging.getLogger(__name__) + +_SESSION_TEMPLATE_RESOURCE_PATTERN = re.compile( + r"^projects/[^/]+/(?:locations|regions)/[^/]+/sessionTemplates/[a-zA-Z0-9_-]+$" +) +_SESSION_TEMPLATE_ID_PATTERN = re.compile(r"^[a-zA-Z0-9_-]+$") + +_EXEC_ENV_VARS = ( + "MANAGED_SPARK_CONNECT_SERVICE_ACCOUNT", + "MANAGED_SPARK_CONNECT_AUTH_TYPE", + "MANAGED_SPARK_CONNECT_SUBNET", + "MANAGED_SPARK_CONNECT_TTL_SECONDS", + "MANAGED_SPARK_CONNECT_IDLE_TTL_SECONDS", +) + +_LEGACY_CLIENT_INSTALL_HINT = ( + "install 'dataproc-spark-connect~=0.9.0' in a Python 3.11 environment" +) + + +def is_valid_session_template_ref(template: str) -> bool: + """Return True if ``template`` is a valid template ID or resource name.""" + if "/" in template: + return bool(_SESSION_TEMPLATE_RESOURCE_PATTERN.match(template)) + return bool(_SESSION_TEMPLATE_ID_PATTERN.match(template)) + + +def qualify_session_template_name( + template_name: str, + project_id: Optional[str], + region: Optional[str], +) -> str: + """Qualify a bare template ID to a full ``sessionTemplates`` resource name.""" + if "/" not in template_name and project_id and region: + return ( + f"projects/{project_id}/locations/{region}" + f"/sessionTemplates/{template_name}" + ) + return template_name + + +def fetch_session_template( + get_client: Callable[[], Any], + template_name: str, + *, + project_id: Optional[str], + region: Optional[str], +) -> Optional[SessionTemplate]: + """Fetch a ``SessionTemplate`` best-effort, returning ``None`` on error.""" + if not template_name: + return None + qualified_name = qualify_session_template_name( + template_name, project_id, region + ) + try: + request = GetSessionTemplateRequest(name=qualified_name) + return get_client().get_session_template(request) + except Exception as e: + logger.debug( + "Could not read session template %s: %s", qualified_name, e + ) + return None + + +def is_below_min_runtime_version( + runtime_version: str, min_version: str +) -> bool: + """Return True when ``runtime_version`` parses below ``min_version``.""" + if not runtime_version: + return False + try: + return version.parse(runtime_version) < version.parse(min_version) + except version.InvalidVersion: + return False + + +def unsupported_runtime_version_message( + runtime_version: str, min_version: str +) -> str: + """Format the error message for an explicitly configured pre-minimum runtime.""" + return ( + f"Specified {runtime_version} Managed Spark Runtime " + "version is not supported by " + "google-cloud-spark-connect, which requires runtime " + f"version {min_version} or higher. To use runtime 2.3 " + f"(Spark 3.5), {_LEGACY_CLIENT_INSTALL_HINT}." + ) + + +@dataclasses.dataclass(frozen=True) +class TemplateRuntimeResolution: + """Outcome of resolving a session's runtime version with an optional Runtime Profile. + + Attributes: + request_version: Version to write onto ``Session.runtime_config.version``, + or ``""`` when leaving the Runtime Profile's version untouched on + the server. + effective_version: Version the session will run for client compatibility + checks, or ``""`` when an unreadable Runtime Profile defers version + discovery until the server resolves the session. + default_version: Client default runtime version used for fallbacks and + overrides. + min_version: Minimum Managed Spark runtime version supported by this + client. + template_name: Runtime Profile reference as configured on the session. + overridden_template_version: Pre-minimum Runtime Profile version that + was overridden with ``default_version``, or ``None``. + """ + + request_version: str + effective_version: str + default_version: str + min_version: str + template_name: str = "" + overridden_template_version: Optional[str] = None + + @property + def awaits_server_version(self) -> bool: + """True when the Runtime Profile could not be read pre-flight.""" + return not self.effective_version + + def override_warning(self) -> Optional[str]: + """Warning text when a pre-minimum Runtime Profile version was overridden.""" + if not self.overridden_template_version: + return None + return ( + f"Runtime Profile '{self.template_name}' specifies Managed Spark " + f"runtime version '{self.overridden_template_version}', which is " + "not supported by google-cloud-spark-connect (minimum " + f"{self.min_version}); overriding with runtime version " + f"'{self.default_version}'. Update the Runtime Profile to " + f"{self.min_version}+, or {_LEGACY_CLIENT_INSTALL_HINT} to use " + f"runtime version '{self.overridden_template_version}'." + ) + + def create_error_note(self) -> Optional[str]: + """Diagnostic note appended when CreateSession rejects an overridden profile.""" + if not self.overridden_template_version or not self.template_name: + return None + return ( + f"Note: Runtime Profile '{self.template_name}' specifies runtime" + f" version '{self.overridden_template_version}', which was" + " overridden because google-cloud-spark-connect requires" + f" runtime {self.min_version} or higher" + f" ({_LEGACY_CLIENT_INSTALL_HINT} to use runtime" + f" '{self.overridden_template_version}')." + ) + + def with_server_version( + self, resolved_version: str + ) -> "TemplateRuntimeResolution": + """Validate a server-resolved runtime version for an unreadable profile.""" + if not resolved_version: + return self + if is_below_min_runtime_version(resolved_version, self.min_version): + raise ManagedSparkConnectException( + f"Runtime Profile '{self.template_name}' resolved to Managed" + f" Spark runtime version '{resolved_version}', which is not" + " supported by google-cloud-spark-connect (minimum" + f" {self.min_version}). Pass .runtimeVersion('3.0') (or" + " '4.0') to override the Runtime Profile's runtime version," + f" or {_LEGACY_CLIENT_INSTALL_HINT} for runtime 2.3." + ) + return dataclasses.replace(self, effective_version=resolved_version) + + +def resolve_runtime_version( + *, + explicit_version: str, + template_name: str, + template_version: Optional[str], + default_version: str, + min_version: str, +) -> TemplateRuntimeResolution: + """Resolve requested and effective runtime versions without I/O. + + Args: + explicit_version: Version set on ``Session.runtime_config.version`` + (``""`` if unset). + template_name: Configured ``Session.session_template`` (``""`` if + unset). + template_version: ``SessionTemplate.runtime_config.version`` when the + template was read (``""`` if unset on the template), or ``None`` if + the template could not be read. Ignored when ``explicit_version`` is + set or ``template_name`` is empty. + default_version: Client default runtime version. + min_version: Minimum supported runtime version. + """ + if explicit_version: + return TemplateRuntimeResolution( + request_version=explicit_version, + effective_version=explicit_version, + default_version=default_version, + min_version=min_version, + template_name=template_name, + ) + if not template_name: + return TemplateRuntimeResolution( + request_version=default_version, + effective_version=default_version, + default_version=default_version, + min_version=min_version, + ) + if template_version is None: + return TemplateRuntimeResolution( + request_version="", + effective_version="", + default_version=default_version, + min_version=min_version, + template_name=template_name, + ) + if not template_version: + # Neither the session nor the template specifies a runtime version; + # populate the client default so the server does not fall back to an + # older non-Spark-Connect runtime. + return TemplateRuntimeResolution( + request_version=default_version, + effective_version=default_version, + default_version=default_version, + min_version=min_version, + template_name=template_name, + ) + if is_below_min_runtime_version(template_version, min_version): + return TemplateRuntimeResolution( + request_version=default_version, + effective_version=default_version, + default_version=default_version, + min_version=min_version, + template_name=template_name, + overridden_template_version=template_version, + ) + return TemplateRuntimeResolution( + request_version="", + effective_version=template_version, + default_version=default_version, + min_version=min_version, + template_name=template_name, + ) + + +def _apply_execution_env_defaults( + exec_config: Any, + template_exec_config: Optional[Any], + *, + template_unreadable: bool, + environ: Mapping[str, str], +) -> None: + """Apply ``MANAGED_SPARK_CONNECT_*`` execution defaults without clobbering a template.""" + # Auth and network check paired fields because setting either side defines + # the template's configuration. + template_has_auth = bool( + template_exec_config + and ( + template_exec_config.service_account + or ( + template_exec_config.authentication_config + and template_exec_config.authentication_config.user_workload_authentication_type + ) + ) + ) + template_has_network = bool( + template_exec_config + and ( + template_exec_config.subnetwork_uri + or template_exec_config.network_uri + ) + ) + template_has_ttl = bool(template_exec_config and template_exec_config.ttl) + template_has_idle_ttl = bool( + template_exec_config and template_exec_config.idle_ttl + ) + + if ( + not template_unreadable + and not template_has_auth + and not exec_config.service_account + and "MANAGED_SPARK_CONNECT_SERVICE_ACCOUNT" in environ + ): + exec_config.service_account = environ[ + "MANAGED_SPARK_CONNECT_SERVICE_ACCOUNT" + ] + + if exec_config.service_account: + exec_config.authentication_config.user_workload_authentication_type = ( + AuthenticationConfig.AuthenticationType.SERVICE_ACCOUNT + ) + elif ( + not template_unreadable + and not template_has_auth + and not exec_config.authentication_config.user_workload_authentication_type + and "MANAGED_SPARK_CONNECT_AUTH_TYPE" in environ + ): + exec_config.authentication_config.user_workload_authentication_type = ( + AuthenticationConfig.AuthenticationType[ + environ["MANAGED_SPARK_CONNECT_AUTH_TYPE"] + ] + ) + + if ( + not template_unreadable + and not template_has_network + and not exec_config.subnetwork_uri + and not exec_config.network_uri + and "MANAGED_SPARK_CONNECT_SUBNET" in environ + ): + exec_config.subnetwork_uri = environ["MANAGED_SPARK_CONNECT_SUBNET"] + + if ( + not template_unreadable + and not template_has_ttl + and not exec_config.ttl + and "MANAGED_SPARK_CONNECT_TTL_SECONDS" in environ + ): + exec_config.ttl = { + "seconds": int(environ["MANAGED_SPARK_CONNECT_TTL_SECONDS"]) + } + + if ( + not template_unreadable + and not template_has_idle_ttl + and not exec_config.idle_ttl + and "MANAGED_SPARK_CONNECT_IDLE_TTL_SECONDS" in environ + ): + exec_config.idle_ttl = { + "seconds": int(environ["MANAGED_SPARK_CONNECT_IDLE_TTL_SECONDS"]) + } + + +def apply_session_template_defaults( + session_config: Session, + *, + get_template: Callable[[str], Optional[SessionTemplate]], + default_version: str, + min_version: str, + environ: Optional[Mapping[str, str]] = None, +) -> TemplateRuntimeResolution: + """Resolve runtime version and execution defaults onto ``session_config``.""" + env = os.environ if environ is None else environ + session_template: Optional[SessionTemplate] = None + template_exec_config = None + template_unreadable = False + + if session_config.session_template and ( + not session_config.runtime_config.version + or any(var in env for var in _EXEC_ENV_VARS) + ): + session_template = get_template(session_config.session_template) + if session_template is not None: + template_exec_config = ( + session_template.environment_config.execution_config + ) + else: + # CreateSession does not require sessionTemplates.get. If the + # template cannot be read, continue without applying env vars or + # client default runtime version rather than risking overriding + # template settings. + template_unreadable = True + + resolution = resolve_runtime_version( + explicit_version=session_config.runtime_config.version, + template_name=session_config.session_template, + template_version=( + None + if session_template is None + else session_template.runtime_config.version + ), + default_version=default_version, + min_version=min_version, + ) + # Avoid assigning "" when leaving runtime_config.version unset, which would + # materialize an empty runtime_config submessage on the Session proto. + if resolution.request_version: + session_config.runtime_config.version = resolution.request_version + + _apply_execution_env_defaults( + session_config.environment_config.execution_config, + template_exec_config, + template_unreadable=template_unreadable, + environ=env, + ) + return resolution diff --git a/google/cloud/managed_spark_connect/session.py b/google/cloud/managed_spark_connect/session.py index 6a6e73c..645cdc6 100644 --- a/google/cloud/managed_spark_connect/session.py +++ b/google/cloud/managed_spark_connect/session.py @@ -95,7 +95,6 @@ install_pypi_packages, ) from google.cloud.dataproc_v1 import ( - AuthenticationConfig, CreateSessionRequest, DeleteSessionRequest, GetSessionRequest, @@ -106,6 +105,7 @@ from google.cloud.dataproc_v1.types import sessions from google.cloud.managed_spark_connect import _logging from google.cloud.managed_spark_connect import _runtimes +from google.cloud.managed_spark_connect import _session_templates from google.cloud.managed_spark_connect import environment from google.cloud.managed_spark_connect import ide from pyspark.sql.connect.session import SparkSession @@ -431,18 +431,6 @@ def _is_valid_session_id(session_id: str) -> bool: return bool(re.match(pattern, session_id)) -_SESSION_TEMPLATE_RESOURCE_PATTERN = re.compile( - r"^projects/[^/]+/(?:locations|regions)/[^/]+/sessionTemplates/[a-zA-Z0-9_-]+$" -) -_SESSION_TEMPLATE_ID_PATTERN = re.compile(r"^[a-zA-Z0-9_-]+$") - - -def _is_valid_session_template_ref(template: str) -> bool: - if "/" in template: - return bool(_SESSION_TEMPLATE_RESOURCE_PATTERN.match(template)) - return bool(_SESSION_TEMPLATE_ID_PATTERN.match(template)) - - def _extract_api_error_details( exc: BaseException, ) -> Tuple[List[str], List[str]]: @@ -477,6 +465,7 @@ def _format_create_session_error( region: Optional[str], api_endpoint: Optional[str], session_template: Optional[str] = None, + runtime_override_note: Optional[str] = None, ) -> str: prefix = "Error while creating Managed Spark Session" raw_msg = (getattr(exc, "message", None) or str(exc)).strip() @@ -524,7 +513,9 @@ def _format_create_session_error( if ( isinstance(exc, InvalidArgument) and session_template - and not _is_valid_session_template_ref(session_template) + and not _session_templates.is_valid_session_template_ref( + session_template + ) and "session_template" not in base_msg and "Session Template" not in base_msg and "session template" not in base_msg @@ -553,6 +544,9 @@ def _format_create_session_error( " one with .subnetwork('') or MANAGED_SPARK_CONNECT_SUBNET." ) + if isinstance(exc, InvalidArgument) and runtime_override_note: + extra_parts.append(runtime_override_note) + if base_msg and extra_parts and base_msg[-1] not in ".!?:;": base_msg = f"{base_msg}." parts = [p for p in (base_msg, *extra_parts) if p] @@ -968,10 +962,14 @@ def __create(self) -> "ManagedSparkSession": "ManagedSparkSession does not support connecting to an existing Spark Connect remote server" ) - session_config: Session = self._get_session_config() + session_config, runtime_resolution = ( + self._prepare_session_config() + ) # Check runtime version compatibility before creating session - self._check_runtime_compatibility(session_config) + self._check_runtime_compatibility( + session_config, runtime_resolution.effective_version + ) # Use custom session ID if provided, otherwise generate one session_id = ( @@ -1040,6 +1038,14 @@ def create_session_pbar(): session_request ) ) + if runtime_resolution.awaits_server_version: + runtime_resolution = ( + self._resolve_created_session_template_runtime( + session_config.name, + session_id, + runtime_resolution, + ) + ) self._display_session_link_on_creation(session_id) self._display_view_session_details_button(session_id) if create_session_pbar_thread is not None: @@ -1057,6 +1063,14 @@ def create_session_pbar(): ) ) wait_span.set(**_session_state_attrs(session_response)) + if runtime_resolution.awaits_server_version: + runtime_resolution = ( + self._apply_server_template_runtime_version( + session_response.runtime_config.version, + session_id, + runtime_resolution, + ) + ) file_path = ( ManagedSparkSession._get_active_session_file_path() ) @@ -1091,6 +1105,7 @@ def create_session_pbar(): self._client_options, "api_endpoint" ), session_config.session_template, + runtime_resolution.create_error_note(), ) ) from e except DefaultCredentialsError as e: @@ -1099,6 +1114,10 @@ def create_session_pbar(): raise ManagedSparkConnectException( "Credentials error while creating Managed Spark Session (see https://docs.cloud.google.com/docs/authentication/provide-credentials-adc for more info)" ) from e + except ManagedSparkConnectException: + ManagedSparkSession._active_s8s_session_id = None + ManagedSparkSession._active_session_uses_custom_id = False + raise except Exception as e: ManagedSparkSession._active_s8s_session_id = None ManagedSparkSession._active_session_uses_custom_id = False @@ -1601,139 +1620,38 @@ def _handle_custom_session_id(self): ManagedSparkSession._active_s8s_session_id = None ManagedSparkSession._active_session_uses_custom_id = False - def _get_session_config(self): + def _get_session_config(self) -> Session: + return self._prepare_session_config(_warning_stacklevel=3)[0] + + def _prepare_session_config( + self, *, _warning_stacklevel: int = 2 + ) -> Tuple[Session, _session_templates.TemplateRuntimeResolution]: + import warnings + # Use the property to ensure we always have a config session_config = self.session_config for k, v in self._options.items(): session_config.runtime_config.properties[k] = v session_config.spark_connect_session = sessions.SparkConnectConfig() - if not session_config.runtime_config.version: - session_config.runtime_config.version = ( - ManagedSparkSession._DEFAULT_RUNTIME_VERSION - ) + + resolution = _session_templates.apply_session_template_defaults( + session_config, + get_template=self._get_session_template, + default_version=ManagedSparkSession._DEFAULT_RUNTIME_VERSION, + min_version=ManagedSparkSession._MIN_RUNTIME_VERSION, + ) + override_warning = resolution.override_warning() + if override_warning: + warnings.warn(override_warning, stacklevel=_warning_stacklevel) # Check for Python version mismatch with runtime for UDF compatibility self._check_python_version_compatibility( - session_config.runtime_config.version + resolution.effective_version ) self._check_pyspark_version_compatibility( - session_config.runtime_config.version + resolution.effective_version ) - # Use local variable to improve readability of deeply nested attribute access - exec_config = session_config.environment_config.execution_config - template_exec_config = None - template_unreadable = False - exec_env_vars = ( - "MANAGED_SPARK_CONNECT_SERVICE_ACCOUNT", - "MANAGED_SPARK_CONNECT_AUTH_TYPE", - "MANAGED_SPARK_CONNECT_SUBNET", - "MANAGED_SPARK_CONNECT_TTL_SECONDS", - "MANAGED_SPARK_CONNECT_IDLE_TTL_SECONDS", - ) - if session_config.session_template and any( - var in os.environ for var in exec_env_vars - ): - session_template = self._get_session_template( - session_config.session_template - ) - if session_template is not None: - template_exec_config = ( - session_template.environment_config.execution_config - ) - else: - # CreateSession does not require sessionTemplates.get. If - # the template cannot be read, continue without applying env - # vars rather than risking overriding template settings. - template_unreadable = True - - # Track which execution settings are already configured on the - # template. Auth and network check paired fields because setting - # either side defines the template's configuration - template_has_auth = bool( - template_exec_config - and ( - template_exec_config.service_account - or ( - template_exec_config.authentication_config - and template_exec_config.authentication_config.user_workload_authentication_type - ) - ) - ) - template_has_network = bool( - template_exec_config - and ( - template_exec_config.subnetwork_uri - or template_exec_config.network_uri - ) - ) - template_has_ttl = bool( - template_exec_config and template_exec_config.ttl - ) - template_has_idle_ttl = bool( - template_exec_config and template_exec_config.idle_ttl - ) - - # Set service account from environment if not already set on the - # session or session template - if ( - not template_unreadable - and not template_has_auth - and not exec_config.service_account - and "MANAGED_SPARK_CONNECT_SERVICE_ACCOUNT" in os.environ - ): - exec_config.service_account = os.getenv( - "MANAGED_SPARK_CONNECT_SERVICE_ACCOUNT" - ) - - # Auto-set authentication type to SERVICE_ACCOUNT when service account is provided - if exec_config.service_account: - # When service account is provided, explicitly set auth type to SERVICE_ACCOUNT - exec_config.authentication_config.user_workload_authentication_type = ( - AuthenticationConfig.AuthenticationType.SERVICE_ACCOUNT - ) - elif ( - not template_unreadable - and not template_has_auth - and not exec_config.authentication_config.user_workload_authentication_type - and "MANAGED_SPARK_CONNECT_AUTH_TYPE" in os.environ - ): - # Only set auth type from environment if no service account is present - exec_config.authentication_config.user_workload_authentication_type = AuthenticationConfig.AuthenticationType[ - os.getenv("MANAGED_SPARK_CONNECT_AUTH_TYPE") - ] - if ( - not template_unreadable - and not template_has_network - and not exec_config.subnetwork_uri - and not exec_config.network_uri - and "MANAGED_SPARK_CONNECT_SUBNET" in os.environ - ): - exec_config.subnetwork_uri = os.getenv( - "MANAGED_SPARK_CONNECT_SUBNET" - ) - if ( - not template_unreadable - and not template_has_ttl - and not exec_config.ttl - and "MANAGED_SPARK_CONNECT_TTL_SECONDS" in os.environ - ): - exec_config.ttl = { - "seconds": int( - os.getenv("MANAGED_SPARK_CONNECT_TTL_SECONDS") - ) - } - if ( - not template_unreadable - and not template_has_idle_ttl - and not exec_config.idle_ttl - and "MANAGED_SPARK_CONNECT_IDLE_TTL_SECONDS" in os.environ - ): - exec_config.idle_ttl = { - "seconds": int( - os.getenv("MANAGED_SPARK_CONNECT_IDLE_TTL_SECONDS") - ) - } client_environment = environment.get_client_environment_label() session_config.labels["dataproc-session-client"] = ( client_environment @@ -1777,30 +1695,16 @@ def _get_session_config(self): f" {default_datasource}. Supported value is 'bigquery'." ) - return session_config + return session_config, resolution def _get_session_template(self, template_name: str): """Fetch the SessionTemplate best-effort, returning None on error.""" - from google.cloud.dataproc_v1 import GetSessionTemplateRequest - - if not template_name: - return None - if "/" not in template_name and self._project_id and self._region: - template_name = ( - f"projects/{self._project_id}/locations/{self._region}" - f"/sessionTemplates/{template_name}" - ) - try: - get_session_template_request = GetSessionTemplateRequest() - get_session_template_request.name = template_name - return self.session_template_controller_client.get_session_template( - get_session_template_request - ) - except Exception as e: - logger.debug( - f"Could not read session template {template_name}: {e}" - ) - return None + return _session_templates.fetch_session_template( + lambda: self.session_template_controller_client, + template_name, + project_id=self._project_id, + region=self._region, + ) def _check_python_version_compatibility(self, runtime_version): """Check if client Python version matches server Python version for UDF compatibility.""" @@ -1862,7 +1766,59 @@ def _check_pyspark_version_compatibility(self, runtime_version): stacklevel=3, ) - def _check_runtime_compatibility(self, session_config): + def _resolve_created_session_template_runtime( + self, + session_name: str, + session_id: str, + resolution: _session_templates.TemplateRuntimeResolution, + ) -> _session_templates.TemplateRuntimeResolution: + """Check the server-resolved runtime version right after CreateSession when GetSessionTemplate failed.""" + try: + created_session = self.session_controller_client.get_session( + GetSessionRequest(name=session_name) + ) + except Exception as e: + logger.debug( + f"Could not inspect created session {session_name}: {e}" + ) + return resolution + return self._apply_server_template_runtime_version( + created_session.runtime_config.version, session_id, resolution + ) + + def _apply_server_template_runtime_version( + self, + resolved_version: str, + session_id: str, + resolution: _session_templates.TemplateRuntimeResolution, + ) -> _session_templates.TemplateRuntimeResolution: + if not resolved_version: + return resolution + try: + updated = resolution.with_server_version(resolved_version) + except ManagedSparkConnectException: + try: + terminate_s8s_session( + self._project_id, + self._region, + session_id, + self._client_options, + recorder=self._recorder, + ) + except Exception: + logger.debug( + "Failed to terminate incompatible session %s", + session_id, + exc_info=True, + ) + raise + self._check_python_version_compatibility(updated.effective_version) + self._check_pyspark_version_compatibility(updated.effective_version) + return updated + + def _check_runtime_compatibility( + self, session_config, runtime_version: Optional[str] = None + ): """Check if runtime version 3.0 client is compatible with older runtime versions. Runtime version 3.0 clients do not support older runtime versions (pre-3.0). @@ -1870,11 +1826,14 @@ def _check_runtime_compatibility(self, session_config): Args: session_config: The Session configuration containing runtime version + runtime_version: Optional effective runtime version (for example, + resolved from a Runtime Profile when unset on session_config) Raises: ManagedSparkConnectException: If server is using pre-3.0 runtime version """ - runtime_version = session_config.runtime_config.version + if runtime_version is None: + runtime_version = session_config.runtime_config.version if not runtime_version: return @@ -1890,8 +1849,10 @@ def _check_runtime_compatibility(self, session_config): if server_version < min_version: raise ManagedSparkConnectException( - f"Specified {runtime_version} Managed Spark Runtime version is not supported, " - f"use {ManagedSparkSession._MIN_RUNTIME_VERSION} version or higher." + _session_templates.unsupported_runtime_version_message( + runtime_version, + ManagedSparkSession._MIN_RUNTIME_VERSION, + ) ) except version.InvalidVersion: # If we can't parse the version, log a warning but continue diff --git a/tests/integration/test_session.py b/tests/integration/test_session.py index 00d6323..ddfe404 100644 --- a/tests/integration/test_session.py +++ b/tests/integration/test_session.py @@ -11,8 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import contextlib import datetime +import os import uuid +import warnings from google.api_core.exceptions import ( InvalidArgument, @@ -285,6 +288,259 @@ def test_create_spark_session_with_session_template_and_user_provided_session_co assert ManagedSparkSession._active_s8s_session_uuid is None +@contextlib.contextmanager +def _temporary_session_template( + test_project, + test_region, + session_template_controller_client, + *, + runtime_version="", + subnetwork_uri="", + staging_bucket="", + service_account="", + single_node=True, +): + template_id = f"sc-it-tmpl-{uuid.uuid4().hex[:12]}" + template_name = ( + f"projects/{test_project}/locations/{test_region}/sessionTemplates/" + f"{template_id}" + ) + session_template = SessionTemplate(name=template_name) + if runtime_version: + session_template.runtime_config.version = runtime_version + if single_node: + session_template.runtime_config.properties["spark.master"] = "local" + exec_cfg = session_template.environment_config.execution_config + exec_cfg.idle_ttl = {"seconds": 600} + exec_cfg.ttl = {"seconds": 600} + if subnetwork_uri: + exec_cfg.subnetwork_uri = subnetwork_uri + if staging_bucket: + exec_cfg.staging_bucket = staging_bucket + if service_account: + exec_cfg.service_account = service_account + create_request = CreateSessionTemplateRequest( + parent=f"projects/{test_project}/locations/{test_region}", + session_template=session_template, + ) + session_template_controller_client.create_session_template(create_request) + try: + yield template_id, template_name + finally: + delete_request = DeleteSessionTemplateRequest(name=template_name) + session_template_controller_client.delete_session_template( + delete_request + ) + + +def test_runtime_profile_preserves_non_default_supported_version( + test_project, + test_region, + test_subnetwork_uri, + session_controller_client, + session_template_controller_client, + isolated_session_state, +): + """A 3.0+ Runtime Profile's runtime_config.version is preserved rather than overridden by the client default.""" + target_version = ( + "4.0" + if ManagedSparkSession._DEFAULT_RUNTIME_VERSION.startswith("3.") + else "3.0" + ) + with _temporary_session_template( + test_project, + test_region, + session_template_controller_client, + runtime_version=target_version, + subnetwork_uri=test_subnetwork_uri, + ) as (template_id, template_name): + connect_session = None + try: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + # Pass a bare profile ID to exercise short-name qualification in + # both GetSessionTemplate and CreateSession. + connect_session = ( + ManagedSparkSession.builder.projectId(test_project) + .location(test_region) + .runtimeProfile(template_id) + .config("spark.executor.cores", "4") + .config("spark.master", "local") + .getOrCreate() + ) + session_name = ( + f"projects/{test_project}/locations/{test_region}/sessions/" + f"{ManagedSparkSession._active_s8s_session_id}" + ) + session = session_controller_client.get_session( + GetSessionRequest(name=session_name) + ) + assert session.state == Session.State.ACTIVE + assert session.session_template == template_name + assert session.runtime_config.version.startswith(target_version) + + from pyspark.version import __version__ as pyspark_version + + if target_version == "3.0" and pyspark_version.startswith("4.2."): + assert any( + "Managed Spark runtime 3.0 runs Spark 4.0" in str(w.message) + for w in caught + ) + else: + assert connect_session.range(5).count() == 5 + finally: + if connect_session is not None: + connect_session.stop() + + +def test_runtime_profile_pre_3_0_warns_and_overrides_to_default( + test_project, + test_region, + test_subnetwork_uri, + session_controller_client, + session_template_controller_client, + isolated_session_state, +): + """A 2.3 Runtime Profile without 2.3-only fields warns and starts on the client default runtime.""" + with _temporary_session_template( + test_project, + test_region, + session_template_controller_client, + runtime_version="2.3", + subnetwork_uri=test_subnetwork_uri, + ) as (_, template_name): + connect_session = None + try: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + connect_session = ( + ManagedSparkSession.builder.projectId(test_project) + .location(test_region) + .runtimeProfile(template_name) + .config("spark.executor.cores", "4") + .config("spark.master", "local") + .getOrCreate() + ) + override_warnings = [ + w for w in caught if "Runtime Profile" in str(w.message) + ] + assert len(override_warnings) == 1 + assert "'2.3'" in str(override_warnings[0].message) + assert "dataproc-spark-connect~=0.9.0" in str( + override_warnings[0].message + ) + + session_name = ( + f"projects/{test_project}/locations/{test_region}/sessions/" + f"{ManagedSparkSession._active_s8s_session_id}" + ) + session = session_controller_client.get_session( + GetSessionRequest(name=session_name) + ) + assert session.state == Session.State.ACTIVE + assert session.runtime_config.version.startswith( + ManagedSparkSession._DEFAULT_RUNTIME_VERSION + ) + assert connect_session.range(5).count() == 5 + finally: + if connect_session is not None: + connect_session.stop() + + +def test_runtime_profile_pre_3_0_with_staging_bucket_appends_override_note_on_error( + test_project, + test_region, + test_subnetwork_uri, + session_template_controller_client, + isolated_session_state, +): + """When overriding a 2.3 Runtime Profile causes CreateSession to reject staging_bucket, the error explains the override.""" + with _temporary_session_template( + test_project, + test_region, + session_template_controller_client, + runtime_version="2.3", + subnetwork_uri=test_subnetwork_uri, + staging_bucket=f"sc-it-staging-{uuid.uuid4().hex[:12]}", + ) as (_, template_name): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + with pytest.raises(ManagedSparkConnectException) as exc_info: + ( + ManagedSparkSession.builder.projectId(test_project) + .location(test_region) + .runtimeProfile(template_name) + .getOrCreate() + ) + + assert any("Runtime Profile" in str(w.message) for w in caught) + err = str(exc_info.value) + assert "staging_bucket" in err + assert f"Note: Runtime Profile '{template_name}'" in err + assert "specifies runtime version '2.3'" in err + assert "dataproc-spark-connect~=0.9.0" in err + + +def test_unreadable_pre_3_0_and_unversioned_runtime_profiles_fail_fast( + test_project, + test_region, + test_subnetwork_uri, + session_controller_client, + session_template_controller_client, + isolated_session_state, + monkeypatch, +): + """When GetSessionTemplate is unavailable, < 3.0 and unversioned (server-default 2.2) profiles fail fast before waiting for startup.""" + monkeypatch.setattr( + ManagedSparkSession.Builder, + "_get_session_template", + lambda self, name: None, + ) + + for configured_version, expected_resolved_prefix in ( + ("2.3", "2.3"), + ("", "2.2"), + ): + with _temporary_session_template( + test_project, + test_region, + session_template_controller_client, + runtime_version=configured_version, + subnetwork_uri=test_subnetwork_uri, + service_account=os.getenv( + "MANAGED_SPARK_CONNECT_SERVICE_ACCOUNT", "" + ), + single_node=False, + ) as (_, template_name): + custom_id = f"sc-it-unreadable-{uuid.uuid4().hex[:8]}" + session_name = ( + f"projects/{test_project}/locations/{test_region}/sessions/" + f"{custom_id}" + ) + with pytest.raises(ManagedSparkConnectException) as exc_info: + ( + ManagedSparkSession.builder.projectId(test_project) + .location(test_region) + .sessionId(custom_id) + .runtimeProfile(template_name) + .getOrCreate() + ) + err = str(exc_info.value) + assert ( + f"resolved to Managed Spark runtime version '{expected_resolved_prefix}" + in err + ) + assert ".runtimeVersion('3.0')" in err + assert "dataproc-spark-connect~=0.9.0" in err + + created = session_controller_client.get_session( + GetSessionRequest(name=session_name) + ) + assert created.runtime_config.version.startswith( + expected_resolved_prefix + ) + + @pytest.mark.skip( reason="Skipping PyPI package installation test since it's not supported yet" ) diff --git a/tests/unit/test_init.py b/tests/unit/test_init.py index ef23581..83779a2 100644 --- a/tests/unit/test_init.py +++ b/tests/unit/test_init.py @@ -221,8 +221,11 @@ def test_older_runtimes_raise_exception(self): min_version = ManagedSparkSession._MIN_RUNTIME_VERSION expected_message = ( - f"Specified {version} Managed Spark Runtime version is not supported, " - f"use {min_version} version or higher." + f"Specified {version} Managed Spark Runtime version is not " + "supported by google-cloud-spark-connect, which requires " + f"runtime version {min_version} or higher. To use runtime " + "2.3 (Spark 3.5), install 'dataproc-spark-connect~=0.9.0' " + "in a Python 3.11 environment." ) self.assertEqual(str(context.exception), expected_message) diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 937b992..53edb3f 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -17,6 +17,7 @@ import threading import time import unittest +import warnings from google.api_core.exceptions import ( Aborted, @@ -550,6 +551,7 @@ def test_create_session_with_env_vars_config( get_session_request ) + @mock.patch("google.cloud.dataproc_v1.SessionTemplateControllerClient") @mock.patch("google.auth.default") @mock.patch("google.cloud.dataproc_v1.SessionControllerClient") @mock.patch("pyspark.sql.connect.client.SparkConnectClient.config") @@ -566,12 +568,20 @@ def test_create_session_with_session_template( mock_client_config, mock_session_controller_client, mock_credentials, + mock_session_template_controller_client, ): session = None mock_is_s8s_session_active.return_value = True mock_session_controller_client_instance = ( mock_session_controller_client.return_value ) + template = SessionTemplate( + name="projects/test-project/locations/test-region/sessionTemplates/test_template" + ) + template.runtime_config.version = "4.0" + mock_session_template_controller_client.return_value.get_session_template.return_value = ( + template + ) mock_session_id.return_value = "sc-20240702-103952-abcdef" mock_client_config.return_value = ConfigResult.fromProto( ConfigResponse() @@ -595,9 +605,6 @@ def test_create_session_with_session_template( "projects/test-project/locations/test-region" ) create_session_request.session.name = "projects/test-project/locations/test-region/sessions/sc-20240702-103952-abcdef" - create_session_request.session.runtime_config.version = ( - self._default_runtime_version - ) create_session_request.session.spark_connect_session = ( SparkConnectConfig() ) @@ -632,6 +639,7 @@ def test_create_session_with_session_template( get_session_request ) + @mock.patch("google.cloud.dataproc_v1.SessionTemplateControllerClient") @mock.patch("google.auth.default") @mock.patch("google.cloud.dataproc_v1.SessionControllerClient") @mock.patch("pyspark.sql.connect.client.SparkConnectClient.config") @@ -648,12 +656,16 @@ def test_create_session_with_user_provided_session_config_and_session_template( mock_client_config, mock_session_controller_client, mock_credentials, + mock_session_template_controller_client, ): session = None mock_is_s8s_session_active.return_value = True mock_session_controller_client_instance = ( mock_session_controller_client.return_value ) + mock_session_template_controller_client.return_value.get_session_template.return_value = ( + SessionTemplate() + ) mock_session_id.return_value = "sc-20240702-103952-abcdef" mock_client_config.return_value = ConfigResult.fromProto( ConfigResponse() @@ -1090,16 +1102,21 @@ def test_create_spark_session_error_detail_fallbacks_and_field_violations( "session.session_template: Template name is malformed.", ) + @mock.patch("google.cloud.dataproc_v1.SessionTemplateControllerClient") @mock.patch("google.auth.default") @mock.patch("google.cloud.dataproc_v1.SessionControllerClient") def test_create_spark_session_with_typo_session_template( self, mock_session_controller_client, mock_credentials, + mock_session_template_controller_client, ): cred = mock.MagicMock() cred.token = "token" mock_credentials.return_value = (cred, "") + mock_session_template_controller_client.return_value.get_session_template.side_effect = NotFound( + "Dataproc Session Template 'interactive-analysis-typo' does not exist" + ) mock_session_controller_client.return_value.create_session.side_effect = InvalidArgument( "Dataproc Session Template 'interactive-analysis-typo' does not exist" ) @@ -1124,16 +1141,21 @@ def test_create_spark_session_with_typo_session_template( ) self.assertIsInstance(ctx.exception.__cause__, InvalidArgument) + @mock.patch("google.cloud.dataproc_v1.SessionTemplateControllerClient") @mock.patch("google.auth.default") @mock.patch("google.cloud.dataproc_v1.SessionControllerClient") def test_create_spark_session_with_malformed_session_template_uri( self, mock_session_controller_client, mock_credentials, + mock_session_template_controller_client, ): cred = mock.MagicMock() cred.token = "token" mock_credentials.return_value = (cred, "") + mock_session_template_controller_client.return_value.get_session_template.side_effect = InvalidArgument( + "Request contains an invalid argument." + ) mock_session_controller_client.return_value.create_session.side_effect = InvalidArgument( "Request contains an invalid argument." ) @@ -3004,6 +3026,7 @@ def test_builder_pattern_ttl_with_timedelta( ) self.stopSession(mock_session_controller_client_instance, session) + @mock.patch("google.cloud.dataproc_v1.SessionTemplateControllerClient") @mock.patch("google.auth.default") @mock.patch("google.cloud.dataproc_v1.SessionControllerClient") @mock.patch("pyspark.sql.connect.client.SparkConnectClient.config") @@ -3020,8 +3043,12 @@ def test_builder_pattern_session_template_and_labels( mock_client_config, mock_session_controller_client, mock_credentials, + mock_session_template_controller_client, ): session = None + mock_session_template_controller_client.return_value.get_session_template.return_value = ( + SessionTemplate() + ) mock_session_controller_client_instance = ( self._setup_session_creation_mocks( mock_is_s8s_session_active, @@ -4158,18 +4185,265 @@ def test_unreadable_template_skips_env_vars(self, mock_template_client): self.assertFalse(exec_config.idle_ttl) @mock.patch("google.cloud.dataproc_v1.SessionTemplateControllerClient") - def test_no_template_lookup_without_exec_env_vars( + def test_no_template_lookup_when_runtime_version_explicit_and_no_exec_env_vars( self, mock_template_client ): - """No GetSessionTemplate call is made when no execution environment variables are set.""" + """No GetSessionTemplate call is made when runtimeVersion is set and no execution env vars are set.""" session_config = ( self._builder() .sessionTemplate(self._TEMPLATE) + .runtimeVersion("3.0") ._get_session_config() ) mock_template_client.return_value.get_session_template.assert_not_called() self.assertEqual(session_config.session_template, self._TEMPLATE) + self.assertEqual(session_config.runtime_config.version, "3.0") + + @mock.patch("google.cloud.dataproc_v1.SessionTemplateControllerClient") + def test_template_runtime_version_preserved(self, mock_template_client): + """A Runtime Profile's runtime_config.version is not overwritten by the client default.""" + template = SessionTemplate(name=self._TEMPLATE) + template.runtime_config.version = "4.0" + mock_template_client.return_value.get_session_template.return_value = ( + template + ) + + session_config, resolution = ( + self._builder() + .runtimeProfile(self._TEMPLATE) + ._prepare_session_config() + ) + + self.assertEqual(session_config.runtime_config.version, "") + self.assertEqual(resolution.effective_version, "4.0") + + @mock.patch("google.cloud.dataproc_v1.SessionTemplateControllerClient") + def test_template_empty_runtime_version_defaults_to_client_runtime( + self, mock_template_client + ): + """When neither builder nor template sets runtime_config.version, the client default is applied.""" + template = SessionTemplate(name=self._TEMPLATE) + mock_template_client.return_value.get_session_template.return_value = ( + template + ) + + session_config, resolution = ( + self._builder() + .runtimeProfile(self._TEMPLATE) + ._prepare_session_config() + ) + + self.assertEqual( + session_config.runtime_config.version, + ManagedSparkSession._DEFAULT_RUNTIME_VERSION, + ) + self.assertEqual( + resolution.effective_version, + ManagedSparkSession._DEFAULT_RUNTIME_VERSION, + ) + + @mock.patch("google.cloud.dataproc_v1.SessionTemplateControllerClient") + def test_unreadable_template_leaves_runtime_version_unset( + self, mock_template_client + ): + """When GetSessionTemplate fails, runtime_config.version is left unset rather than overriding the template.""" + mock_template_client.return_value.get_session_template.side_effect = ( + PermissionDenied("caller lacks dataproc.sessionTemplates.get") + ) + + session_config, resolution = ( + self._builder() + .runtimeProfile(self._TEMPLATE) + ._prepare_session_config() + ) + + self.assertEqual(session_config.runtime_config.version, "") + self.assertTrue(resolution.awaits_server_version) + + @mock.patch("google.cloud.dataproc_v1.SessionTemplateControllerClient") + def test_template_pre_3_0_runtime_version_warns_and_overrides_with_default( + self, mock_template_client + ): + """A Runtime Profile with a pre-3.0 runtime warns and overrides with the client default.""" + template = SessionTemplate(name=self._TEMPLATE) + template.runtime_config.version = "2.3" + mock_template_client.return_value.get_session_template.return_value = ( + template + ) + + builder = self._builder().runtimeProfile(self._TEMPLATE) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + session_config = builder._get_session_config() + + default_version = ManagedSparkSession._DEFAULT_RUNTIME_VERSION + self.assertEqual(session_config.runtime_config.version, default_version) + override_warnings = [ + w for w in caught if "Runtime Profile" in str(w.message) + ] + self.assertEqual(len(override_warnings), 1) + self.assertEqual(override_warnings[0].filename, __file__) + msg = str(override_warnings[0].message) + self.assertIn("'2.3'", msg) + self.assertIn( + f"overriding with runtime version '{default_version}'", msg + ) + self.assertIn("dataproc-spark-connect~=0.9.0", msg) + + @mock.patch("google.cloud.dataproc_v1.SessionTemplateControllerClient") + @mock.patch("google.auth.default") + @mock.patch("google.cloud.dataproc_v1.SessionControllerClient") + def test_template_pre_3_0_override_appends_note_when_create_session_fails( + self, + mock_session_controller_client, + mock_credentials, + mock_template_client, + ): + """When CreateSession fails on a 2.3-only template field, the error includes the override note.""" + cred = mock.MagicMock() + cred.token = "token" + mock_credentials.return_value = (cred, "") + template = SessionTemplate(name=self._TEMPLATE) + template.runtime_config.version = "2.3" + mock_template_client.return_value.get_session_template.return_value = ( + template + ) + mock_session_controller_client.return_value.create_session.side_effect = InvalidArgument( + "3.0 runtime version does not support configuring Dataproc staging" + " bucket via the staging_bucket field." + ) + + with warnings.catch_warnings(record=True): + warnings.simplefilter("always") + with self.assertRaises(ManagedSparkConnectException) as ctx: + self._builder().runtimeProfile(self._TEMPLATE).getOrCreate() + + err = str(ctx.exception) + self.assertIn("staging_bucket", err) + self.assertIn(f"Note: Runtime Profile '{self._TEMPLATE}'", err) + self.assertIn("specifies runtime version '2.3'", err) + self.assertIn("dataproc-spark-connect~=0.9.0", err) + + @mock.patch( + "google.cloud.managed_spark_connect.session.terminate_s8s_session" + ) + @mock.patch("google.cloud.dataproc_v1.SessionTemplateControllerClient") + @mock.patch("google.auth.default") + @mock.patch("google.cloud.dataproc_v1.SessionControllerClient") + def test_unreadable_pre_3_0_template_terminates_creating_session_and_raises( + self, + mock_session_controller_client, + mock_credentials, + mock_template_client, + mock_terminate, + ): + """When GetSessionTemplate fails and S8s resolves the session to < 3.0, it is terminated immediately.""" + cred = mock.MagicMock() + cred.token = "token" + mock_credentials.return_value = (cred, "") + mock_template_client.return_value.get_session_template.side_effect = ( + PermissionDenied("caller lacks dataproc.sessionTemplates.get") + ) + creating_session = Session() + creating_session.state = Session.State.CREATING + creating_session.runtime_config.version = "2.3" + mock_operation = mock.Mock() + mock_session_controller_client.return_value.create_session.return_value = ( + mock_operation + ) + mock_session_controller_client.return_value.get_session.return_value = ( + creating_session + ) + + with self.assertRaises(ManagedSparkConnectException) as ctx: + self._builder().runtimeProfile(self._TEMPLATE).getOrCreate() + + err = str(ctx.exception) + self.assertIn("resolved to Managed Spark runtime version '2.3'", err) + self.assertIn(".runtimeVersion('3.0')", err) + self.assertIn("dataproc-spark-connect~=0.9.0", err) + mock_terminate.assert_called_once() + mock_operation.result.assert_not_called() + + @mock.patch( + "google.cloud.managed_spark_connect.session.terminate_s8s_session" + ) + @mock.patch("google.cloud.dataproc_v1.SessionTemplateControllerClient") + @mock.patch("google.auth.default") + @mock.patch("google.cloud.dataproc_v1.SessionControllerClient") + def test_unreadable_pre_3_0_template_falls_back_to_post_wait_validation( + self, + mock_session_controller_client, + mock_credentials, + mock_template_client, + mock_terminate, + ): + """When early GetSession fails, unreadable < 3.0 templates are caught once operation.result() returns.""" + cred = mock.MagicMock() + cred.token = "token" + mock_credentials.return_value = (cred, "") + mock_template_client.return_value.get_session_template.side_effect = ( + PermissionDenied("caller lacks dataproc.sessionTemplates.get") + ) + mock_session_controller_client.return_value.get_session.side_effect = ( + PermissionDenied( + "caller lacks dataproc.sessions.get while creating" + ) + ) + active_session = Session() + active_session.state = Session.State.ACTIVE + active_session.runtime_config.version = "2.2.10" + mock_operation = mock.Mock() + mock_operation.result.return_value = active_session + mock_session_controller_client.return_value.create_session.return_value = ( + mock_operation + ) + + with self.assertRaises(ManagedSparkConnectException) as ctx: + self._builder().runtimeProfile(self._TEMPLATE).getOrCreate() + + self.assertIn( + "resolved to Managed Spark runtime version '2.2.10'", + str(ctx.exception), + ) + mock_operation.result.assert_called_once() + mock_terminate.assert_called_once() + + @mock.patch("google.cloud.dataproc_v1.SessionTemplateControllerClient") + @mock.patch("pyspark.util.is_remote_only", return_value=True) + @mock.patch("pyspark.version.__version__", "4.2.1") + def test_template_runtime_version_used_for_pyspark_compatibility_warning( + self, _mock_remote_only, mock_template_client + ): + """PySpark compatibility check uses the Runtime Profile's runtime version.""" + template_40 = SessionTemplate(name=self._TEMPLATE) + template_40.runtime_config.version = "4.0" + mock_template_client.return_value.get_session_template.return_value = ( + template_40 + ) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self._builder().runtimeProfile(self._TEMPLATE)._get_session_config() + pyspark_warnings = [ + w for w in caught if "PySpark 4.2.1" in str(w.message) + ] + self.assertEqual(pyspark_warnings, []) + + template_30 = SessionTemplate(name=self._TEMPLATE) + template_30.runtime_config.version = "3.0" + mock_template_client.return_value.get_session_template.return_value = ( + template_30 + ) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self._builder().runtimeProfile(self._TEMPLATE)._get_session_config() + pyspark_warnings = [ + w for w in caught if "PySpark 4.2.1" in str(w.message) + ] + self.assertEqual(len(pyspark_warnings), 1) @mock.patch("google.cloud.dataproc_v1.SessionTemplateControllerClient") def test_short_session_template_name_qualifies_get_session_template_request( diff --git a/tests/unit/test_session_templates.py b/tests/unit/test_session_templates.py new file mode 100644 index 0000000..317f444 --- /dev/null +++ b/tests/unit/test_session_templates.py @@ -0,0 +1,355 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import unittest +from unittest import mock + +from google.api_core.exceptions import PermissionDenied +from google.cloud.dataproc_v1 import ( + AuthenticationConfig, + GetSessionTemplateRequest, + Session, + SessionTemplate, +) +from google.cloud.managed_spark_connect import _session_templates +from google.cloud.managed_spark_connect.exceptions import ( + ManagedSparkConnectException, +) + + +class SessionTemplateRefTests(unittest.TestCase): + + def test_is_valid_session_template_ref(self): + valid = [ + "my-profile", + "profile_123", + "projects/my-proj/locations/us-central1/sessionTemplates/my-profile", + "projects/google.com:proj/regions/us-east4/sessionTemplates/p_1", + ] + for ref in valid: + with self.subTest(ref=ref): + self.assertTrue( + _session_templates.is_valid_session_template_ref(ref) + ) + + invalid = [ + "", + "projects/my-proj/locations/us-central1/sessionTemplate/typo", + "projects/my-proj/sessionTemplates/missing-location", + "invalid space", + ] + for ref in invalid: + with self.subTest(ref=ref): + self.assertFalse( + _session_templates.is_valid_session_template_ref(ref) + ) + + def test_qualify_session_template_name(self): + self.assertEqual( + _session_templates.qualify_session_template_name( + "my-profile", "proj-a", "us-central1" + ), + "projects/proj-a/locations/us-central1/sessionTemplates/my-profile", + ) + full = "projects/other/locations/us-west1/sessionTemplates/p" + self.assertEqual( + _session_templates.qualify_session_template_name( + full, "proj-a", "us-central1" + ), + full, + ) + self.assertEqual( + _session_templates.qualify_session_template_name( + "my-profile", None, "us-central1" + ), + "my-profile", + ) + + def test_fetch_session_template_handles_empty_success_and_error(self): + get_client = mock.Mock() + self.assertIsNone( + _session_templates.fetch_session_template( + get_client, "", project_id="proj", region="us-central1" + ) + ) + get_client.assert_not_called() + + expected = SessionTemplate( + name="projects/proj/locations/us-central1/sessionTemplates/tmpl" + ) + get_client.return_value.get_session_template.return_value = expected + result = _session_templates.fetch_session_template( + get_client, "tmpl", project_id="proj", region="us-central1" + ) + self.assertEqual(result, expected) + get_client.return_value.get_session_template.assert_called_once_with( + GetSessionTemplateRequest( + name="projects/proj/locations/us-central1/sessionTemplates/tmpl" + ) + ) + + get_client.return_value.get_session_template.side_effect = ( + PermissionDenied("denied") + ) + self.assertIsNone( + _session_templates.fetch_session_template( + get_client, "tmpl", project_id="proj", region="us-central1" + ) + ) + + +class RuntimeVersionResolutionTests(unittest.TestCase): + + _TEMPLATE = ( + "projects/test-project/locations/test-region/sessionTemplates/tmpl" + ) + + def test_is_below_min_runtime_version(self): + cases = [ + ("2.2", "3.0", True), + ("2.3", "3.0", True), + ("2.3.14", "3.0", True), + ("3.0", "3.0", False), + ("3.0.12", "3.0", False), + ("4.0", "3.0", False), + ("", "3.0", False), + ("custom-unparseable", "3.0", False), + ] + for candidate, minimum, expected in cases: + with self.subTest(candidate=candidate, minimum=minimum): + self.assertEqual( + _session_templates.is_below_min_runtime_version( + candidate, minimum + ), + expected, + ) + + def test_resolve_runtime_version_matrix(self): + cases = [ + # (label, explicit, template_name, template_version, expected_req, expected_eff, expected_overridden) + ("explicit_no_template", "4.0", "", None, "4.0", "4.0", None), + ( + "explicit_overrides_template", + "3.0", + self._TEMPLATE, + "4.0", + "3.0", + "3.0", + None, + ), + ( + "explicit_pre_min_passes_through_to_compat_check", + "2.3", + self._TEMPLATE, + "4.0", + "2.3", + "2.3", + None, + ), + ("default_when_no_template", "", "", None, "3.0", "3.0", None), + ( + "unreadable_template_defers_to_server", + "", + self._TEMPLATE, + None, + "", + "", + None, + ), + ( + "empty_template_version_applies_default", + "", + self._TEMPLATE, + "", + "3.0", + "3.0", + None, + ), + ( + "supported_template_version_preserved", + "", + self._TEMPLATE, + "4.0", + "", + "4.0", + None, + ), + ( + "unparseable_template_version_preserved", + "", + self._TEMPLATE, + "custom-preview", + "", + "custom-preview", + None, + ), + ( + "pre_min_template_version_overridden", + "", + self._TEMPLATE, + "2.3", + "3.0", + "3.0", + "2.3", + ), + ] + for ( + label, + explicit, + template_name, + template_version, + expected_req, + expected_eff, + expected_overridden, + ) in cases: + with self.subTest(label=label): + res = _session_templates.resolve_runtime_version( + explicit_version=explicit, + template_name=template_name, + template_version=template_version, + default_version="3.0", + min_version="3.0", + ) + self.assertEqual(res.request_version, expected_req) + self.assertEqual(res.effective_version, expected_eff) + self.assertEqual( + res.overridden_template_version, expected_overridden + ) + self.assertEqual(res.awaits_server_version, expected_eff == "") + + def test_override_warning_and_error_note_messages(self): + preserved = _session_templates.resolve_runtime_version( + explicit_version="", + template_name=self._TEMPLATE, + template_version="4.0", + default_version="3.0", + min_version="3.0", + ) + self.assertIsNone(preserved.override_warning()) + self.assertIsNone(preserved.create_error_note()) + + overridden = _session_templates.resolve_runtime_version( + explicit_version="", + template_name=self._TEMPLATE, + template_version="2.3", + default_version="3.0", + min_version="3.0", + ) + self.assertEqual( + overridden.override_warning(), + f"Runtime Profile '{self._TEMPLATE}' specifies Managed Spark " + "runtime version '2.3', which is not supported by " + "google-cloud-spark-connect (minimum 3.0); overriding with " + "runtime version '3.0'. Update the Runtime Profile to 3.0+, or " + "install 'dataproc-spark-connect~=0.9.0' in a Python 3.11 " + "environment to use runtime version '2.3'.", + ) + self.assertEqual( + overridden.create_error_note(), + f"Note: Runtime Profile '{self._TEMPLATE}' specifies runtime" + " version '2.3', which was overridden because" + " google-cloud-spark-connect requires runtime 3.0 or higher" + " (install 'dataproc-spark-connect~=0.9.0' in a Python 3.11" + " environment to use runtime '2.3').", + ) + + def test_with_server_version_transitions_and_rejects_pre_min(self): + pending = _session_templates.resolve_runtime_version( + explicit_version="", + template_name=self._TEMPLATE, + template_version=None, + default_version="3.0", + min_version="3.0", + ) + self.assertTrue(pending.awaits_server_version) + self.assertIs(pending.with_server_version(""), pending) + + resolved = pending.with_server_version("4.0.1") + self.assertFalse(resolved.awaits_server_version) + self.assertEqual(resolved.effective_version, "4.0.1") + self.assertEqual(resolved.request_version, "") + + with self.assertRaises(ManagedSparkConnectException) as ctx: + pending.with_server_version("2.2.10") + self.assertEqual( + str(ctx.exception), + f"Runtime Profile '{self._TEMPLATE}' resolved to Managed Spark" + " runtime version '2.2.10', which is not supported by" + " google-cloud-spark-connect (minimum 3.0). Pass" + " .runtimeVersion('3.0') (or '4.0') to override the Runtime" + " Profile's runtime version, or install" + " 'dataproc-spark-connect~=0.9.0' in a Python 3.11 environment" + " for runtime 2.3.", + ) + + +class ApplySessionTemplateDefaultsTests(unittest.TestCase): + + _TEMPLATE = ( + "projects/test-project/locations/test-region/sessionTemplates/tmpl" + ) + + def test_preserved_template_leaves_runtime_config_submessage_unset(self): + session_config = Session(session_template=self._TEMPLATE) + template = SessionTemplate(name=self._TEMPLATE) + template.runtime_config.version = "4.0" + + res = _session_templates.apply_session_template_defaults( + session_config, + get_template=lambda _: template, + default_version="3.0", + min_version="3.0", + environ={}, + ) + + self.assertFalse("runtime_config" in session_config) + self.assertEqual(res.effective_version, "4.0") + + def test_env_defaults_respect_template_fields_and_service_account_auth( + self, + ): + session_config = Session(session_template=self._TEMPLATE) + template = SessionTemplate(name=self._TEMPLATE) + template.runtime_config.version = "4.0" + template.environment_config.execution_config.network_uri = "tmpl-net" + template.environment_config.execution_config.ttl = {"seconds": 3600} + + env = { + "MANAGED_SPARK_CONNECT_SERVICE_ACCOUNT": "sa@example.com", + "MANAGED_SPARK_CONNECT_SUBNET": "env-subnet", + "MANAGED_SPARK_CONNECT_TTL_SECONDS": "7200", + "MANAGED_SPARK_CONNECT_IDLE_TTL_SECONDS": "900", + } + _session_templates.apply_session_template_defaults( + session_config, + get_template=lambda _: template, + default_version="3.0", + min_version="3.0", + environ=env, + ) + + exec_cfg = session_config.environment_config.execution_config + self.assertEqual(exec_cfg.service_account, "sa@example.com") + self.assertEqual( + exec_cfg.authentication_config.user_workload_authentication_type, + AuthenticationConfig.AuthenticationType.SERVICE_ACCOUNT, + ) + self.assertEqual(exec_cfg.subnetwork_uri, "") + self.assertFalse(exec_cfg.ttl) + self.assertEqual(exec_cfg.idle_ttl, datetime.timedelta(seconds=900)) + + +if __name__ == "__main__": + unittest.main()