diff --git a/.env.example b/.env.example index 5f5d5401..73a80316 100644 --- a/.env.example +++ b/.env.example @@ -48,7 +48,6 @@ GOOGLE_APPLICATION_CREDENTIALS="/path/to/your/service-account-key.json" # ============================================================================ # Default data source for Spark SQL (currently only supports "bigquery") -# Only available for Managed Spark runtime version 2.3 # MANAGED_SPARK_CONNECT_DEFAULT_DATASOURCE="bigquery" # ============================================================================ diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index b4ff259f..323ef5e0 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -127,16 +127,16 @@ jobs: exit 1 fi echo "pr_head_sha=${PR_HEAD_SHA}" >> "$GITHUB_OUTPUT" - for os in ubuntu-latest macos-latest windows-latest; do + for job in ubuntu-latest macos-latest windows-latest "ubuntu-latest, runtime 4.0"; do gh api "repos/${GITHUB_REPOSITORY}/statuses/${PR_HEAD_SHA}" \ -f state="pending" \ -f target_url="${RUN_URL}" \ -f description="Integration tests running via /gcbrun" \ - -f context="Integration Tests / Run integration tests (${os})" + -f context="Integration Tests / Run integration tests (${job})" done integration-test: - name: Run integration tests (${{ matrix.os }}) + name: Run integration tests (${{ matrix.os }}${{ matrix.runtime != 'default' && format(', runtime {0}', matrix.runtime) || '' }}) # `ControlCheck.dominates` (`ControlChecks.qll`) traverses `needs:`, so # `authorize-gcbrun`'s `BashCommentVsHeadDateCheck` dominates this job's checkout. needs: [authorize-gcbrun] @@ -152,6 +152,15 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] + # "default" leaves the runtime to the library, with the [client] + # extra's pyspark-client. Other runtimes need their own pyspark, and + # their own Python for Python UDFs; they run on one OS only. + runtime: ["default"] + include: + - os: "ubuntu-latest" + runtime: "4.0" + python-version: "3.13" + pyspark: "pyspark-client~=4.2.0" # Run on push to main, manual dispatch, non-Dependabot same-repo pull requests, # or authorized /gcbrun PR comments (`needs.authorize-gcbrun.result == 'success'`). @@ -186,7 +195,7 @@ jobs: - name: Setup Python uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: - python-version: "3.12" + python-version: ${{ matrix.python-version || '3.12' }} - name: Set up uv uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 @@ -200,13 +209,22 @@ jobs: ~/.cache/uv ~/Library/Caches/uv ~/AppData/Local/uv/cache - key: ${{ runner.os }}-uv-integration-${{ hashFiles('setup.py', 'requirements-test.txt') }} + key: ${{ runner.os }}-uv-integration-${{ matrix.runtime }}-${{ hashFiles('setup.py', 'requirements-test.txt') }} restore-keys: | - ${{ runner.os }}-uv-integration- + ${{ runner.os }}-uv-integration-${{ matrix.runtime }}- ${{ runner.os }}-uv- + # Without the [client] extra when the runtime needs a different pyspark, + # since the extra would pin the default runtime's. - name: Install dependencies - run: uv pip install --system -e '.[client,interactive]' -r requirements-test.txt + env: + PYSPARK: ${{ matrix.pyspark }} + run: | + if [[ -n "$PYSPARK" ]]; then + uv pip install --system -e '.[interactive]' "$PYSPARK" -r requirements-test.txt + else + uv pip install --system -e '.[client,interactive]' -r requirements-test.txt + fi - name: Authenticate to Google Cloud uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 @@ -237,6 +255,7 @@ jobs: GOOGLE_CLOUD_REGION: ${{ vars.GCP_REGION || secrets.GCP_REGION || 'us-central1,us-east1,us-west1' }} MANAGED_SPARK_CONNECT_SUBNET: ${{ vars.GCP_SUBNET || secrets.GCP_SUBNET }} MANAGED_SPARK_CONNECT_AUTH_TYPE: "SERVICE_ACCOUNT" + MANAGED_SPARK_CONNECT_TEST_RUNTIME_VERSION: ${{ matrix.runtime != 'default' && matrix.runtime || '' }} run: | python -m pytest tests/integration/ -v --tb=short -x @@ -247,7 +266,7 @@ jobs: JOB_STATUS: ${{ job.status }} PR_HEAD_SHA: ${{ needs.authorize-gcbrun.outputs.pr_head_sha }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - STATUS_CONTEXT: Integration Tests / Run integration tests (${{ matrix.os }}) + STATUS_CONTEXT: Integration Tests / Run integration tests (${{ matrix.os }}${{ matrix.runtime != 'default' && format(', runtime {0}', matrix.runtime) || '' }}) run: | if [[ "$JOB_STATUS" == "success" ]]; then STATE="success" diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 9451e045..94b7992c 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -126,13 +126,12 @@ jobs: export JAVA_HOME="${JAVA_HOME_17_X64:-${JAVA_HOME_17_ARM64:-${JAVA_HOME_21_X64:-${JAVA_HOME_21_ARM64:-$JAVA_HOME}}}}" python -m pytest tests/local/ -v --tb=short - # The [client] and [spark] extras pin the pyspark minor version of the - # default Managed Spark runtime. Users of other runtimes, and environments - # that preinstall pyspark, can have a newer one, so also run the full suite - # against the newest pyspark-client. On Python 3.13 rather than 3.12, which - # the main matrix already covers with the pinned pyspark. - test-newest-pyspark: - name: Run unit tests (ubuntu-latest, Python 3.13, newest pyspark-client) + # The main matrix covers the default runtime (3.0): Python 3.12 with the + # pyspark the [client] and [spark] extras pin. This covers runtime 4.0's + # Python and pyspark, installed without an extra, as users of runtime 4.0 + # install it. + test-runtime-4-0: + name: Run unit tests (ubuntu-latest, Python 3.13, runtime 4.0 pyspark-client) runs-on: ubuntu-latest timeout-minutes: 15 defaults: @@ -155,7 +154,7 @@ jobs: # No extra: it would pin pyspark-client back to the default runtime's. - name: Install dependencies - run: uv pip install --system -e '.[interactive]' pyspark-client -r requirements-test.txt + run: uv pip install --system -e '.[interactive]' 'pyspark-client~=4.2.0' -r requirements-test.txt # Serially, without xdist: pyspark 4.2 exits on import when __main__ has # no __file__, as in xdist workers, mistaking that for running its own diff --git a/README.md b/README.md index 05c7da59..6fd613de 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,54 @@ in your code using the builder API: spark = ManagedSparkSession.builder.projectId('my-project').location('us-central1').sessionConfig(session_config).getOrCreate() ``` +### Runtime versions + +Sessions run on Managed Spark runtime 3.0 unless you choose another version +with `runtimeVersion()` or `Session.runtime_config.version`. Spark Connect +supports clients whose PySpark minor version is the same as or older than the +runtime's Spark version, but not newer, and Python UDFs require matching Python +minor versions: + +| Runtime | Spark | Python | Install | +|---------|-------|--------|------------------------------------------------------------------| +| 3.0 | 4.0 | 3.12 | `pip install 'google-cloud-spark-connect[client]'` | +| 4.0 | 4.2 | 3.13 | `pip install google-cloud-spark-connect 'pyspark-client~=4.2.0'` | + +The `client` and `spark` extras install PySpark 4.0, which works with both +runtimes, but only has the APIs of Spark 4.0. To use newer APIs on runtime 4.0, +install a newer PySpark instead of the extras, for example in a fresh Python +3.13 environment: + +```sh +pip install 'google-cloud-spark-connect[interactive]' 'pyspark-client~=4.2.0' +``` + +```python +from google.cloud.managed_spark_connect import ManagedSparkSession + +spark = ( + ManagedSparkSession.builder.runtimeVersion('4.0') + .subnetwork('default') + .getOrCreate() +) +``` + +The client warns when the installed PySpark is newer than the session runtime's +Spark, or the Python version does not match. + +Runtime 4.0 notes (see the +[Managed Spark runtime 4.0 documentation](https://docs.cloud.google.com/managed-spark/docs/concepts/versions/spark-runtime-4.0)): + +* **Subnetwork required:** Runtime 4.0 requires an IPv4-only subnetwork via + `.subnetwork()`, `MANAGED_SPARK_CONNECT_SUBNET`, or `.runtimeProfile()`. +* **Default catalog:** Runtime 4.0 defaults `spark.sql.defaultCatalog` to the + BigLake Iceberg `default_catalog` (with no default namespace, so unqualified + single-part DDL like `CREATE TABLE foo` must either qualify the namespace or + pass `.config('spark.sql.defaultCatalog', 'spark_catalog')` to restore + `spark_catalog`). +* **Remote PyPI installation:** `%dpip` and `spark.addArtifacts(..., pypi=True)` + are not yet supported on runtime 4.0. + ### Builder Configuration The `ManagedSparkSession.builder` provides a fluent API to configure the session. Below is a list of available methods: @@ -119,7 +167,7 @@ The `ManagedSparkSession.builder` provides a fluent API to configure the session | `location(location)` | Sets the Google Cloud region. | | `projectId(project_id)` | Sets the Google Cloud project ID. | | `runtimeProfile(profile)` | Sets the Runtime Profile to use. | -| `runtimeVersion(version)` | Sets the Managed Spark runtime version (e.g., "3.0"). | +| `runtimeVersion(version)` | Sets the Managed Spark runtime version (e.g., `"3.0"`, `"4.0"`). | | `serviceAccount(account)` | Sets the service account for the session. | | `sessionTemplate(profile)` | Alias for `runtimeProfile(profile)`. | | `subnetwork(subnet)` | Sets the subnetwork URI for the session. | diff --git a/google/cloud/managed_spark_connect/_runtimes.py b/google/cloud/managed_spark_connect/_runtimes.py index 7163bc85..79d41f5a 100644 --- a/google/cloud/managed_spark_connect/_runtimes.py +++ b/google/cloud/managed_spark_connect/_runtimes.py @@ -45,6 +45,7 @@ class RuntimeInfo: # without the compatibility warnings. _RUNTIMES = { (3, 0): RuntimeInfo(spark=(4, 0), python=(3, 12)), + (4, 0): RuntimeInfo(spark=(4, 2), python=(3, 13)), } diff --git a/google/cloud/managed_spark_connect/pypi_artifacts.py b/google/cloud/managed_spark_connect/pypi_artifacts.py index e8ed4974..c42637fe 100644 --- a/google/cloud/managed_spark_connect/pypi_artifacts.py +++ b/google/cloud/managed_spark_connect/pypi_artifacts.py @@ -1,15 +1,82 @@ import json import logging import os +import re import tempfile import uuid -from typing import TYPE_CHECKING +from typing import Any, Sequence, TYPE_CHECKING if TYPE_CHECKING: from packaging.requirements import Requirement logger = logging.getLogger(__name__) +DATAPROC_COMMAND_RUNNER = "org.apache.spark.sql.artifact.DataprocCommandRunner" +PIP_INSTALL_COMMAND = "PipInstallPackages" +PIP_INSTALL_FAILURE_MSG = "Pip install failed with non-zero exit code" +_INSTALL_FAILURE_PATTERN = re.compile(r"(?:Pip|Pixi) install failed\b") + + +def validate_pypi_requirements(packages: Sequence[str]) -> list["Requirement"]: + from packaging.requirements import Requirement + + reqs = [Requirement(p) for p in packages] + if any(len(req.specifier) == 0 for req in reqs): + logger.debug("It is recommended to pin the version of the package") + return reqs + + +def install_pypi_packages( + client: Any, + packages: Sequence[str], + *, + validate: bool = True, +) -> str: + """Install PyPI packages on the remote session via DataprocCommandRunner. + + Used by both ``ManagedSparkSession.addArtifacts(..., pypi=True)`` and the + ``%dpip`` notebook magic. + """ + unique_packages = list(dict.fromkeys(packages)) + if not unique_packages: + return "" + if validate: + validate_pypi_requirements(unique_packages) + + # Imported lazily: PySpark (and with it pyarrow) is an optional extra, so + # importing this module or loading %dpip must work without it. + import pyarrow as pa + import pyspark.sql.connect.proto as pb2 + + command = pb2.Command() + command.execute_external_command.runner = DATAPROC_COMMAND_RUNNER + command.execute_external_command.command = PIP_INSTALL_COMMAND + for index, package in enumerate(unique_packages): + command.execute_external_command.options[str(index)] = package + + _, properties, _ = client.execute_command(command) + + try: + binary_data = properties["sql_command_result"].local_relation.data + except (KeyError, AttributeError, TypeError) as e: + raise RuntimeError( + "Unexpected response structure: missing binary data." + ) from e + if not binary_data: + raise RuntimeError( + "Unexpected response structure: missing binary data." + ) + + try: + table = pa.ipc.RecordBatchStreamReader(binary_data).read_all() + output = "\n".join(table.column(0).to_pylist()) + except Exception as e: + raise RuntimeError(f"Error decoding Arrow data: {e}") from e + + if _INSTALL_FAILURE_PATTERN.search(output): + raise RuntimeError(output) + return output + class PyPiArtifacts: """ @@ -17,17 +84,8 @@ class PyPiArtifacts: that Spark Connect server understands """ - @staticmethod - def __try_parsing_package(packages: set[str]) -> list["Requirement"]: - from packaging.requirements import Requirement - - reqs = [Requirement(p) for p in packages] - if 0 in [len(req.specifier) for req in reqs]: - logger.debug("It is recommended to pin the version of the package") - return reqs - def __init__(self, packages: set[str]): - self.requirements = PyPiArtifacts.__try_parsing_package(packages) + self.requirements = validate_pypi_requirements(list(packages)) def write_packages_config(self, s8s_session_uuid: str) -> str: """ diff --git a/google/cloud/managed_spark_connect/session.py b/google/cloud/managed_spark_connect/session.py index 61f5256d..6a6e73c9 100644 --- a/google/cloud/managed_spark_connect/session.py +++ b/google/cloud/managed_spark_connect/session.py @@ -90,7 +90,10 @@ from google.cloud.managed_spark_connect.instrumentation._core import ( _CURRENT_SPAN, ) -from google.cloud.managed_spark_connect.pypi_artifacts import PyPiArtifacts +from google.cloud.managed_spark_connect.pypi_artifacts import ( + PyPiArtifacts, + install_pypi_packages, +) from google.cloud.dataproc_v1 import ( AuthenticationConfig, CreateSessionRequest, @@ -532,6 +535,24 @@ def _format_create_session_error( "sessionTemplates/{template}'." ) + if isinstance(exc, InvalidArgument) and ( + "subnetwork must be specified" in base_msg.lower() + or "specify a subnetwork" in base_msg.lower() + ): + extra_parts.append( + "Pass .subnetwork('') (for example .subnetwork('default'))," + " set MANAGED_SPARK_CONNECT_SUBNET, or configure a subnetwork in" + " .runtimeProfile('')." + ) + elif ( + isinstance(exc, InvalidArgument) + and "stack type, which is not supported" in base_msg.lower() + ): + extra_parts.append( + "Managed Spark runtime 4.0 requires an IPv4-only subnetwork; pass" + " one with .subnetwork('') or MANAGED_SPARK_CONNECT_SUBNET." + ) + if base_msg and extra_parts and base_msg[-1] not in ".!?:;": base_msg = f"{base_msg}." parts = [p for p in (base_msg, *extra_parts) if p] @@ -1739,9 +1760,12 @@ def _get_session_config(self): match default_datasource: case "bigquery": # Merge default configs with existing properties, - # user configs take precedence + # user configs take precedence. Explicitly set + # spark.sql.defaultCatalog so runtime 4.0 does not replace + # spark_catalog with the Lakehouse default_catalog. for k, v in { "spark.sql.catalog.spark_catalog": "com.google.cloud.spark.bigquery.BigQuerySparkSessionCatalog", + "spark.sql.defaultCatalog": "spark_catalog", "spark.sql.sources.default": "bigquery", }.items(): if k not in session_config.runtime_config.properties: @@ -2441,16 +2465,22 @@ def addArtifacts( "Cannot add PyPI artifacts: no active Managed Spark" " session." ) - artifacts = PyPiArtifacts(set(artifact)) - logger.debug("Making addArtifact call to install packages") - self.addArtifact( - _normalize_artifact_path( - artifacts.write_packages_config( - self._active_s8s_session_uuid - ) - ), - file=True, + if not artifact: + return + from packaging.requirements import InvalidRequirement + + logger.debug( + "Installing PyPI packages via DataprocCommandRunner: %s", + artifact, ) + try: + install_pypi_packages(self.client, artifact, validate=True) + except InvalidRequirement: + raise + except Exception as e: + raise ManagedSparkConnectException( + f"Failed to install PyPI packages: {e}" + ) from e else: super().addArtifacts( *(_normalize_artifact_path(p) for p in artifact), diff --git a/google/cloud/managed_spark_magics/magics.py b/google/cloud/managed_spark_magics/magics.py index 9ad075c9..5cb18764 100644 --- a/google/cloud/managed_spark_magics/magics.py +++ b/google/cloud/managed_spark_magics/magics.py @@ -14,7 +14,6 @@ """Managed Spark magic implementations.""" -import re import shlex from IPython.core.magic import (Magics, magics_class, line_magic) @@ -48,10 +47,6 @@ def dpip(self, line): print(f"Installing packages: {packages}") output = self._run_command(packages, session) - failure_match = re.search(self.PIP_INSTALL_FAILURE_MSG, output) - if failure_match: - raise RuntimeError(output) - print(output) print("Finished installing packages.") except Exception as e: @@ -94,30 +89,8 @@ def _check_preconditions(self, args): return packages, session def _run_command(self, packages, session): - # Imported here rather than at module scope, like the package's other - # pyspark imports: PySpark (and with it pyarrow) is an optional extra, - # and this extension must still load without it. - import pyarrow as pa - import pyspark.sql.connect.proto as pb2 - - command = pb2.Command() - command.execute_external_command.runner = self.DATAPROC_COMMAND_RUNNER - command.execute_external_command.command = self.PIP_INSTALL_COMMAND - - for index, package in enumerate(packages): - command.execute_external_command.options[str(index)] = package - - _, properties, _ = session.client.execute_command(command) + from google.cloud.managed_spark_connect.pypi_artifacts import ( + install_pypi_packages, + ) - try: - binary_data = properties["sql_command_result"].local_relation.data - - # decode the Arrow stream and return the output - table = pa.ipc.RecordBatchStreamReader(binary_data).read_all() - return "\n".join(table.column(0).to_pylist()) - except (KeyError, AttributeError) as e: - raise RuntimeError( - "Unexpected response structure: missing binary data." - ) from e - except Exception as e: - raise RuntimeError(f"Error decoding Arrow data: {e}") from e + return install_pypi_packages(session.client, packages, validate=False) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 9ac5049b..f0c0218d 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -169,6 +169,25 @@ def test_project(): return os.getenv("GOOGLE_CLOUD_PROJECT") +@pytest.fixture(scope="session", autouse=True) +def _test_runtime_version(): + """Runs every session on MANAGED_SPARK_CONNECT_TEST_RUNTIME_VERSION, if set. + + Overrides the default rather than setting runtimeVersion() on each builder, + so that tests asserting the default runtime version check this one instead. + Install the pyspark that runtime needs (see _runtimes.py) along with it. + """ + runtime_version = os.getenv("MANAGED_SPARK_CONNECT_TEST_RUNTIME_VERSION") + if not runtime_version: + yield ManagedSparkSession._DEFAULT_RUNTIME_VERSION + return + with pytest.MonkeyPatch.context() as mp: + mp.setattr( + ManagedSparkSession, "_DEFAULT_RUNTIME_VERSION", runtime_version + ) + yield runtime_version + + @pytest.fixture(scope="session") def _region_state(test_subnet): regions_spec = os.getenv("GOOGLE_CLOUD_REGION") or ",".join( diff --git a/tests/integration/managed_spark_magics/test_magics.py b/tests/integration/managed_spark_magics/test_magics.py index 6e172821..dc9b72ef 100644 --- a/tests/integration/managed_spark_magics/test_magics.py +++ b/tests/integration/managed_spark_magics/test_magics.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os + import pytest @@ -33,6 +35,20 @@ def ipython_shell(connect_session): TerminalInteractiveShell.clear_instance() +# Runtime 4.0 manages its Python environment with Pixi instead of pip, and +# server-side DataprocCommandRunner support for Pixi is not rolled out yet. +# Non-strict so CI stays green once the server-side fix lands. +_xfail_on_runtime_4 = pytest.mark.xfail( + os.getenv("MANAGED_SPARK_CONNECT_TEST_RUNTIME_VERSION", "").startswith( + "4." + ), + reason="PyPI package installation in Spark Connect sessions is not" + " available yet on runtime 4.0", + strict=False, +) + + +@_xfail_on_runtime_4 def test_dpip_install_success(connect_session, ipython_shell, capsys): """Test installing PyPI packages with %dpip and running a UDF on remote executors.""" ipython_shell.run_line_magic("dpip", "install roman numpy") @@ -58,6 +74,7 @@ def to_roman(number): assert df_result[0]["roman"] == "MDCLXVI" +@_xfail_on_runtime_4 def test_dpip_install_failure(ipython_shell): """Test error message on remote PyPI installation failure.""" with pytest.raises( diff --git a/tests/integration/test_session.py b/tests/integration/test_session.py index 67ea6b4f..00d6323b 100644 --- a/tests/integration/test_session.py +++ b/tests/integration/test_session.py @@ -51,16 +51,21 @@ def test_create_spark_session_with_default_notebook_behavior( "id", "name" ) assert str(df) == "DataFrame[id: bigint, name: string]" - connect_session.sql("DROP TABLE IF EXISTS FOO") - connect_session.sql("CREATE TABLE FOO (bar long, baz long) USING PARQUET") + # Qualified, since the default catalog differs between runtimes: runtime + # 4.0 defaults to a lakehouse catalog with no current namespace. + table = "spark_catalog.default.FOO" + connect_session.sql(f"DROP TABLE IF EXISTS {table}") + connect_session.sql( + f"CREATE TABLE {table} (bar long, baz long) USING PARQUET" + ) try: with pytest.raises(connect_exceptions.AnalysisException) as ex: connect_session.sql( - "CREATE TABLE FOO (bar long, baz long) USING PARQUET" + f"CREATE TABLE {table} (bar long, baz long) USING PARQUET" ) assert "[TABLE_OR_VIEW_ALREADY_EXISTS]" in str(ex.value) finally: - connect_session.sql("DROP TABLE IF EXISTS FOO") + connect_session.sql(f"DROP TABLE IF EXISTS {table}") assert ManagedSparkSession._active_s8s_session_uuid is not None diff --git a/tests/unit/managed_spark_magics/test_magics.py b/tests/unit/managed_spark_magics/test_magics.py index f8d0435c..825564c9 100644 --- a/tests/unit/managed_spark_magics/test_magics.py +++ b/tests/unit/managed_spark_magics/test_magics.py @@ -156,6 +156,28 @@ def test_dpip_install_failure(self): ): self.magics.dpip("install non-existent-package") + def test_dpip_install_io_exception_failure(self): + mock_session = mock.Mock(spec=ManagedSparkSession) + mock_session.client = mock.Mock() + properties = mock.Mock() + properties.sql_command_result.local_relation.data = ( + self._create_mock_arrow_binary( + ["Pip install failed: IOException during pip install process"] + ) + ) + mock_session.client.execute_command.return_value = ( + None, + {"sql_command_result": properties.sql_command_result}, + None, + ) + self.shell.user_ns["spark"] = mock_session + + with self.assertRaisesRegex( + RuntimeError, + "Failed to install packages: Pip install failed: IOException", + ): + self.magics.dpip("install roman") + def test_dpip_unexpected_response(self): mock_session = mock.Mock(spec=ManagedSparkSession) mock_session.client = mock.Mock() diff --git a/tests/unit/test_init.py b/tests/unit/test_init.py index 78be3cba..ef235818 100644 --- a/tests/unit/test_init.py +++ b/tests/unit/test_init.py @@ -89,6 +89,25 @@ def test_no_warning_for_unknown_runtime_version(self): mock_warn.assert_not_called() + def test_python_version_mismatch_warning_for_runtime_40(self): + """Runtime 4.0 uses Python 3.13.""" + for client_python, expect_warning in [ + ((3, 12), True), + ((3, 13), False), + ]: + with self.subTest(client_python=client_python): + with mock.patch("sys.version_info", (*client_python, 0)): + with mock.patch("warnings.warn") as mock_warn: + ManagedSparkSession.Builder()._check_python_version_compatibility( + "4.0" + ) + self.assertEqual(mock_warn.called, expect_warning) + if expect_warning: + self.assertIn( + "Managed Spark runtime 4.0 uses Python 3.13", + mock_warn.call_args.args[0], + ) + def test_python_version_mismatch_warning_for_subminor_runtime(self): """Subminor runtime versions are looked up by their major.minor.""" with mock.patch("sys.version_info", (3, 11, 0)): @@ -117,9 +136,23 @@ def _check(self, runtime_version, pyspark_version, remote_only=True): return mock_warn def test_no_warning_when_spark_versions_match(self): - for pyspark_version in ["4.0.0", "4.0.5", "4.0.1.dev1"]: - with self.subTest(pyspark_version=pyspark_version): - self._check("3.0", pyspark_version).assert_not_called() + for runtime_version, pyspark_version in [ + ("3.0", "4.0.0"), + ("3.0", "4.0.5"), + ("3.0", "4.0.1.dev1"), + ("4.0", "4.2.0"), + ("4.0", "4.2.1"), + ]: + with self.subTest( + runtime_version=runtime_version, pyspark_version=pyspark_version + ): + self._check( + runtime_version, pyspark_version + ).assert_not_called() + + def test_no_warning_for_runtime_40_with_pyspark_40(self): + """The [client] extra's pyspark-client works with runtime 4.0 too.""" + self._check("4.0", "4.0.5").assert_not_called() def test_no_warning_when_pyspark_older_than_runtime(self): """Spark Connect servers support older clients.""" diff --git a/tests/unit/test_pypi_artifacts.py b/tests/unit/test_pypi_artifacts.py index 580d6646..71ae4255 100644 --- a/tests/unit/test_pypi_artifacts.py +++ b/tests/unit/test_pypi_artifacts.py @@ -9,10 +9,28 @@ import unittest import zipfile +from unittest import mock + from packaging.requirements import InvalidRequirement +import pyarrow as pa +import pyspark.sql.connect.proto as pb2 import pytest -from google.cloud.managed_spark_connect.pypi_artifacts import PyPiArtifacts +from google.cloud.managed_spark_connect.pypi_artifacts import ( + DATAPROC_COMMAND_RUNNER, + PIP_INSTALL_COMMAND, + PyPiArtifacts, + install_pypi_packages, +) + + +def _arrow_lines(lines: list[str]) -> bytes: + schema = pa.schema([pa.field("output", pa.string())]) + table = pa.Table.from_arrays([lines], schema=schema) + sink = pa.BufferOutputStream() + with pa.ipc.RecordBatchStreamWriter(sink, table.schema) as writer: + writer.write_table(table) + return sink.getvalue() class PyPiArtifactsTest(unittest.TestCase): @@ -58,6 +76,75 @@ def test_validate_file_name_pattern(self): self.assertTrue(file_name.startswith("add-artifacts-1729-")) self.assertTrue(file_name.endswith(".json")) + def test_install_pypi_packages_executes_command_runner(self): + client = mock.Mock() + props = mock.Mock() + props.sql_command_result.local_relation.data = _arrow_lines( + ["Successfully installed spacy-1.2.3 roman-4.0"] + ) + client.execute_command.return_value = ( + None, + {"sql_command_result": props.sql_command_result}, + None, + ) + + output = install_pypi_packages( + client, ["spacy==1.2.3", "roman", "spacy==1.2.3"] + ) + self.assertEqual(output, "Successfully installed spacy-1.2.3 roman-4.0") + client.execute_command.assert_called_once() + cmd = client.execute_command.call_args[0][0] + self.assertIsInstance(cmd, pb2.Command) + self.assertEqual( + cmd.execute_external_command.runner, DATAPROC_COMMAND_RUNNER + ) + self.assertEqual( + cmd.execute_external_command.command, PIP_INSTALL_COMMAND + ) + self.assertEqual( + dict(cmd.execute_external_command.options), + {"0": "spacy==1.2.3", "1": "roman"}, + ) + + def test_install_pypi_packages_raises_on_io_and_pixi_failures(self): + for failure_line in ( + "Pip install failed with non-zero exit code: 1", + "Pip install failed: IOException during pip install process", + "Pip install failed: InterruptedException during pip install process", + "Pixi install failed with non-zero exit code: 1", + ): + with self.subTest(failure_line=failure_line): + client = mock.Mock() + props = mock.Mock() + props.sql_command_result.local_relation.data = _arrow_lines( + [failure_line] + ) + client.execute_command.return_value = ( + None, + {"sql_command_result": props.sql_command_result}, + None, + ) + with self.assertRaisesRegex(RuntimeError, failure_line): + install_pypi_packages(client, ["roman"]) + + def test_install_pypi_packages_missing_or_null_properties(self): + empty_result = mock.Mock() + empty_result.local_relation.data = b"" + for bad_props in ( + None, + {}, + {"sql_command_result": None}, + {"sql_command_result": empty_result}, + ): + with self.subTest(bad_props=bad_props): + client = mock.Mock() + client.execute_command.return_value = (None, bad_props, None) + with self.assertRaisesRegex( + RuntimeError, + "Unexpected response structure: missing binary data.", + ): + install_pypi_packages(client, ["roman"]) + class PackageBuildWhiteoutTest(unittest.TestCase): diff --git a/tests/unit/test_runtimes.py b/tests/unit/test_runtimes.py index 31fa81e5..86f30f85 100644 --- a/tests/unit/test_runtimes.py +++ b/tests/unit/test_runtimes.py @@ -43,6 +43,10 @@ def test_known_runtime(self): _runtimes.get_runtime_info("3.0"), _runtimes.RuntimeInfo(spark=(4, 0), python=(3, 12)), ) + self.assertEqual( + _runtimes.get_runtime_info("4.0"), + _runtimes.RuntimeInfo(spark=(4, 2), python=(3, 13)), + ) self.assertEqual( _runtimes.get_runtime_info("3.0.12"), _runtimes.get_runtime_info("3.0"), diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 74a82f70..937b992e 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -272,7 +272,6 @@ def test_pypi_add_artifacts( ) session = ManagedSparkSession.builder.getOrCreate() self.assertTrue(isinstance(session, ManagedSparkSession)) - session.addArtifact = mock.MagicMock() # Setting two flags together with self.assertRaisesRegex( @@ -281,38 +280,44 @@ def test_pypi_add_artifacts( ): session.addArtifacts("abc.txt", file=True, pypi=True) - # Propagate error - session.addArtifact.side_effect = Exception("Error installing") - with self.assertRaisesRegex( - Exception, - "Error installing", - ): + with mock.patch.object( + session_module, "install_pypi_packages" + ) as mock_install_pypi: + # Propagate error wrapped in ManagedSparkConnectException + mock_install_pypi.side_effect = RuntimeError("Error installing") + with self.assertRaisesRegex( + ManagedSparkConnectException, + "Failed to install PyPI packages: Error installing", + ): + session.addArtifacts("spacy", pypi=True) + mock_install_pypi.side_effect = None + + # Do install if earlier addArtifacts resulted in failure session.addArtifacts("spacy", pypi=True) - session.addArtifact.side_effect = None + mock_install_pypi.assert_called_with( + session.client, ("spacy",), validate=True + ) + self.assertEqual(mock_install_pypi.call_count, 2) - # Do install if earlier add artifact resulted in failure - session.addArtifacts("spacy", pypi=True) - self.assertEqual(session.addArtifact.call_count, 2) + # Multiple packages + session.addArtifacts("spacy==1.2.3", "spacy", pypi=True) + mock_install_pypi.assert_called_with( + session.client, ("spacy==1.2.3", "spacy"), validate=True + ) + self.assertEqual(mock_install_pypi.call_count, 3) - # test multiple packages, when already installed - session.addArtifacts("spacy==1.2.3", "spacy", pypi=True) - self.assertEqual(session.addArtifact.call_count, 3) + # Empty package tuple is a no-op + session.addArtifacts(pypi=True) + self.assertEqual(mock_install_pypi.call_count, 3) - # Windows drive-letter paths are converted to file:// URIs before - # delegating to PySpark's ArtifactManager._parse_artifacts (both via - # addArtifacts/addArtifact and via pypi=True). - with mock.patch.object( - session_module.PyPiArtifacts, - "write_packages_config", - return_value=r"C:\Users\runner\AppData\Local\Temp\add-artifacts-1729.json", - ): - session.addArtifacts("spacy", pypi=True) - session.addArtifact.assert_called_with( - "file:///C:/Users/runner/AppData/Local/Temp/add-artifacts-1729.json", - file=True, - ) + # Invalid requirement format still raises InvalidRequirement directly + from packaging.requirements import InvalidRequirement - del session.addArtifact + with self.assertRaises(InvalidRequirement): + session.addArtifacts("pypi://spacy:23", pypi=True) + + # Windows drive-letter paths are converted to file:// URIs before + # delegating to PySpark's ArtifactManager._parse_artifacts. with mock.patch( "pyspark.sql.connect.session.SparkSession.addArtifacts" ) as mock_super_add_artifacts: @@ -1154,6 +1159,66 @@ def test_create_spark_session_with_malformed_session_template_uri( ) self.assertIsInstance(ctx.exception.__cause__, InvalidArgument) + @mock.patch("google.auth.default") + @mock.patch("google.cloud.dataproc_v1.SessionControllerClient") + def test_create_spark_session_with_missing_subnetwork_hint( + self, + mock_session_controller_client, + mock_credentials, + ): + cred = mock.MagicMock() + cred.token = "token" + mock_credentials.return_value = (cred, "") + mock_operation = mock.Mock() + mock_operation.result.side_effect = InvalidArgument( + "a subnetwork must be specified" + ) + mock_session_controller_client.return_value.create_session.return_value = ( + mock_operation + ) + + with self.assertRaises(ManagedSparkConnectException) as ctx: + ( + ManagedSparkSession.builder.projectId("test-project") + .location("us-central1") + .runtimeVersion("4.0") + .getOrCreate() + ) + + self.assertEqual( + ctx.exception.message, + "Error while creating Managed Spark Session: " + "a subnetwork must be specified. " + "Pass .subnetwork('') (for example .subnetwork('default'))," + " set MANAGED_SPARK_CONNECT_SUBNET, or configure a subnetwork in" + " .runtimeProfile('').", + ) + self.assertIsInstance(ctx.exception.__cause__, InvalidArgument) + + mock_operation.result.side_effect = InvalidArgument( + "Subnetwork projects/123/regions/us-central1/subnetworks/default" + " has dual-stack (IPV4_IPV6) stack type, which is not supported." + ) + with self.assertRaises(ManagedSparkConnectException) as stack_ctx: + ( + ManagedSparkSession.builder.projectId("test-project") + .location("us-central1") + .runtimeVersion("4.0") + .subnetwork("default") + .getOrCreate() + ) + + self.assertEqual( + stack_ctx.exception.message, + "Error while creating Managed Spark Session: " + "Subnetwork projects/123/regions/us-central1/subnetworks/default" + " has dual-stack (IPV4_IPV6) stack type, which is not supported. " + "Managed Spark runtime 4.0 requires an IPv4-only subnetwork; pass" + " one with .subnetwork('') or" + " MANAGED_SPARK_CONNECT_SUBNET.", + ) + self.assertIsInstance(stack_ctx.exception.__cause__, InvalidArgument) + @mock.patch("google.auth.default") @mock.patch("google.cloud.dataproc_v1.SessionControllerClient") @mock.patch( @@ -1495,6 +1560,12 @@ def test_create_session_with_default_datasource_env_var( ), "com.google.cloud.spark.bigquery.BigQuerySparkSessionCatalog", ) + self.assertEqual( + create_session_request.session.runtime_config.properties.get( + "spark.sql.defaultCatalog" + ), + "spark_catalog", + ) mock_logger.warning.assert_not_called() self.stopSession(mock_session_controller_client_instance, session) mock_session_controller_client_instance.create_session.reset_mock()