From e6b29836c8073bdb105b7d4ce75f138a48b13126 Mon Sep 17 00:00:00 2001 From: Dave Borowitz Date: Wed, 28 Oct 2026 16:33:12 -0700 Subject: [PATCH 1/2] feat: support Managed Spark runtime 4.0 Runtime 4.0 runs Spark 4.2 and Python 3.13. Add it to the runtime table so that the client warns about a newer PySpark and mismatched Python, append actionable hints when session creation fails due to a missing or dual-stack subnetwork, keep spark_catalog active on runtime 4.0 when MANAGED_SPARK_CONNECT_DEFAULT_DATASOURCE=bigquery is set, and document how to opt into runtime 4.0 and its behavioral differences. The [client]/[spark] extras' PySpark 4.0 works against it too, since Spark Connect supports older clients, so that warns about nothing but Python. CI now runs the full unit suite and the integration tests on each runtime's own Python and pyspark: runtime 3.0 (3.12, the pinned extras) as before, and runtime 4.0 (3.13, pyspark-client 4.2, which replaces the unpinned newest pyspark-client job) on Linux, selected for the integration tests via the new MANAGED_SPARK_CONNECT_TEST_RUNTIME_VERSION. The default runtime and the [client]/[spark] pins stay on 3.0. --- .env.example | 1 - .github/workflows/integration-tests.yaml | 35 +++++++--- .github/workflows/tests.yaml | 15 ++--- README.md | 50 +++++++++++++- .../cloud/managed_spark_connect/_runtimes.py | 1 + google/cloud/managed_spark_connect/session.py | 23 ++++++- tests/integration/conftest.py | 19 ++++++ .../managed_spark_magics/test_magics.py | 17 +++++ tests/integration/test_session.py | 13 ++-- tests/unit/test_init.py | 39 ++++++++++- tests/unit/test_runtimes.py | 4 ++ tests/unit/test_session.py | 66 +++++++++++++++++++ 12 files changed, 257 insertions(+), 26 deletions(-) diff --git a/.env.example b/.env.example index 5f5d5401..73a80316 100644 --- a/.env.example +++ b/.env.example @@ -48,7 +48,6 @@ GOOGLE_APPLICATION_CREDENTIALS="/path/to/your/service-account-key.json" # ============================================================================ # Default data source for Spark SQL (currently only supports "bigquery") -# Only available for Managed Spark runtime version 2.3 # MANAGED_SPARK_CONNECT_DEFAULT_DATASOURCE="bigquery" # ============================================================================ diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index b4ff259f..323ef5e0 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -127,16 +127,16 @@ jobs: exit 1 fi echo "pr_head_sha=${PR_HEAD_SHA}" >> "$GITHUB_OUTPUT" - for os in ubuntu-latest macos-latest windows-latest; do + for job in ubuntu-latest macos-latest windows-latest "ubuntu-latest, runtime 4.0"; do gh api "repos/${GITHUB_REPOSITORY}/statuses/${PR_HEAD_SHA}" \ -f state="pending" \ -f target_url="${RUN_URL}" \ -f description="Integration tests running via /gcbrun" \ - -f context="Integration Tests / Run integration tests (${os})" + -f context="Integration Tests / Run integration tests (${job})" done integration-test: - name: Run integration tests (${{ matrix.os }}) + name: Run integration tests (${{ matrix.os }}${{ matrix.runtime != 'default' && format(', runtime {0}', matrix.runtime) || '' }}) # `ControlCheck.dominates` (`ControlChecks.qll`) traverses `needs:`, so # `authorize-gcbrun`'s `BashCommentVsHeadDateCheck` dominates this job's checkout. needs: [authorize-gcbrun] @@ -152,6 +152,15 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] + # "default" leaves the runtime to the library, with the [client] + # extra's pyspark-client. Other runtimes need their own pyspark, and + # their own Python for Python UDFs; they run on one OS only. + runtime: ["default"] + include: + - os: "ubuntu-latest" + runtime: "4.0" + python-version: "3.13" + pyspark: "pyspark-client~=4.2.0" # Run on push to main, manual dispatch, non-Dependabot same-repo pull requests, # or authorized /gcbrun PR comments (`needs.authorize-gcbrun.result == 'success'`). @@ -186,7 +195,7 @@ jobs: - name: Setup Python uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: - python-version: "3.12" + python-version: ${{ matrix.python-version || '3.12' }} - name: Set up uv uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 @@ -200,13 +209,22 @@ jobs: ~/.cache/uv ~/Library/Caches/uv ~/AppData/Local/uv/cache - key: ${{ runner.os }}-uv-integration-${{ hashFiles('setup.py', 'requirements-test.txt') }} + key: ${{ runner.os }}-uv-integration-${{ matrix.runtime }}-${{ hashFiles('setup.py', 'requirements-test.txt') }} restore-keys: | - ${{ runner.os }}-uv-integration- + ${{ runner.os }}-uv-integration-${{ matrix.runtime }}- ${{ runner.os }}-uv- + # Without the [client] extra when the runtime needs a different pyspark, + # since the extra would pin the default runtime's. - name: Install dependencies - run: uv pip install --system -e '.[client,interactive]' -r requirements-test.txt + env: + PYSPARK: ${{ matrix.pyspark }} + run: | + if [[ -n "$PYSPARK" ]]; then + uv pip install --system -e '.[interactive]' "$PYSPARK" -r requirements-test.txt + else + uv pip install --system -e '.[client,interactive]' -r requirements-test.txt + fi - name: Authenticate to Google Cloud uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 @@ -237,6 +255,7 @@ jobs: GOOGLE_CLOUD_REGION: ${{ vars.GCP_REGION || secrets.GCP_REGION || 'us-central1,us-east1,us-west1' }} MANAGED_SPARK_CONNECT_SUBNET: ${{ vars.GCP_SUBNET || secrets.GCP_SUBNET }} MANAGED_SPARK_CONNECT_AUTH_TYPE: "SERVICE_ACCOUNT" + MANAGED_SPARK_CONNECT_TEST_RUNTIME_VERSION: ${{ matrix.runtime != 'default' && matrix.runtime || '' }} run: | python -m pytest tests/integration/ -v --tb=short -x @@ -247,7 +266,7 @@ jobs: JOB_STATUS: ${{ job.status }} PR_HEAD_SHA: ${{ needs.authorize-gcbrun.outputs.pr_head_sha }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - STATUS_CONTEXT: Integration Tests / Run integration tests (${{ matrix.os }}) + STATUS_CONTEXT: Integration Tests / Run integration tests (${{ matrix.os }}${{ matrix.runtime != 'default' && format(', runtime {0}', matrix.runtime) || '' }}) run: | if [[ "$JOB_STATUS" == "success" ]]; then STATE="success" diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 9451e045..94b7992c 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -126,13 +126,12 @@ jobs: export JAVA_HOME="${JAVA_HOME_17_X64:-${JAVA_HOME_17_ARM64:-${JAVA_HOME_21_X64:-${JAVA_HOME_21_ARM64:-$JAVA_HOME}}}}" python -m pytest tests/local/ -v --tb=short - # The [client] and [spark] extras pin the pyspark minor version of the - # default Managed Spark runtime. Users of other runtimes, and environments - # that preinstall pyspark, can have a newer one, so also run the full suite - # against the newest pyspark-client. On Python 3.13 rather than 3.12, which - # the main matrix already covers with the pinned pyspark. - test-newest-pyspark: - name: Run unit tests (ubuntu-latest, Python 3.13, newest pyspark-client) + # The main matrix covers the default runtime (3.0): Python 3.12 with the + # pyspark the [client] and [spark] extras pin. This covers runtime 4.0's + # Python and pyspark, installed without an extra, as users of runtime 4.0 + # install it. + test-runtime-4-0: + name: Run unit tests (ubuntu-latest, Python 3.13, runtime 4.0 pyspark-client) runs-on: ubuntu-latest timeout-minutes: 15 defaults: @@ -155,7 +154,7 @@ jobs: # No extra: it would pin pyspark-client back to the default runtime's. - name: Install dependencies - run: uv pip install --system -e '.[interactive]' pyspark-client -r requirements-test.txt + run: uv pip install --system -e '.[interactive]' 'pyspark-client~=4.2.0' -r requirements-test.txt # Serially, without xdist: pyspark 4.2 exits on import when __main__ has # no __file__, as in xdist workers, mistaking that for running its own diff --git a/README.md b/README.md index 05c7da59..6fd613de 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,54 @@ in your code using the builder API: spark = ManagedSparkSession.builder.projectId('my-project').location('us-central1').sessionConfig(session_config).getOrCreate() ``` +### Runtime versions + +Sessions run on Managed Spark runtime 3.0 unless you choose another version +with `runtimeVersion()` or `Session.runtime_config.version`. Spark Connect +supports clients whose PySpark minor version is the same as or older than the +runtime's Spark version, but not newer, and Python UDFs require matching Python +minor versions: + +| Runtime | Spark | Python | Install | +|---------|-------|--------|------------------------------------------------------------------| +| 3.0 | 4.0 | 3.12 | `pip install 'google-cloud-spark-connect[client]'` | +| 4.0 | 4.2 | 3.13 | `pip install google-cloud-spark-connect 'pyspark-client~=4.2.0'` | + +The `client` and `spark` extras install PySpark 4.0, which works with both +runtimes, but only has the APIs of Spark 4.0. To use newer APIs on runtime 4.0, +install a newer PySpark instead of the extras, for example in a fresh Python +3.13 environment: + +```sh +pip install 'google-cloud-spark-connect[interactive]' 'pyspark-client~=4.2.0' +``` + +```python +from google.cloud.managed_spark_connect import ManagedSparkSession + +spark = ( + ManagedSparkSession.builder.runtimeVersion('4.0') + .subnetwork('default') + .getOrCreate() +) +``` + +The client warns when the installed PySpark is newer than the session runtime's +Spark, or the Python version does not match. + +Runtime 4.0 notes (see the +[Managed Spark runtime 4.0 documentation](https://docs.cloud.google.com/managed-spark/docs/concepts/versions/spark-runtime-4.0)): + +* **Subnetwork required:** Runtime 4.0 requires an IPv4-only subnetwork via + `.subnetwork()`, `MANAGED_SPARK_CONNECT_SUBNET`, or `.runtimeProfile()`. +* **Default catalog:** Runtime 4.0 defaults `spark.sql.defaultCatalog` to the + BigLake Iceberg `default_catalog` (with no default namespace, so unqualified + single-part DDL like `CREATE TABLE foo` must either qualify the namespace or + pass `.config('spark.sql.defaultCatalog', 'spark_catalog')` to restore + `spark_catalog`). +* **Remote PyPI installation:** `%dpip` and `spark.addArtifacts(..., pypi=True)` + are not yet supported on runtime 4.0. + ### Builder Configuration The `ManagedSparkSession.builder` provides a fluent API to configure the session. Below is a list of available methods: @@ -119,7 +167,7 @@ The `ManagedSparkSession.builder` provides a fluent API to configure the session | `location(location)` | Sets the Google Cloud region. | | `projectId(project_id)` | Sets the Google Cloud project ID. | | `runtimeProfile(profile)` | Sets the Runtime Profile to use. | -| `runtimeVersion(version)` | Sets the Managed Spark runtime version (e.g., "3.0"). | +| `runtimeVersion(version)` | Sets the Managed Spark runtime version (e.g., `"3.0"`, `"4.0"`). | | `serviceAccount(account)` | Sets the service account for the session. | | `sessionTemplate(profile)` | Alias for `runtimeProfile(profile)`. | | `subnetwork(subnet)` | Sets the subnetwork URI for the session. | diff --git a/google/cloud/managed_spark_connect/_runtimes.py b/google/cloud/managed_spark_connect/_runtimes.py index 7163bc85..79d41f5a 100644 --- a/google/cloud/managed_spark_connect/_runtimes.py +++ b/google/cloud/managed_spark_connect/_runtimes.py @@ -45,6 +45,7 @@ class RuntimeInfo: # without the compatibility warnings. _RUNTIMES = { (3, 0): RuntimeInfo(spark=(4, 0), python=(3, 12)), + (4, 0): RuntimeInfo(spark=(4, 2), python=(3, 13)), } diff --git a/google/cloud/managed_spark_connect/session.py b/google/cloud/managed_spark_connect/session.py index 61f5256d..95ff3219 100644 --- a/google/cloud/managed_spark_connect/session.py +++ b/google/cloud/managed_spark_connect/session.py @@ -532,6 +532,24 @@ def _format_create_session_error( "sessionTemplates/{template}'." ) + if isinstance(exc, InvalidArgument) and ( + "subnetwork must be specified" in base_msg.lower() + or "specify a subnetwork" in base_msg.lower() + ): + extra_parts.append( + "Pass .subnetwork('') (for example .subnetwork('default'))," + " set MANAGED_SPARK_CONNECT_SUBNET, or configure a subnetwork in" + " .runtimeProfile('')." + ) + elif ( + isinstance(exc, InvalidArgument) + and "stack type, which is not supported" in base_msg.lower() + ): + extra_parts.append( + "Managed Spark runtime 4.0 requires an IPv4-only subnetwork; pass" + " one with .subnetwork('') or MANAGED_SPARK_CONNECT_SUBNET." + ) + if base_msg and extra_parts and base_msg[-1] not in ".!?:;": base_msg = f"{base_msg}." parts = [p for p in (base_msg, *extra_parts) if p] @@ -1739,9 +1757,12 @@ def _get_session_config(self): match default_datasource: case "bigquery": # Merge default configs with existing properties, - # user configs take precedence + # user configs take precedence. Explicitly set + # spark.sql.defaultCatalog so runtime 4.0 does not replace + # spark_catalog with the Lakehouse default_catalog. for k, v in { "spark.sql.catalog.spark_catalog": "com.google.cloud.spark.bigquery.BigQuerySparkSessionCatalog", + "spark.sql.defaultCatalog": "spark_catalog", "spark.sql.sources.default": "bigquery", }.items(): if k not in session_config.runtime_config.properties: diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 9ac5049b..f0c0218d 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -169,6 +169,25 @@ def test_project(): return os.getenv("GOOGLE_CLOUD_PROJECT") +@pytest.fixture(scope="session", autouse=True) +def _test_runtime_version(): + """Runs every session on MANAGED_SPARK_CONNECT_TEST_RUNTIME_VERSION, if set. + + Overrides the default rather than setting runtimeVersion() on each builder, + so that tests asserting the default runtime version check this one instead. + Install the pyspark that runtime needs (see _runtimes.py) along with it. + """ + runtime_version = os.getenv("MANAGED_SPARK_CONNECT_TEST_RUNTIME_VERSION") + if not runtime_version: + yield ManagedSparkSession._DEFAULT_RUNTIME_VERSION + return + with pytest.MonkeyPatch.context() as mp: + mp.setattr( + ManagedSparkSession, "_DEFAULT_RUNTIME_VERSION", runtime_version + ) + yield runtime_version + + @pytest.fixture(scope="session") def _region_state(test_subnet): regions_spec = os.getenv("GOOGLE_CLOUD_REGION") or ",".join( diff --git a/tests/integration/managed_spark_magics/test_magics.py b/tests/integration/managed_spark_magics/test_magics.py index 6e172821..dc9b72ef 100644 --- a/tests/integration/managed_spark_magics/test_magics.py +++ b/tests/integration/managed_spark_magics/test_magics.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os + import pytest @@ -33,6 +35,20 @@ def ipython_shell(connect_session): TerminalInteractiveShell.clear_instance() +# Runtime 4.0 manages its Python environment with Pixi instead of pip, and +# server-side DataprocCommandRunner support for Pixi is not rolled out yet. +# Non-strict so CI stays green once the server-side fix lands. +_xfail_on_runtime_4 = pytest.mark.xfail( + os.getenv("MANAGED_SPARK_CONNECT_TEST_RUNTIME_VERSION", "").startswith( + "4." + ), + reason="PyPI package installation in Spark Connect sessions is not" + " available yet on runtime 4.0", + strict=False, +) + + +@_xfail_on_runtime_4 def test_dpip_install_success(connect_session, ipython_shell, capsys): """Test installing PyPI packages with %dpip and running a UDF on remote executors.""" ipython_shell.run_line_magic("dpip", "install roman numpy") @@ -58,6 +74,7 @@ def to_roman(number): assert df_result[0]["roman"] == "MDCLXVI" +@_xfail_on_runtime_4 def test_dpip_install_failure(ipython_shell): """Test error message on remote PyPI installation failure.""" with pytest.raises( diff --git a/tests/integration/test_session.py b/tests/integration/test_session.py index 67ea6b4f..00d6323b 100644 --- a/tests/integration/test_session.py +++ b/tests/integration/test_session.py @@ -51,16 +51,21 @@ def test_create_spark_session_with_default_notebook_behavior( "id", "name" ) assert str(df) == "DataFrame[id: bigint, name: string]" - connect_session.sql("DROP TABLE IF EXISTS FOO") - connect_session.sql("CREATE TABLE FOO (bar long, baz long) USING PARQUET") + # Qualified, since the default catalog differs between runtimes: runtime + # 4.0 defaults to a lakehouse catalog with no current namespace. + table = "spark_catalog.default.FOO" + connect_session.sql(f"DROP TABLE IF EXISTS {table}") + connect_session.sql( + f"CREATE TABLE {table} (bar long, baz long) USING PARQUET" + ) try: with pytest.raises(connect_exceptions.AnalysisException) as ex: connect_session.sql( - "CREATE TABLE FOO (bar long, baz long) USING PARQUET" + f"CREATE TABLE {table} (bar long, baz long) USING PARQUET" ) assert "[TABLE_OR_VIEW_ALREADY_EXISTS]" in str(ex.value) finally: - connect_session.sql("DROP TABLE IF EXISTS FOO") + connect_session.sql(f"DROP TABLE IF EXISTS {table}") assert ManagedSparkSession._active_s8s_session_uuid is not None diff --git a/tests/unit/test_init.py b/tests/unit/test_init.py index 78be3cba..ef235818 100644 --- a/tests/unit/test_init.py +++ b/tests/unit/test_init.py @@ -89,6 +89,25 @@ def test_no_warning_for_unknown_runtime_version(self): mock_warn.assert_not_called() + def test_python_version_mismatch_warning_for_runtime_40(self): + """Runtime 4.0 uses Python 3.13.""" + for client_python, expect_warning in [ + ((3, 12), True), + ((3, 13), False), + ]: + with self.subTest(client_python=client_python): + with mock.patch("sys.version_info", (*client_python, 0)): + with mock.patch("warnings.warn") as mock_warn: + ManagedSparkSession.Builder()._check_python_version_compatibility( + "4.0" + ) + self.assertEqual(mock_warn.called, expect_warning) + if expect_warning: + self.assertIn( + "Managed Spark runtime 4.0 uses Python 3.13", + mock_warn.call_args.args[0], + ) + def test_python_version_mismatch_warning_for_subminor_runtime(self): """Subminor runtime versions are looked up by their major.minor.""" with mock.patch("sys.version_info", (3, 11, 0)): @@ -117,9 +136,23 @@ def _check(self, runtime_version, pyspark_version, remote_only=True): return mock_warn def test_no_warning_when_spark_versions_match(self): - for pyspark_version in ["4.0.0", "4.0.5", "4.0.1.dev1"]: - with self.subTest(pyspark_version=pyspark_version): - self._check("3.0", pyspark_version).assert_not_called() + for runtime_version, pyspark_version in [ + ("3.0", "4.0.0"), + ("3.0", "4.0.5"), + ("3.0", "4.0.1.dev1"), + ("4.0", "4.2.0"), + ("4.0", "4.2.1"), + ]: + with self.subTest( + runtime_version=runtime_version, pyspark_version=pyspark_version + ): + self._check( + runtime_version, pyspark_version + ).assert_not_called() + + def test_no_warning_for_runtime_40_with_pyspark_40(self): + """The [client] extra's pyspark-client works with runtime 4.0 too.""" + self._check("4.0", "4.0.5").assert_not_called() def test_no_warning_when_pyspark_older_than_runtime(self): """Spark Connect servers support older clients.""" diff --git a/tests/unit/test_runtimes.py b/tests/unit/test_runtimes.py index 31fa81e5..86f30f85 100644 --- a/tests/unit/test_runtimes.py +++ b/tests/unit/test_runtimes.py @@ -43,6 +43,10 @@ def test_known_runtime(self): _runtimes.get_runtime_info("3.0"), _runtimes.RuntimeInfo(spark=(4, 0), python=(3, 12)), ) + self.assertEqual( + _runtimes.get_runtime_info("4.0"), + _runtimes.RuntimeInfo(spark=(4, 2), python=(3, 13)), + ) self.assertEqual( _runtimes.get_runtime_info("3.0.12"), _runtimes.get_runtime_info("3.0"), diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 74a82f70..fe0f7ca6 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -1154,6 +1154,66 @@ def test_create_spark_session_with_malformed_session_template_uri( ) self.assertIsInstance(ctx.exception.__cause__, InvalidArgument) + @mock.patch("google.auth.default") + @mock.patch("google.cloud.dataproc_v1.SessionControllerClient") + def test_create_spark_session_with_missing_subnetwork_hint( + self, + mock_session_controller_client, + mock_credentials, + ): + cred = mock.MagicMock() + cred.token = "token" + mock_credentials.return_value = (cred, "") + mock_operation = mock.Mock() + mock_operation.result.side_effect = InvalidArgument( + "a subnetwork must be specified" + ) + mock_session_controller_client.return_value.create_session.return_value = ( + mock_operation + ) + + with self.assertRaises(ManagedSparkConnectException) as ctx: + ( + ManagedSparkSession.builder.projectId("test-project") + .location("us-central1") + .runtimeVersion("4.0") + .getOrCreate() + ) + + self.assertEqual( + ctx.exception.message, + "Error while creating Managed Spark Session: " + "a subnetwork must be specified. " + "Pass .subnetwork('') (for example .subnetwork('default'))," + " set MANAGED_SPARK_CONNECT_SUBNET, or configure a subnetwork in" + " .runtimeProfile('').", + ) + self.assertIsInstance(ctx.exception.__cause__, InvalidArgument) + + mock_operation.result.side_effect = InvalidArgument( + "Subnetwork projects/123/regions/us-central1/subnetworks/default" + " has dual-stack (IPV4_IPV6) stack type, which is not supported." + ) + with self.assertRaises(ManagedSparkConnectException) as stack_ctx: + ( + ManagedSparkSession.builder.projectId("test-project") + .location("us-central1") + .runtimeVersion("4.0") + .subnetwork("default") + .getOrCreate() + ) + + self.assertEqual( + stack_ctx.exception.message, + "Error while creating Managed Spark Session: " + "Subnetwork projects/123/regions/us-central1/subnetworks/default" + " has dual-stack (IPV4_IPV6) stack type, which is not supported. " + "Managed Spark runtime 4.0 requires an IPv4-only subnetwork; pass" + " one with .subnetwork('') or" + " MANAGED_SPARK_CONNECT_SUBNET.", + ) + self.assertIsInstance(stack_ctx.exception.__cause__, InvalidArgument) + @mock.patch("google.auth.default") @mock.patch("google.cloud.dataproc_v1.SessionControllerClient") @mock.patch( @@ -1495,6 +1555,12 @@ def test_create_session_with_default_datasource_env_var( ), "com.google.cloud.spark.bigquery.BigQuerySparkSessionCatalog", ) + self.assertEqual( + create_session_request.session.runtime_config.properties.get( + "spark.sql.defaultCatalog" + ), + "spark_catalog", + ) mock_logger.warning.assert_not_called() self.stopSession(mock_session_controller_client_instance, session) mock_session_controller_client_instance.create_session.reset_mock() From 718cc8d2e20341887f920b9137eeba13ad4fbf59 Mon Sep 17 00:00:00 2001 From: Dave Borowitz Date: Tue, 10 Nov 2026 17:13:53 -0800 Subject: [PATCH 2/2] fix: use DataprocCommandRunner for addArtifacts(pypi=True) and catch installer IOExceptions Route ManagedSparkSession.addArtifacts(..., pypi=True) through the same ExecuteExternalCommand / DataprocCommandRunner PipInstallPackages path used by %dpip, rather than uploading legacy add-artifacts-1729-*.json marker files. Broaden failure detection from the narrow 'Pip install failed with non-zero exit code' substring to r'(?:Pip|Pixi) install failed\b' so IOException errors (such as when pip is absent on Managed Spark runtime 4.0), InterruptedException errors, and Pixi installer failures raise instead of silently succeeding. --- .../managed_spark_connect/pypi_artifacts.py | 80 ++++++++++++++--- google/cloud/managed_spark_connect/session.py | 29 +++--- google/cloud/managed_spark_magics/magics.py | 35 +------- .../unit/managed_spark_magics/test_magics.py | 22 +++++ tests/unit/test_pypi_artifacts.py | 89 ++++++++++++++++++- tests/unit/test_session.py | 61 +++++++------ 6 files changed, 235 insertions(+), 81 deletions(-) diff --git a/google/cloud/managed_spark_connect/pypi_artifacts.py b/google/cloud/managed_spark_connect/pypi_artifacts.py index e8ed4974..c42637fe 100644 --- a/google/cloud/managed_spark_connect/pypi_artifacts.py +++ b/google/cloud/managed_spark_connect/pypi_artifacts.py @@ -1,15 +1,82 @@ import json import logging import os +import re import tempfile import uuid -from typing import TYPE_CHECKING +from typing import Any, Sequence, TYPE_CHECKING if TYPE_CHECKING: from packaging.requirements import Requirement logger = logging.getLogger(__name__) +DATAPROC_COMMAND_RUNNER = "org.apache.spark.sql.artifact.DataprocCommandRunner" +PIP_INSTALL_COMMAND = "PipInstallPackages" +PIP_INSTALL_FAILURE_MSG = "Pip install failed with non-zero exit code" +_INSTALL_FAILURE_PATTERN = re.compile(r"(?:Pip|Pixi) install failed\b") + + +def validate_pypi_requirements(packages: Sequence[str]) -> list["Requirement"]: + from packaging.requirements import Requirement + + reqs = [Requirement(p) for p in packages] + if any(len(req.specifier) == 0 for req in reqs): + logger.debug("It is recommended to pin the version of the package") + return reqs + + +def install_pypi_packages( + client: Any, + packages: Sequence[str], + *, + validate: bool = True, +) -> str: + """Install PyPI packages on the remote session via DataprocCommandRunner. + + Used by both ``ManagedSparkSession.addArtifacts(..., pypi=True)`` and the + ``%dpip`` notebook magic. + """ + unique_packages = list(dict.fromkeys(packages)) + if not unique_packages: + return "" + if validate: + validate_pypi_requirements(unique_packages) + + # Imported lazily: PySpark (and with it pyarrow) is an optional extra, so + # importing this module or loading %dpip must work without it. + import pyarrow as pa + import pyspark.sql.connect.proto as pb2 + + command = pb2.Command() + command.execute_external_command.runner = DATAPROC_COMMAND_RUNNER + command.execute_external_command.command = PIP_INSTALL_COMMAND + for index, package in enumerate(unique_packages): + command.execute_external_command.options[str(index)] = package + + _, properties, _ = client.execute_command(command) + + try: + binary_data = properties["sql_command_result"].local_relation.data + except (KeyError, AttributeError, TypeError) as e: + raise RuntimeError( + "Unexpected response structure: missing binary data." + ) from e + if not binary_data: + raise RuntimeError( + "Unexpected response structure: missing binary data." + ) + + try: + table = pa.ipc.RecordBatchStreamReader(binary_data).read_all() + output = "\n".join(table.column(0).to_pylist()) + except Exception as e: + raise RuntimeError(f"Error decoding Arrow data: {e}") from e + + if _INSTALL_FAILURE_PATTERN.search(output): + raise RuntimeError(output) + return output + class PyPiArtifacts: """ @@ -17,17 +84,8 @@ class PyPiArtifacts: that Spark Connect server understands """ - @staticmethod - def __try_parsing_package(packages: set[str]) -> list["Requirement"]: - from packaging.requirements import Requirement - - reqs = [Requirement(p) for p in packages] - if 0 in [len(req.specifier) for req in reqs]: - logger.debug("It is recommended to pin the version of the package") - return reqs - def __init__(self, packages: set[str]): - self.requirements = PyPiArtifacts.__try_parsing_package(packages) + self.requirements = validate_pypi_requirements(list(packages)) def write_packages_config(self, s8s_session_uuid: str) -> str: """ diff --git a/google/cloud/managed_spark_connect/session.py b/google/cloud/managed_spark_connect/session.py index 95ff3219..6a6e73c9 100644 --- a/google/cloud/managed_spark_connect/session.py +++ b/google/cloud/managed_spark_connect/session.py @@ -90,7 +90,10 @@ from google.cloud.managed_spark_connect.instrumentation._core import ( _CURRENT_SPAN, ) -from google.cloud.managed_spark_connect.pypi_artifacts import PyPiArtifacts +from google.cloud.managed_spark_connect.pypi_artifacts import ( + PyPiArtifacts, + install_pypi_packages, +) from google.cloud.dataproc_v1 import ( AuthenticationConfig, CreateSessionRequest, @@ -2462,16 +2465,22 @@ def addArtifacts( "Cannot add PyPI artifacts: no active Managed Spark" " session." ) - artifacts = PyPiArtifacts(set(artifact)) - logger.debug("Making addArtifact call to install packages") - self.addArtifact( - _normalize_artifact_path( - artifacts.write_packages_config( - self._active_s8s_session_uuid - ) - ), - file=True, + if not artifact: + return + from packaging.requirements import InvalidRequirement + + logger.debug( + "Installing PyPI packages via DataprocCommandRunner: %s", + artifact, ) + try: + install_pypi_packages(self.client, artifact, validate=True) + except InvalidRequirement: + raise + except Exception as e: + raise ManagedSparkConnectException( + f"Failed to install PyPI packages: {e}" + ) from e else: super().addArtifacts( *(_normalize_artifact_path(p) for p in artifact), diff --git a/google/cloud/managed_spark_magics/magics.py b/google/cloud/managed_spark_magics/magics.py index 9ad075c9..5cb18764 100644 --- a/google/cloud/managed_spark_magics/magics.py +++ b/google/cloud/managed_spark_magics/magics.py @@ -14,7 +14,6 @@ """Managed Spark magic implementations.""" -import re import shlex from IPython.core.magic import (Magics, magics_class, line_magic) @@ -48,10 +47,6 @@ def dpip(self, line): print(f"Installing packages: {packages}") output = self._run_command(packages, session) - failure_match = re.search(self.PIP_INSTALL_FAILURE_MSG, output) - if failure_match: - raise RuntimeError(output) - print(output) print("Finished installing packages.") except Exception as e: @@ -94,30 +89,8 @@ def _check_preconditions(self, args): return packages, session def _run_command(self, packages, session): - # Imported here rather than at module scope, like the package's other - # pyspark imports: PySpark (and with it pyarrow) is an optional extra, - # and this extension must still load without it. - import pyarrow as pa - import pyspark.sql.connect.proto as pb2 - - command = pb2.Command() - command.execute_external_command.runner = self.DATAPROC_COMMAND_RUNNER - command.execute_external_command.command = self.PIP_INSTALL_COMMAND - - for index, package in enumerate(packages): - command.execute_external_command.options[str(index)] = package - - _, properties, _ = session.client.execute_command(command) + from google.cloud.managed_spark_connect.pypi_artifacts import ( + install_pypi_packages, + ) - try: - binary_data = properties["sql_command_result"].local_relation.data - - # decode the Arrow stream and return the output - table = pa.ipc.RecordBatchStreamReader(binary_data).read_all() - return "\n".join(table.column(0).to_pylist()) - except (KeyError, AttributeError) as e: - raise RuntimeError( - "Unexpected response structure: missing binary data." - ) from e - except Exception as e: - raise RuntimeError(f"Error decoding Arrow data: {e}") from e + return install_pypi_packages(session.client, packages, validate=False) diff --git a/tests/unit/managed_spark_magics/test_magics.py b/tests/unit/managed_spark_magics/test_magics.py index f8d0435c..825564c9 100644 --- a/tests/unit/managed_spark_magics/test_magics.py +++ b/tests/unit/managed_spark_magics/test_magics.py @@ -156,6 +156,28 @@ def test_dpip_install_failure(self): ): self.magics.dpip("install non-existent-package") + def test_dpip_install_io_exception_failure(self): + mock_session = mock.Mock(spec=ManagedSparkSession) + mock_session.client = mock.Mock() + properties = mock.Mock() + properties.sql_command_result.local_relation.data = ( + self._create_mock_arrow_binary( + ["Pip install failed: IOException during pip install process"] + ) + ) + mock_session.client.execute_command.return_value = ( + None, + {"sql_command_result": properties.sql_command_result}, + None, + ) + self.shell.user_ns["spark"] = mock_session + + with self.assertRaisesRegex( + RuntimeError, + "Failed to install packages: Pip install failed: IOException", + ): + self.magics.dpip("install roman") + def test_dpip_unexpected_response(self): mock_session = mock.Mock(spec=ManagedSparkSession) mock_session.client = mock.Mock() diff --git a/tests/unit/test_pypi_artifacts.py b/tests/unit/test_pypi_artifacts.py index 580d6646..71ae4255 100644 --- a/tests/unit/test_pypi_artifacts.py +++ b/tests/unit/test_pypi_artifacts.py @@ -9,10 +9,28 @@ import unittest import zipfile +from unittest import mock + from packaging.requirements import InvalidRequirement +import pyarrow as pa +import pyspark.sql.connect.proto as pb2 import pytest -from google.cloud.managed_spark_connect.pypi_artifacts import PyPiArtifacts +from google.cloud.managed_spark_connect.pypi_artifacts import ( + DATAPROC_COMMAND_RUNNER, + PIP_INSTALL_COMMAND, + PyPiArtifacts, + install_pypi_packages, +) + + +def _arrow_lines(lines: list[str]) -> bytes: + schema = pa.schema([pa.field("output", pa.string())]) + table = pa.Table.from_arrays([lines], schema=schema) + sink = pa.BufferOutputStream() + with pa.ipc.RecordBatchStreamWriter(sink, table.schema) as writer: + writer.write_table(table) + return sink.getvalue() class PyPiArtifactsTest(unittest.TestCase): @@ -58,6 +76,75 @@ def test_validate_file_name_pattern(self): self.assertTrue(file_name.startswith("add-artifacts-1729-")) self.assertTrue(file_name.endswith(".json")) + def test_install_pypi_packages_executes_command_runner(self): + client = mock.Mock() + props = mock.Mock() + props.sql_command_result.local_relation.data = _arrow_lines( + ["Successfully installed spacy-1.2.3 roman-4.0"] + ) + client.execute_command.return_value = ( + None, + {"sql_command_result": props.sql_command_result}, + None, + ) + + output = install_pypi_packages( + client, ["spacy==1.2.3", "roman", "spacy==1.2.3"] + ) + self.assertEqual(output, "Successfully installed spacy-1.2.3 roman-4.0") + client.execute_command.assert_called_once() + cmd = client.execute_command.call_args[0][0] + self.assertIsInstance(cmd, pb2.Command) + self.assertEqual( + cmd.execute_external_command.runner, DATAPROC_COMMAND_RUNNER + ) + self.assertEqual( + cmd.execute_external_command.command, PIP_INSTALL_COMMAND + ) + self.assertEqual( + dict(cmd.execute_external_command.options), + {"0": "spacy==1.2.3", "1": "roman"}, + ) + + def test_install_pypi_packages_raises_on_io_and_pixi_failures(self): + for failure_line in ( + "Pip install failed with non-zero exit code: 1", + "Pip install failed: IOException during pip install process", + "Pip install failed: InterruptedException during pip install process", + "Pixi install failed with non-zero exit code: 1", + ): + with self.subTest(failure_line=failure_line): + client = mock.Mock() + props = mock.Mock() + props.sql_command_result.local_relation.data = _arrow_lines( + [failure_line] + ) + client.execute_command.return_value = ( + None, + {"sql_command_result": props.sql_command_result}, + None, + ) + with self.assertRaisesRegex(RuntimeError, failure_line): + install_pypi_packages(client, ["roman"]) + + def test_install_pypi_packages_missing_or_null_properties(self): + empty_result = mock.Mock() + empty_result.local_relation.data = b"" + for bad_props in ( + None, + {}, + {"sql_command_result": None}, + {"sql_command_result": empty_result}, + ): + with self.subTest(bad_props=bad_props): + client = mock.Mock() + client.execute_command.return_value = (None, bad_props, None) + with self.assertRaisesRegex( + RuntimeError, + "Unexpected response structure: missing binary data.", + ): + install_pypi_packages(client, ["roman"]) + class PackageBuildWhiteoutTest(unittest.TestCase): diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index fe0f7ca6..937b992e 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -272,7 +272,6 @@ def test_pypi_add_artifacts( ) session = ManagedSparkSession.builder.getOrCreate() self.assertTrue(isinstance(session, ManagedSparkSession)) - session.addArtifact = mock.MagicMock() # Setting two flags together with self.assertRaisesRegex( @@ -281,38 +280,44 @@ def test_pypi_add_artifacts( ): session.addArtifacts("abc.txt", file=True, pypi=True) - # Propagate error - session.addArtifact.side_effect = Exception("Error installing") - with self.assertRaisesRegex( - Exception, - "Error installing", - ): + with mock.patch.object( + session_module, "install_pypi_packages" + ) as mock_install_pypi: + # Propagate error wrapped in ManagedSparkConnectException + mock_install_pypi.side_effect = RuntimeError("Error installing") + with self.assertRaisesRegex( + ManagedSparkConnectException, + "Failed to install PyPI packages: Error installing", + ): + session.addArtifacts("spacy", pypi=True) + mock_install_pypi.side_effect = None + + # Do install if earlier addArtifacts resulted in failure session.addArtifacts("spacy", pypi=True) - session.addArtifact.side_effect = None + mock_install_pypi.assert_called_with( + session.client, ("spacy",), validate=True + ) + self.assertEqual(mock_install_pypi.call_count, 2) - # Do install if earlier add artifact resulted in failure - session.addArtifacts("spacy", pypi=True) - self.assertEqual(session.addArtifact.call_count, 2) + # Multiple packages + session.addArtifacts("spacy==1.2.3", "spacy", pypi=True) + mock_install_pypi.assert_called_with( + session.client, ("spacy==1.2.3", "spacy"), validate=True + ) + self.assertEqual(mock_install_pypi.call_count, 3) - # test multiple packages, when already installed - session.addArtifacts("spacy==1.2.3", "spacy", pypi=True) - self.assertEqual(session.addArtifact.call_count, 3) + # Empty package tuple is a no-op + session.addArtifacts(pypi=True) + self.assertEqual(mock_install_pypi.call_count, 3) - # Windows drive-letter paths are converted to file:// URIs before - # delegating to PySpark's ArtifactManager._parse_artifacts (both via - # addArtifacts/addArtifact and via pypi=True). - with mock.patch.object( - session_module.PyPiArtifacts, - "write_packages_config", - return_value=r"C:\Users\runner\AppData\Local\Temp\add-artifacts-1729.json", - ): - session.addArtifacts("spacy", pypi=True) - session.addArtifact.assert_called_with( - "file:///C:/Users/runner/AppData/Local/Temp/add-artifacts-1729.json", - file=True, - ) + # Invalid requirement format still raises InvalidRequirement directly + from packaging.requirements import InvalidRequirement - del session.addArtifact + with self.assertRaises(InvalidRequirement): + session.addArtifacts("pypi://spacy:23", pypi=True) + + # Windows drive-letter paths are converted to file:// URIs before + # delegating to PySpark's ArtifactManager._parse_artifacts. with mock.patch( "pyspark.sql.connect.session.SparkSession.addArtifacts" ) as mock_super_add_artifacts: