Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ GOOGLE_APPLICATION_CREDENTIALS="/path/to/your/service-account-key.json"
# ============================================================================

# Default data source for Spark SQL (currently only supports "bigquery")
# Only available for Managed Spark runtime version 2.3
# MANAGED_SPARK_CONNECT_DEFAULT_DATASOURCE="bigquery"

# ============================================================================
Expand Down
35 changes: 27 additions & 8 deletions .github/workflows/integration-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -127,16 +127,16 @@ jobs:
exit 1
fi
echo "pr_head_sha=${PR_HEAD_SHA}" >> "$GITHUB_OUTPUT"
for os in ubuntu-latest macos-latest windows-latest; do
for job in ubuntu-latest macos-latest windows-latest "ubuntu-latest, runtime 4.0"; do
gh api "repos/${GITHUB_REPOSITORY}/statuses/${PR_HEAD_SHA}" \
-f state="pending" \
-f target_url="${RUN_URL}" \
-f description="Integration tests running via /gcbrun" \
-f context="Integration Tests / Run integration tests (${os})"
-f context="Integration Tests / Run integration tests (${job})"
done

integration-test:
name: Run integration tests (${{ matrix.os }})
name: Run integration tests (${{ matrix.os }}${{ matrix.runtime != 'default' && format(', runtime {0}', matrix.runtime) || '' }})
# `ControlCheck.dominates` (`ControlChecks.qll`) traverses `needs:`, so
# `authorize-gcbrun`'s `BashCommentVsHeadDateCheck` dominates this job's checkout.
needs: [authorize-gcbrun]
Expand All @@ -152,6 +152,15 @@ jobs:
fail-fast: false
matrix:
os: ["ubuntu-latest", "macos-latest", "windows-latest"]
# "default" leaves the runtime to the library, with the [client]
# extra's pyspark-client. Other runtimes need their own pyspark, and
# their own Python for Python UDFs; they run on one OS only.
runtime: ["default"]
include:
- os: "ubuntu-latest"
runtime: "4.0"
python-version: "3.13"
pyspark: "pyspark-client~=4.2.0"

# Run on push to main, manual dispatch, non-Dependabot same-repo pull requests,
# or authorized /gcbrun PR comments (`needs.authorize-gcbrun.result == 'success'`).
Expand Down Expand Up @@ -186,7 +195,7 @@ jobs:
- name: Setup Python
uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
with:
python-version: "3.12"
python-version: ${{ matrix.python-version || '3.12' }}

- name: Set up uv
uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
Expand All @@ -200,13 +209,22 @@ jobs:
~/.cache/uv
~/Library/Caches/uv
~/AppData/Local/uv/cache
key: ${{ runner.os }}-uv-integration-${{ hashFiles('setup.py', 'requirements-test.txt') }}
key: ${{ runner.os }}-uv-integration-${{ matrix.runtime }}-${{ hashFiles('setup.py', 'requirements-test.txt') }}
restore-keys: |
${{ runner.os }}-uv-integration-
${{ runner.os }}-uv-integration-${{ matrix.runtime }}-
${{ runner.os }}-uv-

# Without the [client] extra when the runtime needs a different pyspark,
# since the extra would pin the default runtime's.
- name: Install dependencies
run: uv pip install --system -e '.[client,interactive]' -r requirements-test.txt
env:
PYSPARK: ${{ matrix.pyspark }}
run: |
if [[ -n "$PYSPARK" ]]; then
uv pip install --system -e '.[interactive]' "$PYSPARK" -r requirements-test.txt
else
uv pip install --system -e '.[client,interactive]' -r requirements-test.txt
fi

- name: Authenticate to Google Cloud
uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3
Expand Down Expand Up @@ -237,6 +255,7 @@ jobs:
GOOGLE_CLOUD_REGION: ${{ vars.GCP_REGION || secrets.GCP_REGION || 'us-central1,us-east1,us-west1' }}
MANAGED_SPARK_CONNECT_SUBNET: ${{ vars.GCP_SUBNET || secrets.GCP_SUBNET }}
MANAGED_SPARK_CONNECT_AUTH_TYPE: "SERVICE_ACCOUNT"
MANAGED_SPARK_CONNECT_TEST_RUNTIME_VERSION: ${{ matrix.runtime != 'default' && matrix.runtime || '' }}
run: |
python -m pytest tests/integration/ -v --tb=short -x

Expand All @@ -247,7 +266,7 @@ jobs:
JOB_STATUS: ${{ job.status }}
PR_HEAD_SHA: ${{ needs.authorize-gcbrun.outputs.pr_head_sha }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
STATUS_CONTEXT: Integration Tests / Run integration tests (${{ matrix.os }})
STATUS_CONTEXT: Integration Tests / Run integration tests (${{ matrix.os }}${{ matrix.runtime != 'default' && format(', runtime {0}', matrix.runtime) || '' }})
run: |
if [[ "$JOB_STATUS" == "success" ]]; then
STATE="success"
Expand Down
15 changes: 7 additions & 8 deletions .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -126,13 +126,12 @@ jobs:
export JAVA_HOME="${JAVA_HOME_17_X64:-${JAVA_HOME_17_ARM64:-${JAVA_HOME_21_X64:-${JAVA_HOME_21_ARM64:-$JAVA_HOME}}}}"
python -m pytest tests/local/ -v --tb=short

# The [client] and [spark] extras pin the pyspark minor version of the
# default Managed Spark runtime. Users of other runtimes, and environments
# that preinstall pyspark, can have a newer one, so also run the full suite
# against the newest pyspark-client. On Python 3.13 rather than 3.12, which
# the main matrix already covers with the pinned pyspark.
test-newest-pyspark:
name: Run unit tests (ubuntu-latest, Python 3.13, newest pyspark-client)
# The main matrix covers the default runtime (3.0): Python 3.12 with the
# pyspark the [client] and [spark] extras pin. This covers runtime 4.0's
# Python and pyspark, installed without an extra, as users of runtime 4.0
# install it.
test-runtime-4-0:
name: Run unit tests (ubuntu-latest, Python 3.13, runtime 4.0 pyspark-client)
runs-on: ubuntu-latest
timeout-minutes: 15
defaults:
Expand All @@ -155,7 +154,7 @@ jobs:

# No extra: it would pin pyspark-client back to the default runtime's.
- name: Install dependencies
run: uv pip install --system -e '.[interactive]' pyspark-client -r requirements-test.txt
run: uv pip install --system -e '.[interactive]' 'pyspark-client~=4.2.0' -r requirements-test.txt

# Serially, without xdist: pyspark 4.2 exits on import when __main__ has
# no __file__, as in xdist workers, mistaking that for running its own
Expand Down
50 changes: 49 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,54 @@ in your code using the builder API:
spark = ManagedSparkSession.builder.projectId('my-project').location('us-central1').sessionConfig(session_config).getOrCreate()
```

### Runtime versions

Sessions run on Managed Spark runtime 3.0 unless you choose another version
with `runtimeVersion()` or `Session.runtime_config.version`. Spark Connect
supports clients whose PySpark minor version is the same as or older than the
runtime's Spark version, but not newer, and Python UDFs require matching Python
minor versions:

| Runtime | Spark | Python | Install |
|---------|-------|--------|------------------------------------------------------------------|
| 3.0 | 4.0 | 3.12 | `pip install 'google-cloud-spark-connect[client]'` |
| 4.0 | 4.2 | 3.13 | `pip install google-cloud-spark-connect 'pyspark-client~=4.2.0'` |

The `client` and `spark` extras install PySpark 4.0, which works with both
runtimes, but only has the APIs of Spark 4.0. To use newer APIs on runtime 4.0,
install a newer PySpark instead of the extras, for example in a fresh Python
3.13 environment:

```sh
pip install 'google-cloud-spark-connect[interactive]' 'pyspark-client~=4.2.0'
```

```python
from google.cloud.managed_spark_connect import ManagedSparkSession

spark = (
ManagedSparkSession.builder.runtimeVersion('4.0')
.subnetwork('default')
.getOrCreate()
)
```

The client warns when the installed PySpark is newer than the session runtime's
Spark, or the Python version does not match.

Runtime 4.0 notes (see the
[Managed Spark runtime 4.0 documentation](https://docs.cloud.google.com/managed-spark/docs/concepts/versions/spark-runtime-4.0)):

* **Subnetwork required:** Runtime 4.0 requires an IPv4-only subnetwork via
`.subnetwork()`, `MANAGED_SPARK_CONNECT_SUBNET`, or `.runtimeProfile()`.
* **Default catalog:** Runtime 4.0 defaults `spark.sql.defaultCatalog` to the
BigLake Iceberg `default_catalog` (with no default namespace, so unqualified
single-part DDL like `CREATE TABLE foo` must either qualify the namespace or
pass `.config('spark.sql.defaultCatalog', 'spark_catalog')` to restore
`spark_catalog`).
* **Remote PyPI installation:** `%dpip` and `spark.addArtifacts(..., pypi=True)`
are not yet supported on runtime 4.0.

### Builder Configuration

The `ManagedSparkSession.builder` provides a fluent API to configure the session. Below is a list of available methods:
Expand All @@ -119,7 +167,7 @@ The `ManagedSparkSession.builder` provides a fluent API to configure the session
| `location(location)` | Sets the Google Cloud region. |
| `projectId(project_id)` | Sets the Google Cloud project ID. |
| `runtimeProfile(profile)` | Sets the Runtime Profile to use. |
| `runtimeVersion(version)` | Sets the Managed Spark runtime version (e.g., "3.0"). |
| `runtimeVersion(version)` | Sets the Managed Spark runtime version (e.g., `"3.0"`, `"4.0"`). |
| `serviceAccount(account)` | Sets the service account for the session. |
| `sessionTemplate(profile)` | Alias for `runtimeProfile(profile)`. |
| `subnetwork(subnet)` | Sets the subnetwork URI for the session. |
Expand Down
1 change: 1 addition & 0 deletions google/cloud/managed_spark_connect/_runtimes.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ class RuntimeInfo:
# without the compatibility warnings.
_RUNTIMES = {
(3, 0): RuntimeInfo(spark=(4, 0), python=(3, 12)),
(4, 0): RuntimeInfo(spark=(4, 2), python=(3, 13)),
}


Expand Down
80 changes: 69 additions & 11 deletions google/cloud/managed_spark_connect/pypi_artifacts.py
Original file line number Diff line number Diff line change
@@ -1,33 +1,91 @@
import json
import logging
import os
import re
import tempfile
import uuid
from typing import TYPE_CHECKING
from typing import Any, Sequence, TYPE_CHECKING

if TYPE_CHECKING:
from packaging.requirements import Requirement

logger = logging.getLogger(__name__)

DATAPROC_COMMAND_RUNNER = "org.apache.spark.sql.artifact.DataprocCommandRunner"
PIP_INSTALL_COMMAND = "PipInstallPackages"
PIP_INSTALL_FAILURE_MSG = "Pip install failed with non-zero exit code"
_INSTALL_FAILURE_PATTERN = re.compile(r"(?:Pip|Pixi) install failed\b")


def validate_pypi_requirements(packages: Sequence[str]) -> list["Requirement"]:
from packaging.requirements import Requirement

reqs = [Requirement(p) for p in packages]
if any(len(req.specifier) == 0 for req in reqs):
logger.debug("It is recommended to pin the version of the package")
return reqs


def install_pypi_packages(
client: Any,
packages: Sequence[str],
*,
validate: bool = True,
) -> str:
"""Install PyPI packages on the remote session via DataprocCommandRunner.

Used by both ``ManagedSparkSession.addArtifacts(..., pypi=True)`` and the
``%dpip`` notebook magic.
"""
unique_packages = list(dict.fromkeys(packages))
if not unique_packages:
return ""
if validate:
validate_pypi_requirements(unique_packages)

# Imported lazily: PySpark (and with it pyarrow) is an optional extra, so
# importing this module or loading %dpip must work without it.
import pyarrow as pa
import pyspark.sql.connect.proto as pb2

command = pb2.Command()
command.execute_external_command.runner = DATAPROC_COMMAND_RUNNER
command.execute_external_command.command = PIP_INSTALL_COMMAND
for index, package in enumerate(unique_packages):
command.execute_external_command.options[str(index)] = package

_, properties, _ = client.execute_command(command)

try:
binary_data = properties["sql_command_result"].local_relation.data
except (KeyError, AttributeError, TypeError) as e:
raise RuntimeError(
"Unexpected response structure: missing binary data."
) from e
if not binary_data:
raise RuntimeError(
"Unexpected response structure: missing binary data."
)

try:
table = pa.ipc.RecordBatchStreamReader(binary_data).read_all()
output = "\n".join(table.column(0).to_pylist())
except Exception as e:
raise RuntimeError(f"Error decoding Arrow data: {e}") from e

if _INSTALL_FAILURE_PATTERN.search(output):
raise RuntimeError(output)
return output


class PyPiArtifacts:
"""
This is a helper class to serialize the PYPI package installation request with a "magic" file name
that Spark Connect server understands
"""

@staticmethod
def __try_parsing_package(packages: set[str]) -> list["Requirement"]:
from packaging.requirements import Requirement

reqs = [Requirement(p) for p in packages]
if 0 in [len(req.specifier) for req in reqs]:
logger.debug("It is recommended to pin the version of the package")
return reqs

def __init__(self, packages: set[str]):
self.requirements = PyPiArtifacts.__try_parsing_package(packages)
self.requirements = validate_pypi_requirements(list(packages))

def write_packages_config(self, s8s_session_uuid: str) -> str:
"""
Expand Down
52 changes: 41 additions & 11 deletions google/cloud/managed_spark_connect/session.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,10 @@
from google.cloud.managed_spark_connect.instrumentation._core import (
_CURRENT_SPAN,
)
from google.cloud.managed_spark_connect.pypi_artifacts import PyPiArtifacts
from google.cloud.managed_spark_connect.pypi_artifacts import (
PyPiArtifacts,
install_pypi_packages,
)
from google.cloud.dataproc_v1 import (
AuthenticationConfig,
CreateSessionRequest,
Expand Down Expand Up @@ -532,6 +535,24 @@ def _format_create_session_error(
"sessionTemplates/{template}'."
)

if isinstance(exc, InvalidArgument) and (
"subnetwork must be specified" in base_msg.lower()
or "specify a subnetwork" in base_msg.lower()
):
extra_parts.append(
"Pass .subnetwork('<subnet>') (for example .subnetwork('default')),"
" set MANAGED_SPARK_CONNECT_SUBNET, or configure a subnetwork in"
" .runtimeProfile('<profile>')."
)
elif (
isinstance(exc, InvalidArgument)
and "stack type, which is not supported" in base_msg.lower()
):
extra_parts.append(
"Managed Spark runtime 4.0 requires an IPv4-only subnetwork; pass"
" one with .subnetwork('<subnet>') or MANAGED_SPARK_CONNECT_SUBNET."
)

if base_msg and extra_parts and base_msg[-1] not in ".!?:;":
base_msg = f"{base_msg}."
parts = [p for p in (base_msg, *extra_parts) if p]
Expand Down Expand Up @@ -1739,9 +1760,12 @@ def _get_session_config(self):
match default_datasource:
case "bigquery":
# Merge default configs with existing properties,
# user configs take precedence
# user configs take precedence. Explicitly set
# spark.sql.defaultCatalog so runtime 4.0 does not replace
# spark_catalog with the Lakehouse default_catalog.
for k, v in {
"spark.sql.catalog.spark_catalog": "com.google.cloud.spark.bigquery.BigQuerySparkSessionCatalog",
"spark.sql.defaultCatalog": "spark_catalog",
"spark.sql.sources.default": "bigquery",
}.items():
if k not in session_config.runtime_config.properties:
Expand Down Expand Up @@ -2441,16 +2465,22 @@ def addArtifacts(
"Cannot add PyPI artifacts: no active Managed Spark"
" session."
)
artifacts = PyPiArtifacts(set(artifact))
logger.debug("Making addArtifact call to install packages")
self.addArtifact(
_normalize_artifact_path(
artifacts.write_packages_config(
self._active_s8s_session_uuid
)
),
file=True,
if not artifact:
return
from packaging.requirements import InvalidRequirement

logger.debug(
"Installing PyPI packages via DataprocCommandRunner: %s",
artifact,
)
try:
install_pypi_packages(self.client, artifact, validate=True)
except InvalidRequirement:
raise
except Exception as e:
raise ManagedSparkConnectException(
f"Failed to install PyPI packages: {e}"
) from e
else:
super().addArtifacts(
*(_normalize_artifact_path(p) for p in artifact),
Expand Down
Loading
Loading