diff --git a/.github/workflows/vermin.yaml b/.github/workflows/vermin.yaml index 7a586705..b429bcb7 100644 --- a/.github/workflows/vermin.yaml +++ b/.github/workflows/vermin.yaml @@ -59,4 +59,4 @@ jobs: # only if the code needs a newer version, not if it would also run on an # older one. - name: Run vermin - run: vermin --no-tips --violations --eval-annotations --target=3.10- google tests setup.py + run: vermin --no-tips --violations --eval-annotations --target=3.10- google third_party scripts tests setup.py diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..16204eb5 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +graft third_party/apache_spark +global-exclude *.py[cod] __pycache__/* .wh.* diff --git a/google/cloud/managed_spark_connect/_third_party/__init__.py b/google/cloud/managed_spark_connect/_third_party/__init__.py new file mode 100644 index 00000000..01a69c77 --- /dev/null +++ b/google/cloud/managed_spark_connect/_third_party/__init__.py @@ -0,0 +1,28 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Private namespace for remapped third_party dependencies.""" + +from pathlib import Path + +# In an installed wheel, `setup.py` places `apache_spark/` directly beside this +# file. When imported directly from a source checkout (e.g. `PYTHONPATH=.` or +# pytest's rootdir shadowing a non-editable install), expose `/third_party` +# on `__path__` as well. +_HERE = Path(__file__).resolve().parent +_REPO_THIRD_PARTY = _HERE.parents[3] / "third_party" +if ( + not (_HERE / "apache_spark").is_dir() + and (_REPO_THIRD_PARTY / "apache_spark").is_dir() +): + __path__.append(str(_REPO_THIRD_PARTY)) diff --git a/scripts/regen_protos.py b/scripts/regen_protos.py new file mode 100755 index 00000000..822adf83 --- /dev/null +++ b/scripts/regen_protos.py @@ -0,0 +1,493 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Regenerates vendored Spark Connect GetStatus and Cell Monitor proto stubs. + +Usage: + # Regenerate *_pb2.py from the checked-in .proto files: + python scripts/regen_protos.py + + # Verify checked-in *_pb2.py files match the checked-in .proto files: + python scripts/regen_protos.py --check + + # Refresh .proto files, NOTICE, METADATA, and *_pb2.py from a Git ref/SHA on + # bigdataoss-internal/third_party/apache/spark (e.g. once CL 196087 merges): + python scripts/regen_protos.py --ref refs/heads/dataproc-branch-4.2.0 +""" + +import argparse +import datetime +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +from typing import Iterable, Optional, Sequence + +REPO_ROOT = Path(__file__).resolve().parents[1] +VENDOR_DIR = REPO_ROOT / "third_party" / "apache_spark" +TARGET_PACKAGE = "google.cloud.managed_spark_connect._third_party.apache_spark" +DEFAULT_SPARK_REMOTE = "sso://bigdataoss-internal/third_party/apache/spark" + +UPSTREAM_BASE_PROTO_PATH = ( + "sql/connect/common/src/main/protobuf/spark/connect/base.proto" +) +UPSTREAM_CELL_MONITOR_PROTO_PATH = ( + "sql/connect/common/src/main/protobuf/dataproc/cellmonitor/v1/" + "cell_monitor.proto" +) + +ASF_PYTHON_HEADER = """\ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modifications copyright 2026 Google LLC: +# Generated and post-processed by scripts/regen_protos.py for packaging under +# google.cloud.managed_spark_connect._third_party.apache_spark. +""" + +STATUS_PROTO_HEADER = """\ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright 2026 Google LLC: + * Extracted minimal UserContext, GetStatusRequest, GetStatusResponse, and + * SparkConnectService.GetStatus definitions from + * sql/connect/common/src/main/protobuf/spark/connect/base.proto so clients on + * PySpark 4.0+ can invoke GetStatus without pulling in full plan protos. + */ + +syntax = 'proto3'; + +package spark.connect; + +import "google/protobuf/any.proto"; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; +option go_package = "internal/generated"; +""" + +STATUS_SERVICE_BLOCK = """\ +// Minimal SparkConnectService subset for GetStatus RPC. +service SparkConnectService { + // Get status information of different types. + rpc GetStatus(GetStatusRequest) returns (GetStatusResponse) {} +} +""" + + +def _extract_proto_block(proto_text: str, message_name: str) -> str: + """Extracts a top-level message block with its leading comment lines.""" + lines = proto_text.splitlines() + decl_re = re.compile(rf"^message\s+{re.escape(message_name)}\s*\{{") + start_idx = None + for idx, line in enumerate(lines): + if decl_re.match(line): + start_idx = idx + break + if start_idx is None: + raise ValueError(f"Message {message_name!r} not found in base.proto") + + comment_start = start_idx + while comment_start > 0 and lines[comment_start - 1].lstrip().startswith( + "//" + ): + comment_start -= 1 + + depth = 0 + end_idx = None + for idx in range(start_idx, len(lines)): + # Strip // line comments before counting braces. + code_part = lines[idx].split("//", 1)[0] + depth += code_part.count("{") - code_part.count("}") + if depth == 0: + end_idx = idx + break + if end_idx is None: + raise ValueError( + f"Unbalanced braces while extracting {message_name!r} from" + " base.proto" + ) + + return "\n".join(lines[comment_start : end_idx + 1]) + + +def build_minimal_status_proto(base_proto_text: str) -> str: + """Builds spark_connect_status.proto from upstream base.proto.""" + blocks = [ + _extract_proto_block(base_proto_text, name) + for name in ("UserContext", "GetStatusRequest", "GetStatusResponse") + ] + return ( + STATUS_PROTO_HEADER + + "\n" + + "\n\n".join(blocks) + + "\n\n" + + STATUS_SERVICE_BLOCK + ) + + +def _run_git(repo_dir: Path, args: Sequence[str]) -> str: + git_env = {**os.environ, "GIT_TERMINAL_PROMPT": "0"} + res = subprocess.run( + ["git", "-C", str(repo_dir), *args], + check=False, + capture_output=True, + text=True, + env=git_env, + ) + if res.returncode != 0: + raise RuntimeError( + f"git {' '.join(args)} failed (exit {res.returncode}):\n" + f"{res.stderr.strip()}" + ) + return res.stdout + + +def _git_show_file(repo_dir: Path, commit: str, rel_path: str) -> str: + return _run_git(repo_dir, ["show", f"{commit}:{rel_path}"]) + + +def refresh_from_remote(remote: str, ref: str) -> str: + """Fetches `ref` from `remote` and updates .proto, NOTICE, and metadata.""" + with tempfile.TemporaryDirectory(prefix="spark_proto_sync_") as tmp: + tmp_repo = Path(tmp) / "repo" + tmp_repo.mkdir() + _run_git(tmp_repo, ["init"]) + _run_git(tmp_repo, ["remote", "add", "origin", remote]) + _run_git(tmp_repo, ["config", "remote.origin.promisor", "true"]) + _run_git( + tmp_repo, + ["config", "remote.origin.partialclonefilter", "blob:none"], + ) + _run_git( + tmp_repo, + [ + "fetch", + "--depth=1", + "--filter=blob:none", + "--no-tags", + "origin", + ref, + ], + ) + rev = _run_git(tmp_repo, ["rev-parse", "FETCH_HEAD"]).strip() + + base_proto = _git_show_file(tmp_repo, rev, UPSTREAM_BASE_PROTO_PATH) + cell_monitor_proto = _git_show_file( + tmp_repo, rev, UPSTREAM_CELL_MONITOR_PROTO_PATH + ) + notice_text = _git_show_file(tmp_repo, rev, "NOTICE") + + (VENDOR_DIR / "spark_connect_status.proto").write_text( + build_minimal_status_proto(base_proto), encoding="utf-8" + ) + (VENDOR_DIR / "cell_monitor.proto").write_text( + cell_monitor_proto, encoding="utf-8" + ) + (VENDOR_DIR / "NOTICE").write_text(notice_text, encoding="utf-8") + + today = datetime.date.today() + metadata_path = VENDOR_DIR / "METADATA" + if metadata_path.exists(): + meta = metadata_path.read_text(encoding="utf-8") + meta = re.sub( + r"last_upgrade_date\s*\{\s*year:\s*\d+\s+month:\s*\d+\s+day:\s*\d+\s*\}", + ( + f"last_upgrade_date {{ year: {today.year} " + f"month: {today.month} day: {today.day} }}" + ), + meta, + ) + meta = re.sub( + r'local_modifications:\s*(?:"[^"]*"\s*)+', + ( + "local_modifications:\n" + ' "Extracted minimal UserContext, GetStatusRequest, ' + 'GetStatusResponse, and "\n' + ' "SparkConnectService.GetStatus definitions into ' + 'spark_connect_status.proto "\n' + ' "and added Dataproc cell_monitor.proto from "\n' + ' "bigdataoss-internal/third_party/apache/spark "\n' + f' "(ref {ref}, commit {rev}), then generated Python "\n' + ' "protobuf stubs (*_pb2.py) via scripts/regen_protos.py. ' + 'See README.google.md."\n' + ), + meta, + ) + metadata_path.write_text(meta, encoding="utf-8") + + readme_path = VENDOR_DIR / "README.google.md" + if readme_path.exists(): + readme = readme_path.read_text(encoding="utf-8") + readme = re.sub( + r"(?m)^- \*\*Internal Dataproc Source Commit:\*\* .*$", + f"- **Internal Dataproc Source Commit:** `{rev}` (`{ref}`)", + readme, + ) + readme_path.write_text(readme, encoding="utf-8") + + return rev + + +def _postprocess_pb2(raw_code: str, module_basename: str) -> str: + """Applies cross-version compatibility and DescriptorPool isolation.""" + code = raw_code + + # Remove protoc's strict Gencode/Runtime version pin so generated stubs + # load cleanly across protobuf 4.21+, 5.x, 6.x, and 7.x runtimes. + code = re.sub( + r"^# Protobuf Python Version: .*\n", + "", + code, + flags=re.MULTILINE, + ) + code = re.sub( + r"^from google\.protobuf import runtime_version as _runtime_version\n", + "", + code, + flags=re.MULTILINE, + ) + code = re.sub( + r"_runtime_version\.ValidateProtobufRuntimeVersion\(\n(?:.*?\n)*?\)\n", + "", + code, + ) + + full_module_name = f"{TARGET_PACKAGE}.{module_basename}" + + if module_basename == "spark_connect_status_pb2": + # Isolate spark.connect.* symbols in a private DescriptorPool and + # SymbolDatabase so loading spark_connect_status_pb2 alongside + # pyspark.sql.connect.proto.base_pb2 never raises duplicate-symbol + # errors on PySpark 4.0, 4.1, or 4.2+ (under upb/cpp or pure-Python). + old_add = "DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(" + if old_add not in code: + raise RuntimeError( + "Expected Default().AddSerializedFile in" + " spark_connect_status_pb2" + ) + code = code.replace( + old_add, + "_POOL = _descriptor_pool.DescriptorPool()\n" + "_ANY_DESCRIPTOR = _POOL.AddSerializedFile(\n" + " google_dot_protobuf_dot_any__pb2.DESCRIPTOR.serialized_pb\n" + ")\n" + "DESCRIPTOR = _POOL.AddSerializedFile(", + 1, + ) + old_build = ( + "_builder.BuildTopDescriptorsAndMessages(" + f"DESCRIPTOR, '{module_basename}', _globals)" + ) + if old_build not in code: + raise RuntimeError( + "Expected BuildTopDescriptorsAndMessages in" + " spark_connect_status_pb2" + ) + code = code.replace( + old_build, + "_orig_sym_db = _builder._sym_db\n" + "try:\n" + " _builder._sym_db = _symbol_database.SymbolDatabase(" + "pool=_POOL)\n" + " _builder.BuildTopDescriptorsAndMessages(\n" + f" _ANY_DESCRIPTOR, '{full_module_name}', {{}}\n" + " )\n" + " _builder.BuildTopDescriptorsAndMessages(\n" + f" DESCRIPTOR, '{full_module_name}', _globals\n" + " )\n" + "finally:\n" + " _builder._sym_db = _orig_sym_db", + 1, + ) + else: + old_build = ( + "_builder.BuildTopDescriptorsAndMessages(" + f"DESCRIPTOR, '{module_basename}', _globals)" + ) + if old_build not in code: + raise RuntimeError( + f"Expected BuildTopDescriptorsAndMessages in {module_basename}" + ) + code = code.replace( + old_build, + "_builder.BuildTopDescriptorsAndMessages(" + f"DESCRIPTOR, '{full_module_name}', _globals)", + 1, + ) + + return ASF_PYTHON_HEADER + code + + +def _format_with_pyink(paths: Iterable[Path]) -> None: + path_list = [str(p) for p in paths] + if not path_list: + return + pyink_bin = shutil.which("pyink") + cmd = None + if pyink_bin: + cmd = [pyink_bin, "--config", str(REPO_ROOT / "pyproject.toml")] + else: + probe = subprocess.run( + [sys.executable, "-c", "import pyink"], + capture_output=True, + check=False, + ) + if probe.returncode == 0: + cmd = [ + sys.executable, + "-m", + "pyink", + "--config", + str(REPO_ROOT / "pyproject.toml"), + ] + if cmd is not None: + subprocess.run( + [*cmd, *path_list], + check=True, + capture_output=True, + text=True, + ) + + +def compile_protos(check_only: bool = False) -> int: + """Compiles .proto files into *_pb2.py and optionally checks freshness.""" + try: + import grpc_tools # type: ignore[import-not-found] + from grpc_tools import protoc # type: ignore[import-not-found] + except ImportError as exc: + raise SystemExit( + "grpcio-tools is required to regenerate proto stubs. " + "Install it with: pip install grpcio-tools" + ) from exc + + include_dir = Path(grpc_tools.__file__).parent / "_proto" + proto_files = [ + VENDOR_DIR / "spark_connect_status.proto", + VENDOR_DIR / "cell_monitor.proto", + ] + + with tempfile.TemporaryDirectory(prefix="spark_pb2_gen_") as tmp: + tmp_dir = Path(tmp) + rc = protoc.main( + [ + "grpc_tools.protoc", + f"-I{VENDOR_DIR}", + f"-I{include_dir}", + f"--python_out={tmp_dir}", + *[str(p) for p in proto_files], + ] + ) + if rc != 0: + raise SystemExit(f"protoc failed with exit code {rc}") + + generated_files = [] + for module_basename in ("spark_connect_status_pb2", "cell_monitor_pb2"): + out_file = tmp_dir / f"{module_basename}.py" + processed = _postprocess_pb2( + out_file.read_text(encoding="utf-8"), module_basename + ) + out_file.write_text(processed, encoding="utf-8") + generated_files.append(out_file) + + _format_with_pyink(generated_files) + + if check_only: + stale = [] + for gen_file in generated_files: + target = VENDOR_DIR / gen_file.name + if not target.exists() or ( + target.read_text(encoding="utf-8") + != gen_file.read_text(encoding="utf-8") + ): + stale.append(gen_file.name) + if stale: + sys.stderr.write( + "Stale or missing generated proto stubs: " + + ", ".join(stale) + + "\nRun `python scripts/regen_protos.py` to update.\n" + ) + return 1 + return 0 + + for gen_file in generated_files: + shutil.copy2(gen_file, VENDOR_DIR / gen_file.name) + + return 0 + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--ref", + help=( + "Optional Git ref or commit SHA on --remote to refresh " + "spark_connect_status.proto, cell_monitor.proto, NOTICE, and " + "METADATA before regenerating Python stubs." + ), + ) + parser.add_argument( + "--remote", + default=DEFAULT_SPARK_REMOTE, + help=f"Git remote URL for --ref (default: {DEFAULT_SPARK_REMOTE}).", + ) + parser.add_argument( + "--check", + action="store_true", + help="Verify checked-in *_pb2.py stubs match the .proto files.", + ) + args = parser.parse_args(argv) + + if args.ref and args.check: + parser.error("--ref and --check cannot be used together.") + + if args.ref: + rev = refresh_from_remote(args.remote, args.ref) + print(f"Refreshed .proto definitions from {args.ref} ({rev}).") + + return compile_protos(check_only=args.check) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/setup.cfg b/setup.cfg index e078a91d..8a90975b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,7 @@ [check-manifest] ignore = .github/** + scripts/** [metadata] long_description_content_type = text/markdown \ No newline at end of file diff --git a/setup.py b/setup.py index 0026c541..41632da4 100644 --- a/setup.py +++ b/setup.py @@ -93,7 +93,21 @@ def run(self) -> None: "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", ], - packages=find_namespace_packages(include=["google.*"]), + packages=( + find_namespace_packages(include=["google.*"]) + + ["google.cloud.managed_spark_connect._third_party.apache_spark"] + ), + package_dir={ + "google.cloud.managed_spark_connect._third_party.apache_spark": ( + "third_party/apache_spark" + ), + }, + package_data={ + "google.cloud.managed_spark_connect._third_party.apache_spark": [ + "LICENSE", + "NOTICE", + ], + }, exclude_package_data={"*": [".wh.*"]}, cmdclass={"egg_info": _CleanEggInfo}, install_requires=[ diff --git a/tests/unit/test_pypi_artifacts.py b/tests/unit/test_pypi_artifacts.py index 580d6646..0592b8dc 100644 --- a/tests/unit/test_pypi_artifacts.py +++ b/tests/unit/test_pypi_artifacts.py @@ -95,6 +95,7 @@ def test_sdist_and_wheel_exclude_oci_whiteouts_and_stale_egg_info(self): "setup.py", "setup.cfg", "pyproject.toml", + "MANIFEST.in", "README.md", "LICENSE", ): @@ -102,6 +103,7 @@ def test_sdist_and_wheel_exclude_oci_whiteouts_and_stale_egg_info(self): if src.exists(): shutil.copy2(src, work_dir / name) shutil.copytree(repo_root / "google", work_dir / "google") + shutil.copytree(repo_root / "third_party", work_dir / "third_party") # Simulate overlayfs .wh..wh..opq whiteout markers and stale # egg-info files left in an existing working tree before building. @@ -173,3 +175,32 @@ def test_sdist_and_wheel_exclude_oci_whiteouts_and_stale_egg_info(self): ) self.assertNotIn(".wh.", record_text) self.assertNotIn("stale.txt", record_text) + + # Verify third_party/apache_spark is included in sdist and remapped + # into our private namespace inside the wheel (never top-level). + self.assertTrue( + any( + name.endswith("third_party/apache_spark/cell_monitor.proto") + for name in tar_names + ), + tar_names, + ) + self.assertFalse( + any( + name == "third_party" or name.startswith("third_party/") + for name in whl_names + ), + whl_names, + ) + expected_wheel_entries = { + "google/cloud/managed_spark_connect/_third_party/__init__.py", + "google/cloud/managed_spark_connect/_third_party/apache_spark/__init__.py", + "google/cloud/managed_spark_connect/_third_party/apache_spark/cell_monitor_pb2.py", + "google/cloud/managed_spark_connect/_third_party/apache_spark/spark_connect_status_pb2.py", + "google/cloud/managed_spark_connect/_third_party/apache_spark/LICENSE", + "google/cloud/managed_spark_connect/_third_party/apache_spark/NOTICE", + } + self.assertTrue( + expected_wheel_entries.issubset(set(whl_names)), + expected_wheel_entries - set(whl_names), + ) diff --git a/tests/unit/test_vendored_protos.py b/tests/unit/test_vendored_protos.py new file mode 100644 index 00000000..d7f6389e --- /dev/null +++ b/tests/unit/test_vendored_protos.py @@ -0,0 +1,492 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for vendored Spark Connect GetStatus and Cell Monitor protos.""" + +import importlib.resources as importlib_resources +import importlib.util +import os +from pathlib import Path +import subprocess +import sys +import unittest + +from google.protobuf import any_pb2 +from google.protobuf import descriptor_pool + +from google.cloud.managed_spark_connect._third_party import apache_spark +from google.cloud.managed_spark_connect._third_party.apache_spark import ( + cell_monitor_pb2, + spark_connect_status_pb2, +) + +REPO_ROOT = Path(__file__).resolve().parents[2] +VENDOR_DIR = REPO_ROOT / "third_party" / "apache_spark" + + +def _run_python(code: str, env: dict[str, str] | None = None): + """Runs Python code via stdin (avoiding PySpark 4.2 -c doctest quirk).""" + return subprocess.run( + [sys.executable, "-"], + input=code, + capture_output=True, + text=True, + check=False, + env=env, + ) + + +class TestVendoredProtoCompliance(unittest.TestCase): + """Verifies third_party/apache_spark layout and packaged license files.""" + + def test_third_party_directory_layout(self): + required_files = ( + "LICENSE", + "NOTICE", + "METADATA", + "README.google.md", + "spark_connect_status.proto", + "cell_monitor.proto", + "__init__.py", + "spark_connect_status_pb2.py", + "cell_monitor_pb2.py", + ) + for filename in required_files: + path = VENDOR_DIR / filename + self.assertTrue(path.is_file(), f"Missing {path}") + + # Only dependency subdirectories belong directly under third_party/. + top_entries = [ + p.name + for p in (REPO_ROOT / "third_party").iterdir() + if p.name != "__pycache__" + ] + self.assertEqual(top_entries, ["apache_spark"]) + + def test_metadata_contains_required_fields(self): + metadata = (VENDOR_DIR / "METADATA").read_text(encoding="utf-8") + for needle in ( + 'name: "Apache Spark"', + "third_party {", + 'homepage: "https://spark.apache.org/"', + 'type: "Git"', + 'value: "https://github.com/apache/spark"', + "primary_source: true", + "last_upgrade_date {", + "license_type: NOTICE", + "local_modifications:", + ): + self.assertIn(needle, metadata) + # Ensure deprecated METADATA fields are not used. + for deprecated in ("license_note:", "last_upgrade_cl:", "\n url {"): + self.assertNotIn(deprecated, metadata) + + def test_packaged_license_and_notice_resources(self): + pkg_files = importlib_resources.files( + "google.cloud.managed_spark_connect._third_party.apache_spark" + ) + license_text = (pkg_files / "LICENSE").read_text(encoding="utf-8") + notice_text = (pkg_files / "NOTICE").read_text(encoding="utf-8") + self.assertIn("Apache License", license_text) + self.assertIn("Version 2.0, January 2004", license_text) + self.assertIn("Apache Spark", notice_text) + self.assertIn("The Apache Software Foundation", notice_text) + + def test_grpc_method_constant_matches_service_descriptor(self): + svc = spark_connect_status_pb2.DESCRIPTOR.services_by_name[ + "SparkConnectService" + ] + method = svc.methods_by_name["GetStatus"] + expected = f"/{svc.full_name}/{method.name}" + self.assertEqual( + expected, "/spark.connect.SparkConnectService/GetStatus" + ) + self.assertEqual(apache_spark.GRPC_GET_STATUS_METHOD, expected) + self.assertEqual( + method.input_type.full_name, "spark.connect.GetStatusRequest" + ) + self.assertEqual( + method.output_type.full_name, "spark.connect.GetStatusResponse" + ) + + +class TestDescriptorPoolIsolation(unittest.TestCase): + """Verifies zero DescriptorPool collisions alongside PySpark.""" + + def test_pool_separation_in_current_process(self): + default_pool = descriptor_pool.Default() + self.assertIsNot(spark_connect_status_pb2._POOL, default_pool) + self.assertIs(cell_monitor_pb2.DESCRIPTOR.pool, default_pool) + self.assertIsNotNone( + default_pool.FindMessageTypeByName( + "dataproc.cellmonitor.v1.CellMonitorTelemetryRequest" + ) + ) + self.assertIsNotNone( + default_pool.FindMessageTypeByName( + "dataproc.cellmonitor.v1.CellMonitorTelemetry" + ) + ) + + def test_import_orders_with_pyspark_base_pb2(self): + if ( + importlib.util.find_spec("pyspark.sql.connect.proto.base_pb2") + is None + ): + self.skipTest("pyspark connect protos not installed") + + pyspark_first = """\ +import pyspark.sql.connect.proto.base_pb2 as base_pb2 +from google.cloud.managed_spark_connect._third_party.apache_spark import ( + cell_monitor_pb2, + spark_connect_status_pb2, +) +assert base_pb2.UserContext is not spark_connect_status_pb2.UserContext +assert base_pb2.UserContext.DESCRIPTOR.full_name == "spark.connect.UserContext" +assert ( + spark_connect_status_pb2.GetStatusRequest.DESCRIPTOR.full_name + == "spark.connect.GetStatusRequest" +) +assert ( + cell_monitor_pb2.CellMonitorTelemetry.DESCRIPTOR.full_name + == "dataproc.cellmonitor.v1.CellMonitorTelemetry" +) +""" + vendored_first = """\ +from google.cloud.managed_spark_connect._third_party.apache_spark import ( + cell_monitor_pb2, + spark_connect_status_pb2, +) +import pyspark.sql.connect.proto.base_pb2 as base_pb2 +assert base_pb2.UserContext is not spark_connect_status_pb2.UserContext +assert base_pb2.UserContext.DESCRIPTOR.full_name == "spark.connect.UserContext" +""" + for label, snippet in ( + ("pyspark_first", pyspark_first), + ("vendored_first", vendored_first), + ): + with self.subTest(order=label): + res = _run_python(snippet) + self.assertEqual(res.returncode, 0, res.stderr) + + def test_pure_python_protobuf_backend_no_collision(self): + if ( + importlib.util.find_spec("pyspark.sql.connect.proto.base_pb2") + is None + ): + self.skipTest("pyspark connect protos not installed") + + snippet = """\ +from google.cloud.managed_spark_connect._third_party.apache_spark import ( + cell_monitor_pb2, + spark_connect_status_pb2, +) +import pyspark.sql.connect.proto.base_pb2 as base_pb2 +req = spark_connect_status_pb2.GetStatusRequest(session_id="s1") +treq = cell_monitor_pb2.CellMonitorTelemetryRequest(since_version=3) +req.operation_status.extensions.add().Pack(treq) +parsed = spark_connect_status_pb2.GetStatusRequest.FromString( + req.SerializeToString() +) +out = cell_monitor_pb2.CellMonitorTelemetryRequest() +assert parsed.operation_status.extensions[0].Unpack(out) +assert out.since_version == 3 +assert base_pb2.UserContext(user_id="u").user_id == "u" +""" + env = { + **os.environ, + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": "python", + } + res = _run_python(snippet, env=env) + self.assertEqual(res.returncode, 0, res.stderr) + + +class TestProtoWireCompatibilityAndRoundTrip(unittest.TestCase): + """Verifies wire compatibility and Any.Pack/Unpack roundtrips.""" + + def test_user_context_wire_compatibility_with_pyspark(self): + if ( + importlib.util.find_spec("pyspark.sql.connect.proto.base_pb2") + is None + ): + self.skipTest("pyspark connect protos not installed") + + from pyspark.sql.connect.proto import base_pb2 as pyspark_base_pb2 + + ext_payload = cell_monitor_pb2.CellMonitorTelemetryRequest( + since_version=99 + ) + pyspark_uc = pyspark_base_pb2.UserContext( + user_id="alice", user_name="Alice" + ) + pyspark_uc.extensions.add().Pack(ext_payload) + + vendored_uc = spark_connect_status_pb2.UserContext.FromString( + pyspark_uc.SerializeToString() + ) + self.assertEqual(vendored_uc.user_id, "alice") + self.assertEqual(vendored_uc.user_name, "Alice") + self.assertEqual(len(vendored_uc.extensions), 1) + unpacked = cell_monitor_pb2.CellMonitorTelemetryRequest() + self.assertTrue(vendored_uc.extensions[0].Unpack(unpacked)) + self.assertEqual(unpacked.since_version, 99) + + roundtripped_pyspark_uc = pyspark_base_pb2.UserContext.FromString( + vendored_uc.SerializeToString() + ) + self.assertEqual(roundtripped_pyspark_uc, pyspark_uc) + + # On PySpark 4.2+, base_pb2 also includes GetStatusRequest/Response; + # verify full wire roundtrip between PySpark 4.2 and our vendored stubs. + if hasattr(pyspark_base_pb2, "GetStatusRequest"): + vendored_req = spark_connect_status_pb2.GetStatusRequest( + session_id="00112233-4455-6677-8899-aabbccddeeff", + client_type="msc", + client_observed_server_side_session_id="srv-42", + ) + vendored_req.user_context.user_id = "alice" + vendored_req.operation_status.operation_ids.append("op-42") + vendored_req.operation_status.extensions.add().Pack(ext_payload) + + pyspark_req = pyspark_base_pb2.GetStatusRequest.FromString( + vendored_req.SerializeToString() + ) + self.assertEqual(pyspark_req.session_id, vendored_req.session_id) + self.assertEqual( + list(pyspark_req.operation_status.operation_ids), ["op-42"] + ) + unpacked_42 = cell_monitor_pb2.CellMonitorTelemetryRequest() + self.assertTrue( + pyspark_req.operation_status.extensions[0].Unpack(unpacked_42) + ) + self.assertEqual(unpacked_42.since_version, 99) + + def test_get_status_request_pack_unpack_single_and_multi_op(self): + telemetry_req = cell_monitor_pb2.CellMonitorTelemetryRequest( + since_version=14 + ) + telemetry_req.since_versions["op-a"] = 14 + telemetry_req.since_versions["op-b"] = 21 + + req = spark_connect_status_pb2.GetStatusRequest( + session_id="00112233-4455-6677-8899-aabbccddeeff", + client_type="google-cloud-spark-connect", + client_observed_server_side_session_id="srv-session-1", + ) + req.user_context.user_id = "user-123" + req.user_context.user_name = "ptch" + req.operation_status.operation_ids.extend(["op-a", "op-b"]) + req.operation_status.extensions.add().Pack(telemetry_req) + + self.assertEqual( + req.operation_status.extensions[0].type_url, + "type.googleapis.com/" + "dataproc.cellmonitor.v1.CellMonitorTelemetryRequest", + ) + + wire_bytes = req.SerializeToString() + decoded = spark_connect_status_pb2.GetStatusRequest.FromString( + wire_bytes + ) + self.assertEqual( + decoded.session_id, "00112233-4455-6677-8899-aabbccddeeff" + ) + self.assertEqual(decoded.user_context.user_id, "user-123") + self.assertEqual(decoded.client_type, "google-cloud-spark-connect") + self.assertEqual( + decoded.client_observed_server_side_session_id, "srv-session-1" + ) + self.assertEqual( + list(decoded.operation_status.operation_ids), ["op-a", "op-b"] + ) + + ext = decoded.operation_status.extensions[0] + self.assertTrue( + ext.Is(cell_monitor_pb2.CellMonitorTelemetryRequest.DESCRIPTOR) + ) + unpacked_req = cell_monitor_pb2.CellMonitorTelemetryRequest() + self.assertTrue(ext.Unpack(unpacked_req)) + self.assertTrue(unpacked_req.HasField("since_version")) + self.assertEqual(unpacked_req.since_version, 14) + self.assertEqual( + dict(unpacked_req.since_versions), {"op-a": 14, "op-b": 21} + ) + + def test_get_status_response_and_full_telemetry_roundtrip(self): + OpStatus = spark_connect_status_pb2.GetStatusResponse.OperationStatus + OpState = OpStatus.OperationState + Telemetry = cell_monitor_pb2.CellMonitorTelemetry + + # Verify all expected server enum values match Spark 4.2 & CL 196087. + self.assertEqual(OpState.OPERATION_STATE_UNSPECIFIED, 0) + self.assertEqual(OpState.OPERATION_STATE_UNKNOWN, 1) + self.assertEqual(OpState.OPERATION_STATE_RUNNING, 2) + self.assertEqual(OpState.OPERATION_STATE_TERMINATING, 3) + self.assertEqual(OpState.OPERATION_STATE_SUCCEEDED, 4) + self.assertEqual(OpState.OPERATION_STATE_FAILED, 5) + self.assertEqual(OpState.OPERATION_STATE_CANCELLED, 6) + + self.assertEqual(Telemetry.JOB_STATUS_UNSPECIFIED, 0) + self.assertEqual(Telemetry.JOB_STATUS_RUNNING, 1) + self.assertEqual(Telemetry.JOB_STATUS_SUCCEEDED, 2) + self.assertEqual(Telemetry.JOB_STATUS_FAILED, 3) + self.assertEqual(Telemetry.JOB_STATUS_CANCELLED, 4) + + self.assertEqual(Telemetry.STAGE_STATUS_UNSPECIFIED, 0) + self.assertEqual(Telemetry.STAGE_STATUS_PENDING, 1) + self.assertEqual(Telemetry.STAGE_STATUS_RUNNING, 2) + self.assertEqual(Telemetry.STAGE_STATUS_COMPLETED, 3) + self.assertEqual(Telemetry.STAGE_STATUS_FAILED, 4) + self.assertEqual(Telemetry.STAGE_STATUS_SKIPPED, 5) + + telemetry = Telemetry( + version=42, + unchanged=False, + final_snapshot=True, + ) + job = telemetry.jobs.add( + job_id=7, + name="count at cell.py:1", + description="SELECT count(*)", + status=Telemetry.JOB_STATUS_SUCCEEDED, + submission_time=1762900000100, + completion_time=1762900001500, + stage_ids=[10, 11], + ) + self.assertTrue(job.HasField("submission_time")) + self.assertTrue(job.HasField("completion_time")) + + telemetry.stages.add( + stage_id=10, + attempt_id=0, + name="scan parquet", + status=Telemetry.STAGE_STATUS_COMPLETED, + num_tasks=8, + num_completed_tasks=8, + num_active_tasks=0, + num_failed_tasks=0, + input_bytes_read=65536, + parent_ids=[], + submission_time=1762900000120, + completion_time=1762900000900, + ) + stage_pending = telemetry.stages.add( + stage_id=11, + attempt_id=1, + name="aggregate", + status=Telemetry.STAGE_STATUS_SKIPPED, + num_tasks=1, + num_completed_tasks=1, + parent_ids=[10], + ) + self.assertFalse(stage_pending.HasField("submission_time")) + self.assertFalse(stage_pending.HasField("completion_time")) + + telemetry.running_tasks.slot_width_ms = 200 + telemetry.running_tasks.start_time = 1762900000000 + telemetry.running_tasks.values.extend([4, 8, 8, 0]) + # Includes -1 sentinel to exercise sint64 zigzag encoding. + telemetry.running_tasks.last_event_offsets_ms.extend([15, 90, -1, 140]) + + # Simulate server packing via standard google.protobuf.any_pb2.Any. + packed_any = any_pb2.Any() + packed_any.Pack(telemetry) + self.assertEqual( + packed_any.type_url, + "type.googleapis.com/dataproc.cellmonitor.v1.CellMonitorTelemetry", + ) + + resp = spark_connect_status_pb2.GetStatusResponse( + session_id="sess-1", + server_side_session_id="srv-1", + ) + op_ok = resp.operation_statuses.add( + operation_id="op-1", + state=OpState.OPERATION_STATE_SUCCEEDED, + ) + op_ok.extensions.add().ParseFromString(packed_any.SerializeToString()) + # Second operation with OPERATION_STATE_UNKNOWN and no extension (race). + resp.operation_statuses.add( + operation_id="op-race", + state=OpState.OPERATION_STATE_UNKNOWN, + ) + + decoded_resp = spark_connect_status_pb2.GetStatusResponse.FromString( + resp.SerializeToString() + ) + self.assertEqual(len(decoded_resp.operation_statuses), 2) + self.assertEqual( + decoded_resp.operation_statuses[1].state, + OpState.OPERATION_STATE_UNKNOWN, + ) + self.assertEqual(len(decoded_resp.operation_statuses[1].extensions), 0) + + decoded_ext = decoded_resp.operation_statuses[0].extensions[0] + self.assertTrue(decoded_ext.Is(Telemetry.DESCRIPTOR)) + decoded_telemetry = Telemetry() + self.assertTrue(decoded_ext.Unpack(decoded_telemetry)) + + self.assertEqual(decoded_telemetry.version, 42) + self.assertFalse(decoded_telemetry.unchanged) + self.assertTrue(decoded_telemetry.final_snapshot) + self.assertEqual(len(decoded_telemetry.jobs), 1) + self.assertEqual( + decoded_telemetry.jobs[0].status, Telemetry.JOB_STATUS_SUCCEEDED + ) + self.assertEqual(list(decoded_telemetry.jobs[0].stage_ids), [10, 11]) + self.assertEqual(len(decoded_telemetry.stages), 2) + self.assertEqual(decoded_telemetry.stages[0].input_bytes_read, 65536) + self.assertEqual( + decoded_telemetry.stages[1].status, Telemetry.STAGE_STATUS_SKIPPED + ) + self.assertEqual(decoded_telemetry.running_tasks.slot_width_ms, 200) + self.assertEqual( + list(decoded_telemetry.running_tasks.values), [4, 8, 8, 0] + ) + self.assertEqual( + list(decoded_telemetry.running_tasks.last_event_offsets_ms), + [15, 90, -1, 140], + ) + + +class TestRegenProtosScript(unittest.TestCase): + """Verifies scripts/regen_protos.py extraction and stub freshness.""" + + @classmethod + def _load_regen_module(cls): + script_path = REPO_ROOT / "scripts" / "regen_protos.py" + spec = importlib.util.spec_from_file_location( + "regen_protos", script_path + ) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + def test_build_minimal_status_proto_roundtrip(self): + regen = self._load_regen_module() + current_proto = (VENDOR_DIR / "spark_connect_status.proto").read_text( + encoding="utf-8" + ) + rebuilt = regen.build_minimal_status_proto(current_proto) + self.assertEqual(rebuilt, current_proto) + + def test_checked_in_stubs_are_up_to_date(self): + if importlib.util.find_spec("grpc_tools") is None: + self.skipTest("grpcio-tools not installed") + regen = self._load_regen_module() + self.assertEqual(regen.compile_protos(check_only=True), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/third_party/apache_spark/LICENSE b/third_party/apache_spark/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/third_party/apache_spark/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third_party/apache_spark/METADATA b/third_party/apache_spark/METADATA new file mode 100644 index 00000000..9f648383 --- /dev/null +++ b/third_party/apache_spark/METADATA @@ -0,0 +1,30 @@ +# proto-file: devtools/metadata/metadata.proto +# proto-message: MetaData + +name: "Apache Spark" +description: + "Minimal Spark Connect GetStatus RPC and Dataproc Cell Monitor " + "telemetry protocol buffer definitions and generated Python stubs." + +third_party { + homepage: "https://spark.apache.org/" + identifier { + type: "Git" + value: "https://github.com/apache/spark" + version: "v4.2.0" + primary_source: true + upstream_paths: "sql/connect/common/src/main/protobuf/spark/connect/base.proto" + } + version: "v4.2.0" + last_upgrade_date { year: 2026 month: 11 day: 11 } + license_type: NOTICE + local_modifications: + "Extracted minimal UserContext, GetStatusRequest, GetStatusResponse, and " + "SparkConnectService.GetStatus definitions into spark_connect_status.proto " + "from bigdataoss-internal/third_party/apache/spark dataproc-branch-4.2.0 " + "(commit 298a4f5511997642dbe4cc3ee50303cf4e28b281), added Dataproc " + "cell_monitor.proto from Gerrit CL 196087 PS 7 " + "(ref refs/changes/87/196087/7, commit bb002ebc1834d6a587f9c1ee6bc1b11a7d74b748), " + "and generated Python protobuf stubs (*_pb2.py) via " + "scripts/regen_protos.py. See README.google.md." +} diff --git a/third_party/apache_spark/NOTICE b/third_party/apache_spark/NOTICE new file mode 100644 index 00000000..d5ea8ded --- /dev/null +++ b/third_party/apache_spark/NOTICE @@ -0,0 +1,41 @@ +Apache Spark +Copyright 2014 and onwards The Apache Software Foundation. + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + +Export Control Notice +--------------------- + +This distribution includes cryptographic software. The country in which you currently reside may have +restrictions on the import, possession, use, and/or re-export to another country, of encryption software. +BEFORE using any encryption software, please check your country's laws, regulations and policies concerning +the import, possession, or use, and re-export of encryption software, to see if this is permitted. See + for more information. + +The U.S. Government Department of Commerce, Bureau of Industry and Security (BIS), has classified this +software as Export Commodity Control Number (ECCN) 5D002.C.1, which includes information security software +using or performing cryptographic functions with asymmetric algorithms. The form and manner of this Apache +Software Foundation distribution makes it eligible for export under the License Exception ENC Technology +Software Unrestricted (TSU) exception (see the BIS Export Administration Regulations, Section 740.13) for +both object code and source code. + +The following provides more details on the included cryptographic software: + +This software uses Apache Commons Crypto (https://commons.apache.org/proper/commons-crypto/) to +support authentication, and encryption and decryption of data sent across the network between +services. + + +Metrics +Copyright 2010-2013 Coda Hale and Yammer, Inc. + +This product includes software developed by Coda Hale and Yammer, Inc. + +This product includes code derived from the JSR-166 project (ThreadLocalRandom, Striped64, +LongAdder), which was released with the following comments: + + Written by Doug Lea with assistance from members of JCP JSR-166 + Expert Group and released to the public domain, as explained at + http://creativecommons.org/publicdomain/zero/1.0/ \ No newline at end of file diff --git a/third_party/apache_spark/README.google.md b/third_party/apache_spark/README.google.md new file mode 100644 index 00000000..545418fe --- /dev/null +++ b/third_party/apache_spark/README.google.md @@ -0,0 +1,55 @@ +# Apache Spark Connect `GetStatus` & Dataproc Cell Monitor Protos + +This directory contains minimal protocol buffer definitions and generated Python +stubs derived from Apache Spark 4.2 (`dataproc-branch-4.2.0`), following +Google's non-google3 third-party policy (`go/thirdparty/non-google3`). + +At wheel build time (`setup.py`), this directory is remapped into the private +package namespace +`google.cloud.managed_spark_connect._third_party.apache_spark` so no top-level +`third_party` package is ever installed into `site-packages/`. + +## Source Provenance + +- **Upstream Apache Spark Base Release:** [`v4.2.0`](https://github.com/apache/spark/tree/v4.2.0) (`32f7299601108917fb01920a54e084595b7b3bf8`) +- **Internal Dataproc Branch (`dataproc-branch-4.2.0` HEAD):** `298a4f5511997642dbe4cc3ee50303cf4e28b281` (`bigdataoss-internal/third_party/apache/spark`) +- **Internal Dataproc Source Commit:** `bb002ebc1834d6a587f9c1ee6bc1b11a7d74b748` (`refs/changes/87/196087/7`) + +## Included Files & Local Modifications + +- `LICENSE` & `NOTICE`: Apache License 2.0 and upstream Apache Spark `NOTICE` + attribution file (also packaged into the wheel under + `google/cloud/managed_spark_connect/_third_party/apache_spark/`). +- `METADATA`: Third-party compliance metadata (`devtools.metadata.Metadata`). +- `spark_connect_status.proto`: Extracted subset of + `sql/connect/common/src/main/protobuf/spark/connect/base.proto` containing + only `UserContext`, `GetStatusRequest`, `GetStatusResponse`, and + `SparkConnectService.GetStatus` (`package spark.connect;`). Extracting only + these messages avoids bundling ~15,000 lines of Spark Connect plan/expression + protos already provided by `pyspark-client` / `pyspark`. +- `cell_monitor.proto`: Verbatim copy of + `sql/connect/common/src/main/protobuf/dataproc/cellmonitor/v1/cell_monitor.proto` + (`package dataproc.cellmonitor.v1;`) defining `CellMonitorTelemetryRequest` + and `CellMonitorTelemetry`. +- `spark_connect_status_pb2.py` & `cell_monitor_pb2.py`: Generated Python + protobuf stubs produced by `scripts/regen_protos.py`. + `spark_connect_status_pb2.py` loads its `spark.connect.*` descriptors into an + isolated `DescriptorPool` and `SymbolDatabase` so importing it alongside + `pyspark.sql.connect.proto.base_pb2` never causes duplicate-symbol collisions + across PySpark 4.0, 4.1, or 4.2+. + +## Regenerating Stubs + +From a virtual environment with `grpcio-tools` (and `pyink`) installed: + +```sh +# Regenerate *_pb2.py from local .proto files: +python scripts/regen_protos.py + +# Verify checked-in *_pb2.py files are up to date: +python scripts/regen_protos.py --check + +# Refresh .proto files, NOTICE, METADATA, and stubs from bigdataoss-internal +# (for example, once Gerrit CL 196087 merges into dataproc-branch-4.2.0): +python scripts/regen_protos.py --ref refs/heads/dataproc-branch-4.2.0 +``` diff --git a/third_party/apache_spark/__init__.py b/third_party/apache_spark/__init__.py new file mode 100644 index 00000000..48423af9 --- /dev/null +++ b/third_party/apache_spark/__init__.py @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modifications copyright 2026 Google LLC. +"""Vendored Apache Spark Connect GetStatus and Dataproc Cell Monitor stubs.""" + +GRPC_GET_STATUS_METHOD = "/spark.connect.SparkConnectService/GetStatus" + +__all__ = ["GRPC_GET_STATUS_METHOD"] diff --git a/third_party/apache_spark/cell_monitor.proto b/third_party/apache_spark/cell_monitor.proto new file mode 100644 index 00000000..c0f1444c --- /dev/null +++ b/third_party/apache_spark/cell_monitor.proto @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +// Dataproc Cell Monitor: job, stage and task telemetry of a Spark Connect operation. +// +// This file is owned by Dataproc and is not part of the Apache Spark Connect protocol. The +// messages are exchanged through the generic extension points of the GetStatus RPC +// (spark.connect.GetStatusRequest / GetStatusResponse), packed in google.protobuf.Any: +// +// - Request: add a packed CellMonitorTelemetryRequest to +// GetStatusRequest.OperationStatusRequest.extensions. +// - Response: every GetStatusResponse.OperationStatus for which telemetry is available carries +// a packed CellMonitorTelemetry in its extensions. No extension is returned when the Cell +// Monitor is disabled (spark.dataproc.cellMonitor.enabled=false) or when the server has no +// telemetry for the operation (e.g. an unknown operation, an operation of another session, a +// running operation whose start event the server has not processed yet, an operation whose +// telemetry was evicted by spark.dataproc.cellMonitor.closedOperationTtl or +// maxTrackedOperations, or an ended operation whose events never reached the server). +// +// The telemetry is query-level only: it describes the Spark jobs, stages and tasks run by one +// operation. It intentionally carries no application or executor level information. +package dataproc.cellmonitor.v1; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.dataproc.proto"; + +// Operation-level request extension. +// +// Telemetry versions come from a counter shared across operations: a client must only pass a +// version that it received for that same operation. +message CellMonitorTelemetryRequest { + // The `version` of the last CellMonitorTelemetry the client received for the requested + // operation (for single-operation requests; if set on a multi-operation request without + // `since_versions`, it applies to every operation in the request). If the telemetry version + // equals `since_version`, the response has `unchanged` set and carries no jobs, stages or + // running tasks; any other `since_version` returns a full snapshot. Ignored when + // `since_versions` is non-empty. + optional int64 since_version = 1; + + // Per-operation `since_version`, keyed by `operation_id`, for requests that query multiple + // operations (or all operations of the session) at once. Operations absent from a non-empty map + // receive a full snapshot. + map since_versions = 2; +} + +// Operation-level response extension: a snapshot of the jobs, stages and running tasks of one +// operation. +message CellMonitorTelemetry { + // Monotonically increasing version of the telemetry of this operation. It changes on every + // update of the jobs, stages or running tasks, including the eviction of completed entries, + // and keeps increasing if the telemetry of the operation is evicted and tracked again. + int64 version = 1; + + // True if the telemetry did not change since `CellMonitorTelemetryRequest.since_version`. In + // that case `jobs` and `stages` are empty, `running_tasks` is unset, and the client keeps its + // previous snapshot. + bool unchanged = 2; + + // True once the server observed the end of the operation and of all its jobs, i.e. the + // snapshot is not expected to change anymore. The tasks of a job that were still running when + // it ended (e.g. tasks killed when the job failed, which end asynchronously) are no longer + // counted as running, and a stage of the job that is still running for a job of another + // operation is completed (or failed) with the job. + // + // A terminal OperationStatus.state (`SUCCEEDED`, `FAILED` or `CANCELLED`; note that + // `OPERATION_STATE_TERMINATING` is non-terminal before close) does not imply this: the end of + // the operation and of its jobs reaches the server-side listener asynchronously, usually within + // milliseconds. The server waits a bounded time for it (at most once per operation, from a + // per-request wait budget) before answering; if `final_snapshot` is still false once the + // operation reaches a terminal state (e.g. the server dropped events because the queue of its + // listener was full, or earlier operations in the request exhausted the wait budget), the client + // must treat the jobs and stages that are still RUNNING or PENDING, and the tasks still counted + // as running, as finished, with the outcome of the operation. + bool final_snapshot = 3; + + // Jobs of the operation, ordered by job id. Completed jobs beyond + // spark.dataproc.cellMonitor.maxRetainedJobsPerOperation are evicted (oldest first); running + // jobs are always included. + repeated JobTelemetry jobs = 4; + + // Stages of the jobs of the operation, ordered by stage id. Completed, failed or skipped + // stages beyond spark.dataproc.cellMonitor.maxRetainedStagesPerOperation are evicted (oldest + // first; a failed stage only once its job ended); other stages are always included. + repeated StageTelemetry stages = 5; + + // Number of running tasks of the operation over time, on the clock of the driver. On a changed + // response (`unchanged` = false), unset when the server has no series for the operation (before + // any task with a known launch time started, or if the telemetry was evicted and tracked again). + RunningTasks running_tasks = 6; + + enum JobStatus { + JOB_STATUS_UNSPECIFIED = 0; + JOB_STATUS_RUNNING = 1; + JOB_STATUS_SUCCEEDED = 2; + JOB_STATUS_FAILED = 3; + JOB_STATUS_CANCELLED = 4; + } + + enum StageStatus { + STAGE_STATUS_UNSPECIFIED = 0; + // Part of a job, but not submitted yet. + STAGE_STATUS_PENDING = 1; + STAGE_STATUS_RUNNING = 2; + STAGE_STATUS_COMPLETED = 3; + STAGE_STATUS_FAILED = 4; + // Will not run: its output was already available, or its job ended before it was submitted. + STAGE_STATUS_SKIPPED = 5; + } + + message JobTelemetry { + int64 job_id = 1; + // Name of the job (the name of its last stage, as in the Spark UI). + string name = 2; + // Description of the job (`spark.job.description`), if any. + string description = 3; + JobStatus status = 4; + // Epoch milliseconds. + optional int64 submission_time = 5; + // Epoch milliseconds. + optional int64 completion_time = 6; + repeated int64 stage_ids = 7; + } + + // Task counters are cumulative across the attempts of the stage (as in the ExecutionProgress + // of Spark Connect): a partition is counted as completed once, the first time a task computing + // it succeeds, and `num_tasks` never decreases when a retry computes only the missing + // partitions, so the progress of a stage does not move backwards when the stage is retried. A + // COMPLETED stage reports `num_completed_tasks` = `num_tasks` (partitions whose output was + // already available count as completed). The counters of a stage that runs again after it was + // evicted only cover the attempts since then. + message StageTelemetry { + int64 stage_id = 1; + int64 attempt_id = 2; + string name = 3; + StageStatus status = 4; + int64 num_tasks = 5; + int64 num_completed_tasks = 6; + int64 num_active_tasks = 7; + int64 num_failed_tasks = 8; + int64 input_bytes_read = 9; + repeated int64 parent_ids = 10; + // Epoch milliseconds. + optional int64 submission_time = 11; + // Epoch milliseconds. + optional int64 completion_time = 12; + } + + // The number of running tasks of the operation (all its jobs and stages) over time, in + // consecutive fixed-width time slots. The value of a slot is the number of tasks running at + // the end of the slot, from the launch and finish times of the tasks: a task counts from the + // slot in which it was launched to the one before the one in which it finished. Tasks (or + // stages) that start and finish within the same slot—including once slots have been merged to a + // wider `slot_width_ms`—are not visible. A task finishes once the driver has processed its + // result, so a task launched on the core of a task whose result is still being fetched counts + // alongside it until then, as in the Spark UI. The tasks that the server stops following while + // they run (e.g. tasks killed when their job failed, which end asynchronously after the job, or + // the tasks of an evicted stage) finish at the latest time observed for the operation (e.g. the + // end of their job). The series is complete on every response that carries it and replaces the + // series of the client. + message RunningTasks { + // Width of the slots in milliseconds: initially 100 ms, doubled as many times as needed when + // an event falls beyond the number of slots that the server keeps (the slots are then merged + // two by two, keeping the value at the end of the merged slot), so long operations, or a + // forward jump of the driver clock, are reported at a coarser resolution. + int64 slot_width_ms = 1; + + // Start of the first slot, epoch milliseconds (a multiple of `slot_width_ms`). Slot `i` + // starts at `start_time + i * slot_width_ms`. + int64 start_time = 2; + + // The number of running tasks at the end of each slot, from the first slot to the slot of the + // latest recorded event (a task launch or finish, or the release of running tasks at a job or + // stage event). The last value holds after the last slot. + repeated int32 values = 3; + + // One entry per slot: the offset in milliseconds, within the slot, of the latest event in the + // slot, or -1 if no event fell in the slot (its value is then the value of the slot before + // it). The value of a slot is reached at that time, so a client drawing the series as a step + // function can place each slot's final change at the time of its latest event rather than at + // the slot boundary. + repeated sint64 last_event_offsets_ms = 4; + } +} diff --git a/third_party/apache_spark/cell_monitor_pb2.py b/third_party/apache_spark/cell_monitor_pb2.py new file mode 100644 index 00000000..e5626815 --- /dev/null +++ b/third_party/apache_spark/cell_monitor_pb2.py @@ -0,0 +1,78 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modifications copyright 2026 Google LLC: +# Generated and post-processed by scripts/regen_protos.py for packaging under +# google.cloud.managed_spark_connect._third_party.apache_spark. +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: cell_monitor.proto +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x12\x63\x65ll_monitor.proto\x12\x17\x64\x61taproc.cellmonitor.v1"\xe2\x01\n\x1b\x43\x65llMonitorTelemetryRequest\x12\x1a\n\rsince_version\x18\x01 \x01(\x03H\x00\x88\x01\x01\x12_\n\x0esince_versions\x18\x02 \x03(\x0b\x32G.dataproc.cellmonitor.v1.CellMonitorTelemetryRequest.SinceVersionsEntry\x1a\x34\n\x12SinceVersionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x03:\x02\x38\x01\x42\x10\n\x0e_since_version"\xf1\n\n\x14\x43\x65llMonitorTelemetry\x12\x0f\n\x07version\x18\x01 \x01(\x03\x12\x11\n\tunchanged\x18\x02 \x01(\x08\x12\x16\n\x0e\x66inal_snapshot\x18\x03 \x01(\x08\x12H\n\x04jobs\x18\x04 \x03(\x0b\x32:.dataproc.cellmonitor.v1.CellMonitorTelemetry.JobTelemetry\x12L\n\x06stages\x18\x05 \x03(\x0b\x32<.dataproc.cellmonitor.v1.CellMonitorTelemetry.StageTelemetry\x12Q\n\rrunning_tasks\x18\x06 \x01(\x0b\x32:.dataproc.cellmonitor.v1.CellMonitorTelemetry.RunningTasks\x1a\x81\x02\n\x0cJobTelemetry\x12\x0e\n\x06job_id\x18\x01 \x01(\x03\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12G\n\x06status\x18\x04 \x01(\x0e\x32\x37.dataproc.cellmonitor.v1.CellMonitorTelemetry.JobStatus\x12\x1c\n\x0fsubmission_time\x18\x05 \x01(\x03H\x00\x88\x01\x01\x12\x1c\n\x0f\x63ompletion_time\x18\x06 \x01(\x03H\x01\x88\x01\x01\x12\x11\n\tstage_ids\x18\x07 \x03(\x03\x42\x12\n\x10_submission_timeB\x12\n\x10_completion_time\x1a\x85\x03\n\x0eStageTelemetry\x12\x10\n\x08stage_id\x18\x01 \x01(\x03\x12\x12\n\nattempt_id\x18\x02 \x01(\x03\x12\x0c\n\x04name\x18\x03 \x01(\t\x12I\n\x06status\x18\x04 \x01(\x0e\x32\x39.dataproc.cellmonitor.v1.CellMonitorTelemetry.StageStatus\x12\x11\n\tnum_tasks\x18\x05 \x01(\x03\x12\x1b\n\x13num_completed_tasks\x18\x06 \x01(\x03\x12\x18\n\x10num_active_tasks\x18\x07 \x01(\x03\x12\x18\n\x10num_failed_tasks\x18\x08 \x01(\x03\x12\x18\n\x10input_bytes_read\x18\t \x01(\x03\x12\x12\n\nparent_ids\x18\n \x03(\x03\x12\x1c\n\x0fsubmission_time\x18\x0b \x01(\x03H\x00\x88\x01\x01\x12\x1c\n\x0f\x63ompletion_time\x18\x0c \x01(\x03H\x01\x88\x01\x01\x42\x12\n\x10_submission_timeB\x12\n\x10_completion_time\x1ah\n\x0cRunningTasks\x12\x15\n\rslot_width_ms\x18\x01 \x01(\x03\x12\x12\n\nstart_time\x18\x02 \x01(\x03\x12\x0e\n\x06values\x18\x03 \x03(\x05\x12\x1d\n\x15last_event_offsets_ms\x18\x04 \x03(\x12"\x8a\x01\n\tJobStatus\x12\x1a\n\x16JOB_STATUS_UNSPECIFIED\x10\x00\x12\x16\n\x12JOB_STATUS_RUNNING\x10\x01\x12\x18\n\x14JOB_STATUS_SUCCEEDED\x10\x02\x12\x15\n\x11JOB_STATUS_FAILED\x10\x03\x12\x18\n\x14JOB_STATUS_CANCELLED\x10\x04"\xae\x01\n\x0bStageStatus\x12\x1c\n\x18STAGE_STATUS_UNSPECIFIED\x10\x00\x12\x18\n\x14STAGE_STATUS_PENDING\x10\x01\x12\x18\n\x14STAGE_STATUS_RUNNING\x10\x02\x12\x1a\n\x16STAGE_STATUS_COMPLETED\x10\x03\x12\x17\n\x13STAGE_STATUS_FAILED\x10\x04\x12\x18\n\x14STAGE_STATUS_SKIPPED\x10\x05\x42+\n\'org.apache.spark.connect.dataproc.protoP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages( + DESCRIPTOR, + "google.cloud.managed_spark_connect._third_party.apache_spark.cell_monitor_pb2", + _globals, +) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = ( + b"\n'org.apache.spark.connect.dataproc.protoP\001" + ) + _globals[ + "_CELLMONITORTELEMETRYREQUEST_SINCEVERSIONSENTRY" + ]._loaded_options = None + _globals[ + "_CELLMONITORTELEMETRYREQUEST_SINCEVERSIONSENTRY" + ]._serialized_options = b"8\001" + _globals["_CELLMONITORTELEMETRYREQUEST"]._serialized_start = 48 + _globals["_CELLMONITORTELEMETRYREQUEST"]._serialized_end = 274 + _globals[ + "_CELLMONITORTELEMETRYREQUEST_SINCEVERSIONSENTRY" + ]._serialized_start = 204 + _globals[ + "_CELLMONITORTELEMETRYREQUEST_SINCEVERSIONSENTRY" + ]._serialized_end = 256 + _globals["_CELLMONITORTELEMETRY"]._serialized_start = 277 + _globals["_CELLMONITORTELEMETRY"]._serialized_end = 1670 + _globals["_CELLMONITORTELEMETRY_JOBTELEMETRY"]._serialized_start = 597 + _globals["_CELLMONITORTELEMETRY_JOBTELEMETRY"]._serialized_end = 854 + _globals["_CELLMONITORTELEMETRY_STAGETELEMETRY"]._serialized_start = 857 + _globals["_CELLMONITORTELEMETRY_STAGETELEMETRY"]._serialized_end = 1246 + _globals["_CELLMONITORTELEMETRY_RUNNINGTASKS"]._serialized_start = 1248 + _globals["_CELLMONITORTELEMETRY_RUNNINGTASKS"]._serialized_end = 1352 + _globals["_CELLMONITORTELEMETRY_JOBSTATUS"]._serialized_start = 1355 + _globals["_CELLMONITORTELEMETRY_JOBSTATUS"]._serialized_end = 1493 + _globals["_CELLMONITORTELEMETRY_STAGESTATUS"]._serialized_start = 1496 + _globals["_CELLMONITORTELEMETRY_STAGESTATUS"]._serialized_end = 1670 +# @@protoc_insertion_point(module_scope) diff --git a/third_party/apache_spark/spark_connect_status.proto b/third_party/apache_spark/spark_connect_status.proto new file mode 100644 index 00000000..b751d3fd --- /dev/null +++ b/third_party/apache_spark/spark_connect_status.proto @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright 2026 Google LLC: + * Extracted minimal UserContext, GetStatusRequest, GetStatusResponse, and + * SparkConnectService.GetStatus definitions from + * sql/connect/common/src/main/protobuf/spark/connect/base.proto so clients on + * PySpark 4.0+ can invoke GetStatus without pulling in full plan protos. + */ + +syntax = 'proto3'; + +package spark.connect; + +import "google/protobuf/any.proto"; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; +option go_package = "internal/generated"; + +// User Context is used to refer to one particular user session that is executing +// queries in the backend. +message UserContext { + string user_id = 1; + string user_name = 2; + + // To extend the existing user context message that is used to identify incoming requests, + // Spark Connect leverages the Any protobuf type that can be used to inject arbitrary other + // messages into this message. Extensions are stored as a `repeated` type to be able to + // handle multiple active extensions. + repeated google.protobuf.Any extensions = 999; +} + +// Next ID: 6 +message GetStatusRequest { + // (Required) + // + // The session_id specifies a Spark session for a user identified by user_context.user_id. + // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + string session_id = 1; + + // (Required) + // + // user_context.user_id and session_id both identify a unique remote spark session on the + // server side. + UserContext user_context = 2; + + // (Optional) + // + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 3; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 4; + + // (Optional) + // + // Get status of operations in the session. + optional OperationStatusRequest operation_status = 5; + + // Extension point for custom status request types. + repeated google.protobuf.Any extensions = 999; + + message OperationStatusRequest { + // Get status of operations with these operation_ids. + // If unset or empty, returns status of all operations in the session. + repeated string operation_ids = 1; + + // Extension point for custom operation-level status requests. + repeated google.protobuf.Any extensions = 999; + } +} + +// Next ID: 4 +message GetStatusResponse { + // Session id of the session for which the status was requested. + string session_id = 1; + + // Server-side generated idempotency key that the client can use to assert that the server side + // session has not changed. + string server_side_session_id = 2; + + // Status information about requested operations. + repeated OperationStatus operation_statuses = 3; + + // Extension point for custom status response types. + repeated google.protobuf.Any extensions = 999; + + // Status information for a single operation. + message OperationStatus { + // The operation_id of the operation. + string operation_id = 1; + + // The current status of the operation. + OperationState state = 2; + + // Extension point for custom operation-level status fields. + repeated google.protobuf.Any extensions = 999; + + enum OperationState { + OPERATION_STATE_UNSPECIFIED = 0; + OPERATION_STATE_UNKNOWN = 1; + OPERATION_STATE_RUNNING = 2; + OPERATION_STATE_TERMINATING = 3; + OPERATION_STATE_SUCCEEDED = 4; + OPERATION_STATE_FAILED = 5; + OPERATION_STATE_CANCELLED = 6; + } + } +} + +// Minimal SparkConnectService subset for GetStatus RPC. +service SparkConnectService { + // Get status information of different types. + rpc GetStatus(GetStatusRequest) returns (GetStatusResponse) {} +} diff --git a/third_party/apache_spark/spark_connect_status_pb2.py b/third_party/apache_spark/spark_connect_status_pb2.py new file mode 100644 index 00000000..4dec13a9 --- /dev/null +++ b/third_party/apache_spark/spark_connect_status_pb2.py @@ -0,0 +1,86 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modifications copyright 2026 Google LLC: +# Generated and post-processed by scripts/regen_protos.py for packaging under +# google.cloud.managed_spark_connect._third_party.apache_spark. +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: spark_connect_status.proto +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2 + +_POOL = _descriptor_pool.DescriptorPool() +_ANY_DESCRIPTOR = _POOL.AddSerializedFile( + google_dot_protobuf_dot_any__pb2.DESCRIPTOR.serialized_pb +) +DESCRIPTOR = _POOL.AddSerializedFile( + b'\n\x1aspark_connect_status.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto"\\\n\x0bUserContext\x12\x0f\n\x07user_id\x18\x01 \x01(\t\x12\x11\n\tuser_name\x18\x02 \x01(\t\x12)\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.Any"\xd5\x03\n\x10GetStatusRequest\x12\x12\n\nsession_id\x18\x01 \x01(\t\x12\x30\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContext\x12\x18\n\x0b\x63lient_type\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x33\n&client_observed_server_side_session_id\x18\x04 \x01(\tH\x01\x88\x01\x01\x12U\n\x10operation_status\x18\x05 \x01(\x0b\x32\x36.spark.connect.GetStatusRequest.OperationStatusRequestH\x02\x88\x01\x01\x12)\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.Any\x1aZ\n\x16OperationStatusRequest\x12\x15\n\roperation_ids\x18\x01 \x03(\t\x12)\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyB\x0e\n\x0c_client_typeB)\n\'_client_observed_server_side_session_idB\x13\n\x11_operation_status"\xce\x04\n\x11GetStatusResponse\x12\x12\n\nsession_id\x18\x01 \x01(\t\x12\x1e\n\x16server_side_session_id\x18\x02 \x01(\t\x12L\n\x12operation_statuses\x18\x03 \x03(\x0b\x32\x30.spark.connect.GetStatusResponse.OperationStatus\x12)\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.Any\x1a\x8b\x03\n\x0fOperationStatus\x12\x14\n\x0coperation_id\x18\x01 \x01(\t\x12N\n\x05state\x18\x02 \x01(\x0e\x32?.spark.connect.GetStatusResponse.OperationStatus.OperationState\x12)\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.Any"\xe6\x01\n\x0eOperationState\x12\x1f\n\x1bOPERATION_STATE_UNSPECIFIED\x10\x00\x12\x1b\n\x17OPERATION_STATE_UNKNOWN\x10\x01\x12\x1b\n\x17OPERATION_STATE_RUNNING\x10\x02\x12\x1f\n\x1bOPERATION_STATE_TERMINATING\x10\x03\x12\x1d\n\x19OPERATION_STATE_SUCCEEDED\x10\x04\x12\x1a\n\x16OPERATION_STATE_FAILED\x10\x05\x12\x1d\n\x19OPERATION_STATE_CANCELLED\x10\x06\x32g\n\x13SparkConnectService\x12P\n\tGetStatus\x12\x1f.spark.connect.GetStatusRequest\x1a .spark.connect.GetStatusResponse"\x00\x42\x36\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_orig_sym_db = _builder._sym_db +try: + _builder._sym_db = _symbol_database.SymbolDatabase(pool=_POOL) + _builder.BuildTopDescriptorsAndMessages( + _ANY_DESCRIPTOR, + "google.cloud.managed_spark_connect._third_party.apache_spark.spark_connect_status_pb2", + {}, + ) + _builder.BuildTopDescriptorsAndMessages( + DESCRIPTOR, + "google.cloud.managed_spark_connect._third_party.apache_spark.spark_connect_status_pb2", + _globals, + ) +finally: + _builder._sym_db = _orig_sym_db +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = ( + b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated" + ) + _globals["_USERCONTEXT"]._serialized_start = 72 + _globals["_USERCONTEXT"]._serialized_end = 164 + _globals["_GETSTATUSREQUEST"]._serialized_start = 167 + _globals["_GETSTATUSREQUEST"]._serialized_end = 636 + _globals["_GETSTATUSREQUEST_OPERATIONSTATUSREQUEST"]._serialized_start = 466 + _globals["_GETSTATUSREQUEST_OPERATIONSTATUSREQUEST"]._serialized_end = 556 + _globals["_GETSTATUSRESPONSE"]._serialized_start = 639 + _globals["_GETSTATUSRESPONSE"]._serialized_end = 1229 + _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS"]._serialized_start = 834 + _globals["_GETSTATUSRESPONSE_OPERATIONSTATUS"]._serialized_end = 1229 + _globals[ + "_GETSTATUSRESPONSE_OPERATIONSTATUS_OPERATIONSTATE" + ]._serialized_start = 999 + _globals[ + "_GETSTATUSRESPONSE_OPERATIONSTATUS_OPERATIONSTATE" + ]._serialized_end = 1229 + _globals["_SPARKCONNECTSERVICE"]._serialized_start = 1231 + _globals["_SPARKCONNECTSERVICE"]._serialized_end = 1334 +# @@protoc_insertion_point(module_scope)