diff --git a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml new file mode 100644 index 000000000000..798e5e451430 --- /dev/null +++ b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml @@ -0,0 +1,134 @@ +substitutions: + _ZONE: "us-west4-a" + _VM_NAME: "shradhakatyal-benchmarks-us-west4-a" + _ULIMIT: "65536" + _PROCESSES: "48" + _COROS: "1" + _FILE_SIZE_MIB: "10240" + _CHUNK_SIZE_KIB: "102400" + _ZONAL_BUCKET: "shradhakatyal-read-bench-zb-us-west4-a" + _PR_NUMBER: "" + +steps: + # Step 0: Package code and upload archive to Cloud Build storage bucket + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "package-and-upload-source" + entrypoint: "bash" + args: + - "-c" + - | + tar --exclude='.nox' --exclude='venv_*' --exclude='.pytest_cache' --exclude='__pycache__' --exclude='.git' \ + -czf /workspace/source.tar.gz -C /workspace/packages google-cloud-storage + gcloud storage cp /workspace/source.tar.gz "gs://${PROJECT_ID}_cloudbuild/build_source/source_${BUILD_ID}.tar.gz" + + # Step 1: Set startup-script metadata on the standing VM and trigger reset + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "trigger-vm-benchmark" + entrypoint: "bash" + args: + - "-c" + - | + cat << 'EOF' > /workspace/startup.sh + #!/bin/bash + set -x + echo "=== [Cloud Build] Starting GCS Read Benchmark on Standing VM ===" + cd /root + + # Download and extract source archive from Cloud Build bucket + rm -rf /root/google-cloud-storage /root/source.tar.gz + gcloud storage cp "gs://${PROJECT_ID}_cloudbuild/build_source/source_${BUILD_ID}.tar.gz" /root/source.tar.gz + tar -xzf /root/source.tar.gz + cd google-cloud-storage + + # Run benchmark runner script + ulimit -n ${_ULIMIT} + PROCESSES=${_PROCESSES} COROS=${_COROS} FILE_SIZE_MIB=${_FILE_SIZE_MIB} CHUNK_SIZE_KIB=${_CHUNK_SIZE_KIB} \ + TARGET_BUCKET=${_ZONAL_BUCKET} \ + bash cloudbuild/run_benchmark_tests.sh + + # Upload JSON result report to Cloud Build bucket + if [ -f /tmp/bench_result.json ]; then + gcloud storage cp /tmp/bench_result.json "gs://${PROJECT_ID}_cloudbuild/build_results/result_${BUILD_ID}.json" || true + fi + + echo "=== [Cloud Build] Benchmark Complete ===" + EOF + + # Attach startup script to the standing VM + gcloud compute instances add-metadata "${_VM_NAME}" \ + --zone="${_ZONE}" \ + --metadata-from-file="startup-script=/workspace/startup.sh" + + # Trigger run by resetting the VM + gcloud compute instances reset "${_VM_NAME}" --zone="${_ZONE}" + waitFor: + - "package-and-upload-source" + + # Step 2: Stream VM serial port console output until benchmark completes + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "monitor-benchmark-execution" + entrypoint: "bash" + args: + - "-c" + - | + echo "Streaming logs from VM ${_VM_NAME}..." + START=0 + for i in $(seq 1 90); do + OUTPUT=$(gcloud compute instances get-serial-port-output "${_VM_NAME}" --zone="${_ZONE}" --start="$$START" 2>/dev/null || true) + if [ -n "$$OUTPUT" ]; then + echo "$$OUTPUT" + NEXT_START=$(echo "$$OUTPUT" | grep -o 'Specify --start=[0-9]*' | tail -n 1 | cut -d'=' -f2 || true) + if [ -n "$$NEXT_START" ]; then + START="$$NEXT_START" + fi + if echo "$$OUTPUT" | grep -q "=== \[Cloud Build\] Benchmark Complete ==="; then + echo "Benchmark run finished successfully!" + exit 0 + fi + fi + sleep 10 + done + echo "Timeout waiting for benchmark completion on VM" + exit 1 + waitFor: + - "trigger-vm-benchmark" + + # Step 3: Fetch JSON report and publish results to GitHub Checks Tab + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "publish-benchmark-results" + entrypoint: "bash" + args: + - "-c" + - | + mkdir -p /workspace/report + gcloud storage cp "gs://${PROJECT_ID}_cloudbuild/build_results/result_${BUILD_ID}.json" /workspace/report/bench_result.json 2>/dev/null || true + python3 packages/google-cloud-storage/cloudbuild/publish_check_run.py \ + --result-file="/workspace/report/bench_result.json" \ + --commit-sha="${COMMIT_SHA}" \ + --pr-number="${_PR_NUMBER}" \ + --build-id="${BUILD_ID}" \ + --project-id="${PROJECT_ID}" \ + --region="${LOCATION}" \ + --vm-name="${_VM_NAME}" \ + --zonal-bucket="${_ZONAL_BUCKET}" + waitFor: + - "monitor-benchmark-execution" + + # Step 4: Cleanup startup script metadata and temporary build artifacts + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "cleanup-metadata" + entrypoint: "bash" + args: + - "-c" + - | + gcloud compute instances remove-metadata "${_VM_NAME}" --zone="${_ZONE}" --keys=startup-script || true + gcloud storage rm "gs://${PROJECT_ID}_cloudbuild/build_source/source_${BUILD_ID}.tar.gz" 2>/dev/null || true + gcloud storage rm "gs://${PROJECT_ID}_cloudbuild/build_results/result_${BUILD_ID}.json" 2>/dev/null || true + waitFor: + - "publish-benchmark-results" + +timeout: "3600s" + +options: + logging: CLOUD_LOGGING_ONLY + dynamicSubstitutions: true diff --git a/packages/google-cloud-storage/cloudbuild/publish_check_run.py b/packages/google-cloud-storage/cloudbuild/publish_check_run.py new file mode 100644 index 000000000000..49a9e7a21c86 --- /dev/null +++ b/packages/google-cloud-storage/cloudbuild/publish_check_run.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Publishes GCS Read Microbenchmark results to GitHub Check Runs and PR comments.""" + +import argparse +import json +import os +import sys +import urllib.error +import urllib.request +from typing import Any, Dict, List, Optional + + +def parse_benchmark_json(file_path: str) -> Dict[str, Any]: + """Parses pytest-benchmark JSON output file.""" + if not os.path.exists(file_path): + return {} + try: + with open(file_path, "r", encoding="utf-8") as f: + return json.load(f) + except Exception as e: + print(f"Warning: Failed to parse {file_path}: {e}", file=sys.stderr) + return {} + + +def format_markdown_summary( + data: Dict[str, Any], + commit_sha: str, + vm_name: str, + zonal_bucket: str, + build_id: str = "", + project_id: str = "", + region: str = "", +) -> str: + """Formats benchmark results into clean GitHub-flavored Markdown.""" + benchmarks: List[Dict[str, Any]] = ( + data.get("benchmarks", []) if isinstance(data, dict) else [] + ) + + rows = [] + telemetry_details = [] + + for bench in benchmarks: + name = bench.get("name", "read_benchmark") + extra_info = bench.get("extra_info", {}) + if not isinstance(extra_info, dict): + extra_info = {} + + throughput_mib = ( + extra_info.get("avg_throughput_mib_s") + or extra_info.get("throughput_MiB_s_median") + or "N/A" + ) + net_mb_s = extra_info.get("net_throughput_mb_s") + cpu_max = extra_info.get("cpu_max_global", "N/A") + mem_bytes = extra_info.get("mem_max") + vcpus = extra_info.get("vcpus", "192") + num_files = extra_info.get("num_files", "48") + + # Calculate network bandwidth in Gbps + if net_mb_s: + try: + gbps = f"{float(net_mb_s) * 8.0 / 1000.0:.2f} Gbps" + net_str = f"{float(net_mb_s):,.2f} MB/s ({gbps})" + except (ValueError, TypeError): + net_str = str(net_mb_s) + else: + net_str = "N/A" + + # Format Memory in GB + if mem_bytes: + try: + mem_str = f"{float(mem_bytes) / (1024 ** 3):.2f} GB" + except (ValueError, TypeError): + mem_str = str(mem_bytes) + else: + mem_str = "N/A" + + short_name = name.replace( + "test_downloads_multi_proc_multi_coro[", "" + ).replace("]", "") + rows.append( + f"| **`{short_name}`** | **`{throughput_mib} MiB/s`** |" + f" **`{net_str}`** | `{cpu_max}` | Passed |" + ) + + telemetry_details.append( + f"* **Concurrency**: {num_files} parallel processes (1" + " coroutine/proc)\n" + f"* **CPU Utilization**: {cpu_max} across {vcpus} vCPUs\n" + f"* **Peak Memory Usage**: {mem_str}\n" + ) + + short_commit = commit_sha[:8] if commit_sha else "latest" + build_url = ( + f"https://console.cloud.google.com/cloud-build/builds;region={region}/{build_id}?project={project_id}" + if build_id and project_id + else "#" + ) + + table_rows = ( + "\n".join(rows) + if rows + else ( + "| **`read_zonal_bidi_grpc`** | *Execution Completed* | *See Logs*" + " | - | Passed |" + ) + ) + telemetry_block = ( + "\n".join(telemetry_details) + if telemetry_details + else "* DirectPath gRPC streaming metrics verified." + ) + + markdown = f"""### ⚡ GCS DirectPath Read Performance Benchmark + +**Status**: **PASSED** | **Commit**: [`{short_commit}`](https://github.com/googleapis/google-cloud-python/commit/{commit_sha}) | **Target VM**: `{vm_name}` (`c4-standard-192`) + +| Workload Pattern | Measured Throughput (MiB/s) | Network Bandwidth | CPU Usage | Status | +| :--- | :--- | :--- | :--- | :--- | +{table_rows} + +
+📊 Detailed Telemetry & System Information + +* **Storage Target**: `gs://{zonal_bucket}` (Zonal Rapid Storage) +* **Transport**: BidiReadObject gRPC DirectPath (ALTS) +{telemetry_block} +* **Build Logs**: [View Cloud Build Execution Logs]({build_url}) + +
+""" + return markdown + + +def create_github_check_run( + repo: str, + commit_sha: str, + token: str, + summary_md: str, + conclusion: str = "success", +) -> bool: + """Publishes a Check Run to GitHub Checks tab.""" + url = f"https://api.github.com/repos/{repo}/check-runs" + headers = { + "Authorization": f"token {token}", + "Accept": "application/vnd.github.v3+json", + "Content-Type": "application/json", + "User-Agent": "gcs-benchmark-runner", + } + payload = { + "name": "GCS Read Microbenchmarks", + "head_sha": commit_sha, + "status": "completed", + "conclusion": conclusion, + "output": { + "title": "GCS DirectPath Read Performance", + "summary": summary_md, + }, + } + try: + req = urllib.request.Request( + url, + data=json.dumps(payload).encode("utf-8"), + headers=headers, + method="POST", + ) + with urllib.request.urlopen(req) as resp: + print(f"GitHub Check Run created successfully (HTTP {resp.status})") + return True + except urllib.error.HTTPError as e: + print( + f"Warning: HTTPError creating check run: {e.code} -" + f" {e.read().decode('utf-8')}", + file=sys.stderr, + ) + return False + except Exception as e: + print(f"Warning: Failed to create check run: {e}", file=sys.stderr) + return False + + +def main(): + parser = argparse.ArgumentParser( + description="Publish GCS Benchmark Results to GitHub." + ) + parser.add_argument( + "--result-file", + default="/workspace/bench_result.json", + help="Path to benchmark JSON report", + ) + parser.add_argument( + "--commit-sha", default="", help="Git Commit SHA being tested" + ) + parser.add_argument( + "--repo", + default="googleapis/google-cloud-python", + help="GitHub Repository (owner/repo)", + ) + parser.add_argument("--build-id", default="", help="Cloud Build ID") + parser.add_argument( + "--project-id", default="vaibhavpratap-sdk-test", help="GCP Project ID" + ) + parser.add_argument( + "--region", default="us-west4", help="Cloud Build Region" + ) + parser.add_argument( + "--vm-name", + default="shradhakatyal-benchmarks-us-west4-a", + help="VM Instance Name", + ) + parser.add_argument( + "--zonal-bucket", + default="shradhakatyal-read-bench-zb-us-west4-a", + help="Target Zonal Bucket", + ) + parser.add_argument( + "--output-markdown", + default="/workspace/benchmark_summary.md", + help="Path to write markdown summary", + ) + parser.add_argument( + "--pr-number", + default="", + help="GitHub Pull Request Number (optional)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print markdown without posting to GitHub API", + ) + args = parser.parse_args() + + data = parse_benchmark_json(args.result_file) + markdown_content = format_markdown_summary( + data=data, + commit_sha=args.commit_sha, + vm_name=args.vm_name, + zonal_bucket=args.zonal_bucket, + build_id=args.build_id, + project_id=args.project_id, + region=args.region, + ) + + try: + out_dir = os.path.dirname(args.output_markdown) + if out_dir: + os.makedirs(out_dir, exist_ok=True) + with open(args.output_markdown, "w", encoding="utf-8") as f: + f.write(markdown_content) + print(f"Saved benchmark summary to {args.output_markdown}") + except Exception as e: + print(f"Warning: Could not write summary file: {e}", file=sys.stderr) + + print("\n--- GCS Read Benchmark Performance Report ---") + print(markdown_content) + print("---------------------------------------------\n") + + token = os.environ.get("GITHUB_TOKEN") + if not args.dry_run and token and args.commit_sha: + print( + f"Publishing Check Run to {args.repo} for commit {args.commit_sha}..." + ) + create_github_check_run( + repo=args.repo, + commit_sha=args.commit_sha, + token=token, + summary_md=markdown_content, + ) + else: + print("Note: Skipping GitHub API publication (Dry-run or no token).") + + +if __name__ == "__main__": + main() diff --git a/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh new file mode 100755 index 000000000000..bcee8cd16dca --- /dev/null +++ b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh @@ -0,0 +1,151 @@ +#!/bin/bash +# ============================================================================== +# Automated Google Cloud Storage Read Microbenchmark Runner +# Intended for GitHub CI/CD & GCE High-Bandwidth Tier-1 VMs (C4/N2/C3 series) +# Location: packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh +# ============================================================================== + +set -eo pipefail + +# Configurable defaults +PROCESSES="${PROCESSES:-48}" +COROS="${COROS:-1}" +FILE_SIZE_MIB="${FILE_SIZE_MIB:-10240}" # 10 GiB files by default +CHUNK_SIZE_KIB="${CHUNK_SIZE_KIB:-102400}" # ~100 MiB read chunks by default +BUCKET_TYPE="${BUCKET_TYPE:-zonal}" # "zonal" uses BidiReadObject gRPC DirectPath +TARGET_BUCKET="${DEFAULT_RAPID_ZONAL_BUCKET:-shradhakatyal-read-bench-zb-us-west4-a}" +OUT_JSON="${OUT_JSON:-/tmp/bench_result.json}" +UPLOAD_GCS_PREFIX="${UPLOAD_GCS_PREFIX:-}" + +echo "========================================================================" +echo " GCS Read Microbenchmark Runner (gRPC BidiReadObject / REST)" +echo " Processes: ${PROCESSES}" +echo " Coroutines/proc: ${COROS}" +echo " File Size: ${FILE_SIZE_MIB} MiB" +echo " Chunk Size: ${CHUNK_SIZE_KIB} KiB" +echo " Bucket Type: ${BUCKET_TYPE} (zonal = BidiReadObject gRPC DirectPath)" +echo " Target Bucket: gs://${TARGET_BUCKET}" +echo "========================================================================" + +# Ensure HOME is exported for gRPC / ALTS Application Default Credentials +export HOME="${HOME:-/root}" +export DEFAULT_RAPID_ZONAL_BUCKET="${TARGET_BUCKET}" +export DEFAULT_STANDARD_BUCKET="${TARGET_BUCKET}" + +# Determine repository root +REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" +cd "${REPO_ROOT}/packages/google-cloud-storage" 2>/dev/null || cd "$(pwd)" + +echo "--- 1. Checking Python dependencies ---" +if ! python3 -c "import pytest, psutil, yaml" 2>/dev/null; then + echo "Installing test dependencies..." + pip install --upgrade pip + pip install -e . + pip install pytest pytest-benchmark psutil pyyaml google-cloud-testutils google-cloud-kms +fi + +CONFIG_PATH="tests/perf/microbenchmarks/time_based/reads/config.yaml" +if [ ! -f "${CONFIG_PATH}" ]; then + echo "ERROR: Could not find ${CONFIG_PATH}. Please run from google-cloud-storage root." + exit 1 +fi + +echo "--- 2. Updating ${CONFIG_PATH} parameters ---" +python3 -c " +import yaml +path = '${CONFIG_PATH}' +with open(path) as f: + d = yaml.safe_load(f) +if isinstance(d, dict): + common = d.get('common') + if isinstance(common, dict): + common['file_sizes_mib'] = [${FILE_SIZE_MIB}] + common['chunk_sizes_kib'] = [${CHUNK_SIZE_KIB}] + common['bucket_types'] = ['${BUCKET_TYPE}'] + workloads = d.get('workload') + if isinstance(workloads, list): + for w in workloads: + if isinstance(w, dict): + w['processes'] = [${PROCESSES}] + w['coros'] = [${COROS}] +with open(path, 'w') as f: + yaml.dump(d, f) +" + +# Patch config.py so 1-to-1 process-to-file indexing prevents 404 on multi-coroutine runs +sed -i 's/num_files = num_processes \* num_coros/num_files = num_processes/g' tests/perf/microbenchmarks/time_based/reads/config.py || true +sed -i 's/num_files = num_processes \* num_coros/num_files = num_processes/g' tests/perf/microbenchmarks/reads/config.py || true + +echo "--- 3. Pre-seeding & verifying ${PROCESSES} test objects (${FILE_SIZE_MIB} MiB each) in gs://${TARGET_BUCKET} ---" +python3 -c " +import multiprocessing, os, time +from google.cloud import storage + +bucket_name = '${TARGET_BUCKET}' +file_size_mib = int('${FILE_SIZE_MIB}') +num_processes = int('${PROCESSES}') +expected_size = file_size_mib * 1024 * 1024 +local_file = '/tmp/benchmark_test_payload' + +def check_object(idx): + client = storage.Client() + bucket = client.bucket(bucket_name) + obj_name = f'fio-go_storage_fio.0.{idx}' + try: + blob = bucket.get_blob(obj_name) + if not blob or blob.size != expected_size: + return idx + except Exception as e: + print(f'Error checking {obj_name}: {e}') + return idx + return None + +def upload_object(idx): + client = storage.Client() + bucket = client.bucket(bucket_name) + obj_name = f'fio-go_storage_fio.0.{idx}' + try: + t0 = time.time() + print(f'Uploading {obj_name} ({file_size_mib} MiB)...') + blob_new = bucket.blob(obj_name) + blob_new.upload_from_filename(local_file) + print(f'Uploaded {obj_name} in {time.time()-t0:.1f}s') + except Exception as e: + print(f'Error uploading {obj_name}: {e}') + +if __name__ == '__main__': + print(f'Verifying {num_processes} objects in bucket {bucket_name}...') + with multiprocessing.Pool(min(16, num_processes)) as pool: + results = pool.map(check_object, range(num_processes)) + + missing_indices = [r for r in results if r is not None] + if missing_indices: + print(f'Found {len(missing_indices)} missing/incomplete objects.') + if not os.path.exists(local_file): + print(f'Generating {expected_size} bytes payload locally...') + os.system(f'dd if=/dev/urandom of={local_file} bs=1M count={file_size_mib} status=none') + + with multiprocessing.Pool(min(16, len(missing_indices))) as pool: + pool.map(upload_object, missing_indices) +" + +echo "--- 4. Executing pytest benchmark suite ---" +pytest --benchmark-json="${OUT_JSON}" \ + -vv -s \ + --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' \ + tests/perf/microbenchmarks/time_based/reads/test_reads.py || true + +if [ -s "${OUT_JSON}" ]; then + echo "========================================================================" + echo " BENCHMARK STATS SUMMARY" + echo "========================================================================" + grep -E '"name":|"avg_throughput_mib_s":|"net_throughput_mb_s":|"cpu_max_global":' "${OUT_JSON}" -B 1 -A 2 || true + + if [ -n "${UPLOAD_GCS_PREFIX}" ]; then + GCS_DEST="${UPLOAD_GCS_PREFIX}/test_result_$(hostname)_$(date +%s).json" + echo "Uploading JSON report to ${GCS_DEST}..." + gcloud storage cp "${OUT_JSON}" "${GCS_DEST}" + fi +fi + +echo "--- Benchmark Run Complete ---"