GH-3708: Inline parquet.thrift

dev/check-parquet-thrift-release.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Verifies that the parquet-format.version sidecar file is tracking a released parquet-format
+# version and that the inlined parquet.thrift byte-matches the upstream file at that version.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+THRIFT_FILE="${REPO_ROOT}/parquet-format-structures/src/main/thrift/parquet.thrift"
+SIDECAR_FILE="${REPO_ROOT}/parquet-format-structures/src/main/thrift/parquet-format.version"
+
+# shellcheck source=parquet-thrift-lib.sh
+source "${SCRIPT_DIR}/parquet-thrift-lib.sh"
+
+if [[ ! -f "$SIDECAR_FILE" ]]; then
+  echo "ERROR: sidecar not found: ${SIDECAR_FILE}" >&2
+  exit 1
+fi
+
+if [[ ! -f "$THRIFT_FILE" ]]; then
+  echo "ERROR: inlined thrift not found: ${THRIFT_FILE}" >&2
+  exit 1
+fi
+
+version="$(grep '^parquet-format.version=' "$SIDECAR_FILE" | cut -d= -f2 || true)"
+commit="$(grep '^parquet-format.commit=' "$SIDECAR_FILE" | cut -d= -f2 || true)"
+
+# Require a valid X.Y.Z release version
+if [[ -z "$version" ]] || ! [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+  echo "ERROR: inlined parquet.thrift is not from a released parquet-format version." >&2
+  echo "  parquet-format.version = '${version}'" >&2
+  echo "" >&2
+  echo "A parquet-java release must use a released parquet-format IDL." >&2
+  echo "Run: dev/update-parquet-thrift.sh --version <X.Y.Z>" >&2
+  exit 1
+fi
+
+echo "Checking inlined parquet.thrift against parquet-format ${version} (commit ${commit}) ..."
+
+# Fetch the upstream IDL from the release tag.
+tmp="$(mktemp)"
+trap 'rm -f "$tmp"' EXIT
+
+if ! fetch_thrift "apache-parquet-format-${version}" "$tmp"; then
+  echo "ERROR: failed to fetch parquet.thrift for tag apache-parquet-format-${version}" >&2
+  echo "Cannot verify inlined IDL — failing closed." >&2
+  exit 1
+fi
+
+if [[ ! -s "$tmp" ]]; then
+  echo "ERROR: fetched parquet.thrift is empty for tag apache-parquet-format-${version}" >&2
+  echo "Cannot verify inlined IDL — failing closed." >&2
+  exit 1
+fi
+
+if diff -u "$tmp" "$THRIFT_FILE"; then
+  echo "OK: inlined parquet.thrift matches parquet-format ${version}."
+  exit 0
+else
+  echo "" >&2
+  echo "ERROR: inlined parquet.thrift differs from parquet-format ${version}." >&2
+  echo "Run: dev/update-parquet-thrift.sh --version ${version}" >&2
+  exit 1
+fi

dev/parquet-thrift-lib.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Shared helpers for managing the inlined parquet.thrift file and its relation to the upstream
+# apache/parquet-format repository.
+
+PARQUET_FORMAT_REPO="https://github.com/apache/parquet-format"
+PARQUET_FORMAT_RAW="https://raw.githubusercontent.com/apache/parquet-format"
+THRIFT_PATH_IN_FORMAT="src/main/thrift/parquet.thrift"
+
+# Resolve an apache/parquet-format release version (X.Y.Z) to its full commit SHA using git
+# ls-remote. Echoes the SHA and returns 0 on success, returns 1 if the tag does not exist, and
+# returns the exit code of any other commands on failure.
+resolve_version_to_commit() {
+  local ver="$1" sha ls_out ls_rc
+  ls_out="$(git ls-remote --tags "$PARQUET_FORMAT_REPO" \
+              "refs/tags/apache-parquet-format-${ver}" \
+              "refs/tags/apache-parquet-format-${ver}^{}")" || { ls_rc=$?; return $ls_rc; }
+  sha="$(printf '%s\n' "$ls_out" \
+         | awk '{ if ($2 ~ /\^\{\}$/) deref=$1; else plain=$1 }
+                END { print (deref != "" ? deref : plain) }')"
+  [[ -n "$sha" ]] || return 1
+  printf '%s\n' "$sha"
+}
+
+# Fetch the parquet.thrift IDL from the parquet-format repo at the given commit
+# sha or tag, writing output to $2. Reads from raw.githubusercontent.com, which
+# serves any commit sha or tag and is not subject to the GitHub REST API rate
+# limit. Returns 0 on success, 1 on any failure.
+# The caller is responsible for providing a temp path for $2 so tracked source
+# files are never overwritten on failure.
+
+# Fetch the canonical parquet.thrift IDL from the parquet-format repo at a given commit SHA or tag
+# (supplied as $1) and write it to the destination specified by $2.
+fetch_thrift() {
+  local ref="$1" dest="$2" url attempt
+  url="${PARQUET_FORMAT_RAW}/${ref}/${THRIFT_PATH_IN_FORMAT}"
+  for attempt in 1 2 3; do
+    if curl -fsSL -o "$dest" "$url" && [[ -s "$dest" ]]; then
+      return 0
+    fi
+    if [[ "$attempt" -eq 3 ]]; then
+      echo "ERROR: failed to fetch ${url} after ${attempt} attempts" >&2
+      return 1
+    fi
+    sleep $((attempt * 2))
+  done
+}

dev/prepare-release.sh

@@ -38,6 +38,9 @@ new_development_version="$release_version-SNAPSHOT"
 
 tag="apache-parquet-$release_version-rc$2"
 
+# Ensure the inlined parquet.thrift matches an official parquet-format release.
+"$(dirname "$0")/check-parquet-thrift-release.sh"
+
 ./mvnw release:clean
 ./mvnw release:prepare -DskipTests -Darguments=-DskipTests -Dtag="$tag" "-DreleaseVersion=$release_version" -DdevelopmentVersion="$new_development_version"
 

dev/update-parquet-thrift.sh

@@ -0,0 +1,161 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Updates the inlined parquet.thrift and sidecar to a specific parquet-format
+# commit or release.
+#
+# Usage:
+#   update-parquet-thrift.sh <full-40-char-commit-hash>  -- POC: sets version=UNRELEASED
+#   update-parquet-thrift.sh --version <X.Y.Z>           -- release sync: derives commit from tag
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+THRIFT_FILE="${REPO_ROOT}/parquet-format-structures/src/main/thrift/parquet.thrift"
+SIDECAR_FILE="${REPO_ROOT}/parquet-format-structures/src/main/thrift/parquet-format.version"
+
+# shellcheck source=parquet-thrift-lib.sh
+source "${SCRIPT_DIR}/parquet-thrift-lib.sh"
+
+usage() {
+  cat <<EOF
+Usage:
+  $0 <full-40-char-commit-hash>  POC sync — fetch parquet.thrift at that commit; version=UNRELEASED
+  $0 --version <X.Y.Z>           Release sync — resolve tag apache-parquet-format-<X.Y.Z>
+
+Only one of a commit hash or --version may be given, not both.
+Commit mode requires a full 40-character sha (short shas cannot be fetched from a remote).
+EOF
+  exit 1
+}
+
+if [[ $# -eq 0 ]]; then
+  usage
+fi
+
+mode=""
+commit_arg=""
+version_arg=""
+
+if [[ "$1" == "--version" ]]; then
+  [[ $# -ne 2 ]] && usage
+  mode="version"
+  version_arg="$2"
+elif [[ "$1" == --* ]]; then
+  usage
+else
+  [[ $# -ne 1 ]] && usage
+  mode="commit"
+  commit_arg="$1"
+fi
+
+# Validate arguments
+if [[ "$mode" == "version" ]]; then
+  if ! [[ "$version_arg" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    echo "ERROR: version must be X.Y.Z (e.g. 2.13.0), got: $version_arg" >&2
+    exit 1
+  fi
+else
+  # Full 40-char sha required — short shas cannot be fetched from a remote (git exits 128).
+  if ! [[ "$commit_arg" =~ ^[0-9a-f]{40}$ ]]; then
+    echo "ERROR: commit hash must be exactly 40 hex chars, got: $commit_arg" >&2
+    echo "       Obtain the full sha with: git ls-remote ${PARQUET_FORMAT_REPO} HEAD" >&2
+    exit 1
+  fi
+fi
+
+# Resolve the commit sha to use
+resolved_sha=""
+resolved_version=""
+
+if [[ "$mode" == "version" ]]; then
+  echo "Resolving tag apache-parquet-format-${version_arg} ..."
+  resolved_sha="$(resolve_version_to_commit "$version_arg")" || {
+    rc=$?
+    if [[ $rc -eq 1 ]]; then
+      echo "ERROR: tag apache-parquet-format-${version_arg} not found in ${PARQUET_FORMAT_REPO}" >&2
+      echo "       Check that the version exists: git ls-remote --tags ${PARQUET_FORMAT_REPO}" >&2
+    else
+      echo "ERROR: git/network failure resolving tag apache-parquet-format-${version_arg}" >&2
+    fi
+    exit 1
+  }
+  resolved_version="$version_arg"
+  echo "  -> commit: $resolved_sha"
+else
+  resolved_sha="$commit_arg"
+  resolved_version="UNRELEASED"
+fi
+
+# Read the old commit from the sidecar (for the summary)
+old_commit="(none)"
+if [[ -f "$SIDECAR_FILE" ]]; then
+  old_commit="$(grep '^parquet-format.commit=' "$SIDECAR_FILE" | cut -d= -f2 || true)"
+fi
+
+# Fetch upstream parquet.thrift to a temp file. Only overwrite the inlined file after fetch and
+# validation succeed, so a failure never deletes the inlined version.
+tmp_thrift="$(mktemp)"
+trap 'rm -f "$tmp_thrift"' EXIT
+
+echo "Fetching parquet.thrift at ${resolved_sha} ..."
+if ! fetch_thrift "$resolved_sha" "$tmp_thrift"; then
+  echo "ERROR: could not fetch parquet.thrift at ${resolved_sha}" >&2
+  echo "       The tracked files were not modified." >&2
+  exit 1
+fi
+
+if [[ ! -s "$tmp_thrift" ]]; then
+  echo "ERROR: fetched parquet.thrift is empty" >&2
+  echo "       The tracked files were not modified." >&2
+  exit 1
+fi
+if ! grep -q 'namespace java org.apache.parquet.format' "$tmp_thrift"; then
+  echo "ERROR: fetched file does not look like parquet.thrift (missing namespace declaration)" >&2
+  echo "       The tracked files were not modified." >&2
+  exit 1
+fi
+
+# Commit the validated thrift file, then write the sidecar.
+mv "$tmp_thrift" "$THRIFT_FILE"
+chmod 644 "$THRIFT_FILE"
+echo "  -> written to ${THRIFT_FILE}"
+
+cat > "$SIDECAR_FILE" <<SIDECAR
+# Provenance of the inlined parquet.thrift. Maintained by dev/update-parquet-thrift.sh.
+parquet-format.commit=${resolved_sha}
+parquet-format.version=${resolved_version}
+SIDECAR
+echo "  -> sidecar updated: ${SIDECAR_FILE}"
+
+# Summary
+echo ""
+echo "Update complete:"
+echo "  old commit: ${old_commit}"
+echo "  new commit: ${resolved_sha}"
+if [[ "$resolved_version" == "UNRELEASED" ]]; then
+  echo "  version:    UNRELEASED (POC sync — run --version <ver> for a release sync)"
+else
+  echo "  version:    ${resolved_version} (release sync)"
+fi
+echo ""
+echo "Next steps: rebuild parquet-format-structures and review the diff:"
+echo "  ./mvnw -pl parquet-format-structures -am install -DskipTests"
+echo "  git diff parquet-format-structures/src/main/thrift/parquet.thrift"

parquet-format-structures/pom.xml

@@ -34,44 +34,14 @@
   <url>https://parquet.apache.org/</url>
   <description>Parquet-mr related java classes to use the parquet-format thrift structures.</description>
 
-  <properties>
-    <parquet.thrift.path>${project.build.directory}/parquet-format-thrift</parquet.thrift.path>
-  </properties>
-
   <build>
     <plugins>
-      <!-- Getting the parquet-format thrift file -->
-       <plugin>
-         <groupId>org.apache.maven.plugins</groupId>
-         <artifactId>maven-dependency-plugin</artifactId>
-         <executions>
-           <execution>
-             <id>unpack</id>
-             <phase>initialize</phase>
-             <goals>
-               <goal>unpack</goal>
-             </goals>
-             <configuration>
-               <artifactItems>
-                 <artifactItem>
-                   <groupId>org.apache.parquet</groupId>
-                   <artifactId>parquet-format</artifactId>
-                   <version>${parquet.format.version}</version>
-                   <type>jar</type>
-                 </artifactItem>
-               </artifactItems>
-               <includes>parquet.thrift</includes>
-               <outputDirectory>${parquet.thrift.path}</outputDirectory>
-             </configuration>
-           </execution>
-         </executions>
-       </plugin>
       <!-- thrift -->
       <plugin>
         <groupId>org.apache.thrift</groupId>
         <artifactId>thrift-maven-plugin</artifactId>
         <configuration>
-          <thriftSourceRoot>${parquet.thrift.path}</thriftSourceRoot>
+          <thriftSourceRoot>${project.basedir}/src/main/thrift</thriftSourceRoot>
           <thriftExecutable>${format.thrift.executable}</thriftExecutable>
         </configuration>
         <executions>

parquet-format-structures/src/main/thrift/parquet-format.version

@@ -0,0 +1,3 @@
+# Provenance of the inlined parquet.thrift. Maintained by dev/update-parquet-thrift.sh.
+parquet-format.commit=c47e2a66e88943fc46fde1b028a9432f14fdf5c0
+parquet-format.version=2.13.0