From 7614f34f133727fc744bba51e3c613496d707960 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Fri, 21 Aug 2026 19:20:49 -0700 Subject: [PATCH 1/2] Fix Variant field name comparisons to use UTF-8 byte order --- .../org/apache/parquet/variant/Variant.java | 2 +- .../parquet/variant/VariantBuilder.java | 2 +- .../apache/parquet/variant/VariantUtil.java | 17 +++++ .../variant/TestVariantObjectBuilder.java | 68 +++++++++++++++++++ 4 files changed, 87 insertions(+), 2 deletions(-) diff --git a/parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java b/parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java index 3fdfc00605..0eb2ba48c1 100644 --- a/parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java +++ b/parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java @@ -282,7 +282,7 @@ public Variant getFieldByKey(String key) { int mid = (low + high) >>> 1; int midId = VariantUtil.readUnsignedLittleEndian(value, idStart + info.idSize * mid, info.idSize); String midKey = getMetadataKeyCached(midId); - int cmp = midKey.compareTo(key); + int cmp = VariantUtil.compareKeys(midKey, key); if (cmp < 0) { low = mid + 1; } else if (cmp > 0) { diff --git a/parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java b/parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java index c692d3119e..61ee7782c3 100644 --- a/parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java +++ b/parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java @@ -691,7 +691,7 @@ void updateValueSize(int size) { @Override public int compareTo(FieldEntry other) { - return key.compareTo(other.key); + return VariantUtil.compareKeys(key, other.key); } } diff --git a/parquet-variant/src/main/java/org/apache/parquet/variant/VariantUtil.java b/parquet-variant/src/main/java/org/apache/parquet/variant/VariantUtil.java index ad7165fcfe..2c5f3c3287 100644 --- a/parquet-variant/src/main/java/org/apache/parquet/variant/VariantUtil.java +++ b/parquet-variant/src/main/java/org/apache/parquet/variant/VariantUtil.java @@ -301,6 +301,23 @@ static int readUnsigned(ByteBuffer bytes, int pos, int numBytes) { return result; } + /** + * Compares two object field keys by the unsigned lexicographic order of their UTF-8 encoded + * bytes, as required by the Variant spec for object field ordering. + * + *

This intentionally differs from {@link String#compareTo}, which compares UTF-16 code + * units. The two orderings agree for all keys in the Basic Multilingual Plane but diverge for + * supplementary-plane characters (U+10000 and above): {@code String#compareTo} orders a leading + * high surrogate (0xD800-0xDBFF) before code points in U+E000..U+FFFF, whereas UTF-8 byte order + * (and the spec) orders them after. Using UTF-16 order here would produce objects whose field + * ids are mis-sorted relative to the spec, breaking binary-search lookups by any reader that + * follows the spec's UTF-8 byte ordering. + */ + static int compareKeys(String a, String b) { + return Arrays.compareUnsigned( + a.getBytes(StandardCharsets.UTF_8), b.getBytes(StandardCharsets.UTF_8)); + } + /** * Fast little-endian unsigned read using bulk ByteBuffer operations. * Requires the buffer to have {@link java.nio.ByteOrder#LITTLE_ENDIAN} byte order. diff --git a/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantObjectBuilder.java b/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantObjectBuilder.java index d739fdba15..67d0141564 100644 --- a/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantObjectBuilder.java +++ b/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantObjectBuilder.java @@ -85,6 +85,74 @@ public void testLargeObjectBuilder() { }); } + /** + * Object field keys must be ordered by the unsigned byte order of their UTF-8 encoding, not by + * {@link String#compareTo} (UTF-16 code-unit order). The two orderings disagree for + * supplementary-plane keys: U+FFFF encodes to UTF-8 {@code EF BF BF} and U+10000 to + * {@code F0 90 80 80}, so U+FFFF must sort first; but in UTF-16 the leading high surrogate + * 0xD800 of U+10000 sorts before 0xFFFF, which would wrongly put U+10000 first. See + * {@link VariantUtil#compareKeys}. + */ + @Test + public void testObjectKeysSortedByUtf8ByteOrder() { + String bmpKey = "￿"; // U+FFFF -> UTF-8 EF BF BF + String supplementaryKey = "𐀀"; // U+10000 -> UTF-8 F0 90 80 80 + + VariantBuilder b = new VariantBuilder(); + VariantObjectBuilder o = b.startObject(); + // Appended in the "wrong" order on purpose, to prove the builder sorts rather than + // preserving insertion order. + o.appendKey(supplementaryKey); + o.appendLong(2); + o.appendKey(bmpKey); + o.appendLong(1); + b.endObject(); + + VariantTestUtil.testVariant(b.build(), v -> { + VariantTestUtil.checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT); + assertThat(v.numObjectElements()).isEqualTo(2); + // UTF-8 byte order: EF BF BF < F0 90 80 80, so the BMP key comes first. + assertThat(v.getFieldAtIndex(0).key).isEqualTo(bmpKey); + assertThat(v.getFieldAtIndex(1).key).isEqualTo(supplementaryKey); + assertThat(v.getFieldByKey(bmpKey).getLong()).isEqualTo(1); + assertThat(v.getFieldByKey(supplementaryKey).getLong()).isEqualTo(2); + }); + } + + /** + * A large object (>= BINARY_SEARCH_THRESHOLD) that mixes ASCII keys with U+FFFF and a + * supplementary-plane key, exercising the reader's binary-search path in + * {@link Variant#getFieldByKey}. The binary search must use the same UTF-8 byte ordering as the + * builder's sort; with a UTF-16 comparator on the read side, the supplementary key would be + * mis-navigated and not found. + */ + @Test + public void testLargeObjectBinarySearchWithSupplementaryKey() { + String bmpKey = "￿"; // UTF-8 EF BF BF + String supplementaryKey = "𐀀"; // UTF-8 F0 90 80 80 + + VariantBuilder b = new VariantBuilder(); + VariantObjectBuilder o = b.startObject(); + for (int i = 0; i < 40; i++) { // well above BINARY_SEARCH_THRESHOLD (32) + o.appendKey(String.format("a%03d", i)); + o.appendLong(i); + } + o.appendKey(bmpKey); + o.appendLong(998); + o.appendKey(supplementaryKey); + o.appendLong(999); + b.endObject(); + + VariantTestUtil.testVariant(b.build(), v -> { + assertThat(v.numObjectElements()).isEqualTo(42); + assertThat(v.getFieldByKey(bmpKey)).isNotNull(); + assertThat(v.getFieldByKey(bmpKey).getLong()).isEqualTo(998); + assertThat(v.getFieldByKey(supplementaryKey)).isNotNull(); + assertThat(v.getFieldByKey(supplementaryKey).getLong()).isEqualTo(999); + assertThat(v.getFieldByKey("a037").getLong()).isEqualTo(37); + }); + } + @Test public void testMixedObjectBuilder() { VariantBuilder b = new VariantBuilder(); From eb71e5658df60b729a3b921c9bdd270011f04cd0 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Fri, 21 Aug 2026 21:22:49 -0700 Subject: [PATCH 2/2] Minor optimization --- .../org/apache/parquet/variant/Variant.java | 4 +++- .../apache/parquet/variant/VariantBuilder.java | 15 ++++++++++++++- .../apache/parquet/variant/VariantUtil.java | 18 +++++++++++++----- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java b/parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java index 0eb2ba48c1..5be8d1a497 100644 --- a/parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java +++ b/parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java @@ -273,6 +273,8 @@ public Variant getFieldByKey(String key) { } } } else { + // Encode the lookup key once, outside the loop, rather than on every comparison. + byte[] keyBytes = VariantUtil.encodeKey(key); int low = 0; int high = info.numElements - 1; while (low <= high) { @@ -282,7 +284,7 @@ public Variant getFieldByKey(String key) { int mid = (low + high) >>> 1; int midId = VariantUtil.readUnsignedLittleEndian(value, idStart + info.idSize * mid, info.idSize); String midKey = getMetadataKeyCached(midId); - int cmp = VariantUtil.compareKeys(midKey, key); + int cmp = VariantUtil.compareKeys(VariantUtil.encodeKey(midKey), keyBytes); if (cmp < 0) { low = mid + 1; } else if (cmp > 0) { diff --git a/parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java b/parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java index 61ee7782c3..8525abf0c0 100644 --- a/parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java +++ b/parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java @@ -679,6 +679,12 @@ static final class FieldEntry implements Comparable { final int offset; int valueSize = 0; + /** + * Lazy cache of the UTF-8 encoding of `key`, which sorting an object compares O(log n) times + * per entry. Encoded on demand so single-field objects, which are never compared, skip it. + */ + private byte[] keyBytes; + FieldEntry(String key, int id, int offset) { this.key = key; this.id = id; @@ -689,9 +695,16 @@ void updateValueSize(int size) { valueSize = size; } + private byte[] keyBytes() { + if (keyBytes == null) { + keyBytes = VariantUtil.encodeKey(key); + } + return keyBytes; + } + @Override public int compareTo(FieldEntry other) { - return VariantUtil.compareKeys(key, other.key); + return VariantUtil.compareKeys(keyBytes(), other.keyBytes()); } } diff --git a/parquet-variant/src/main/java/org/apache/parquet/variant/VariantUtil.java b/parquet-variant/src/main/java/org/apache/parquet/variant/VariantUtil.java index 2c5f3c3287..a5aa88f4ab 100644 --- a/parquet-variant/src/main/java/org/apache/parquet/variant/VariantUtil.java +++ b/parquet-variant/src/main/java/org/apache/parquet/variant/VariantUtil.java @@ -302,8 +302,17 @@ static int readUnsigned(ByteBuffer bytes, int pos, int numBytes) { } /** - * Compares two object field keys by the unsigned lexicographic order of their UTF-8 encoded - * bytes, as required by the Variant spec for object field ordering. + * Encodes an object field key to the UTF-8 bytes that {@link #compareKeys} orders. Callers that + * compare the same key repeatedly - sorting an object, or binary-searching it for one key - + * should encode it once and reuse the result rather than re-encoding per comparison. + */ + static byte[] encodeKey(String key) { + return key.getBytes(StandardCharsets.UTF_8); + } + + /** + * Compares two object field keys, given their UTF-8 encodings, by unsigned lexicographic byte + * order, as required by the Variant spec for object field ordering. * *

This intentionally differs from {@link String#compareTo}, which compares UTF-16 code * units. The two orderings agree for all keys in the Basic Multilingual Plane but diverge for @@ -313,9 +322,8 @@ static int readUnsigned(ByteBuffer bytes, int pos, int numBytes) { * ids are mis-sorted relative to the spec, breaking binary-search lookups by any reader that * follows the spec's UTF-8 byte ordering. */ - static int compareKeys(String a, String b) { - return Arrays.compareUnsigned( - a.getBytes(StandardCharsets.UTF_8), b.getBytes(StandardCharsets.UTF_8)); + static int compareKeys(byte[] a, byte[] b) { + return Arrays.compareUnsigned(a, b); } /**