diff --git a/parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java b/parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java index 3fdfc00605..5be8d1a497 100644 --- a/parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java +++ b/parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java @@ -273,6 +273,8 @@ public Variant getFieldByKey(String key) { } } } else { + // Encode the lookup key once, outside the loop, rather than on every comparison. + byte[] keyBytes = VariantUtil.encodeKey(key); int low = 0; int high = info.numElements - 1; while (low <= high) { @@ -282,7 +284,7 @@ public Variant getFieldByKey(String key) { int mid = (low + high) >>> 1; int midId = VariantUtil.readUnsignedLittleEndian(value, idStart + info.idSize * mid, info.idSize); String midKey = getMetadataKeyCached(midId); - int cmp = midKey.compareTo(key); + int cmp = VariantUtil.compareKeys(VariantUtil.encodeKey(midKey), keyBytes); if (cmp < 0) { low = mid + 1; } else if (cmp > 0) { diff --git a/parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java b/parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java index c692d3119e..8525abf0c0 100644 --- a/parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java +++ b/parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java @@ -679,6 +679,12 @@ static final class FieldEntry implements Comparable { final int offset; int valueSize = 0; + /** + * Lazy cache of the UTF-8 encoding of `key`, which sorting an object compares O(log n) times + * per entry. Encoded on demand so single-field objects, which are never compared, skip it. + */ + private byte[] keyBytes; + FieldEntry(String key, int id, int offset) { this.key = key; this.id = id; @@ -689,9 +695,16 @@ void updateValueSize(int size) { valueSize = size; } + private byte[] keyBytes() { + if (keyBytes == null) { + keyBytes = VariantUtil.encodeKey(key); + } + return keyBytes; + } + @Override public int compareTo(FieldEntry other) { - return key.compareTo(other.key); + return VariantUtil.compareKeys(keyBytes(), other.keyBytes()); } } diff --git a/parquet-variant/src/main/java/org/apache/parquet/variant/VariantUtil.java b/parquet-variant/src/main/java/org/apache/parquet/variant/VariantUtil.java index ad7165fcfe..a5aa88f4ab 100644 --- a/parquet-variant/src/main/java/org/apache/parquet/variant/VariantUtil.java +++ b/parquet-variant/src/main/java/org/apache/parquet/variant/VariantUtil.java @@ -301,6 +301,31 @@ static int readUnsigned(ByteBuffer bytes, int pos, int numBytes) { return result; } + /** + * Encodes an object field key to the UTF-8 bytes that {@link #compareKeys} orders. Callers that + * compare the same key repeatedly - sorting an object, or binary-searching it for one key - + * should encode it once and reuse the result rather than re-encoding per comparison. + */ + static byte[] encodeKey(String key) { + return key.getBytes(StandardCharsets.UTF_8); + } + + /** + * Compares two object field keys, given their UTF-8 encodings, by unsigned lexicographic byte + * order, as required by the Variant spec for object field ordering. + * + *

This intentionally differs from {@link String#compareTo}, which compares UTF-16 code + * units. The two orderings agree for all keys in the Basic Multilingual Plane but diverge for + * supplementary-plane characters (U+10000 and above): {@code String#compareTo} orders a leading + * high surrogate (0xD800-0xDBFF) before code points in U+E000..U+FFFF, whereas UTF-8 byte order + * (and the spec) orders them after. Using UTF-16 order here would produce objects whose field + * ids are mis-sorted relative to the spec, breaking binary-search lookups by any reader that + * follows the spec's UTF-8 byte ordering. + */ + static int compareKeys(byte[] a, byte[] b) { + return Arrays.compareUnsigned(a, b); + } + /** * Fast little-endian unsigned read using bulk ByteBuffer operations. * Requires the buffer to have {@link java.nio.ByteOrder#LITTLE_ENDIAN} byte order. diff --git a/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantObjectBuilder.java b/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantObjectBuilder.java index d739fdba15..67d0141564 100644 --- a/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantObjectBuilder.java +++ b/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantObjectBuilder.java @@ -85,6 +85,74 @@ public void testLargeObjectBuilder() { }); } + /** + * Object field keys must be ordered by the unsigned byte order of their UTF-8 encoding, not by + * {@link String#compareTo} (UTF-16 code-unit order). The two orderings disagree for + * supplementary-plane keys: U+FFFF encodes to UTF-8 {@code EF BF BF} and U+10000 to + * {@code F0 90 80 80}, so U+FFFF must sort first; but in UTF-16 the leading high surrogate + * 0xD800 of U+10000 sorts before 0xFFFF, which would wrongly put U+10000 first. See + * {@link VariantUtil#compareKeys}. + */ + @Test + public void testObjectKeysSortedByUtf8ByteOrder() { + String bmpKey = "￿"; // U+FFFF -> UTF-8 EF BF BF + String supplementaryKey = "𐀀"; // U+10000 -> UTF-8 F0 90 80 80 + + VariantBuilder b = new VariantBuilder(); + VariantObjectBuilder o = b.startObject(); + // Appended in the "wrong" order on purpose, to prove the builder sorts rather than + // preserving insertion order. + o.appendKey(supplementaryKey); + o.appendLong(2); + o.appendKey(bmpKey); + o.appendLong(1); + b.endObject(); + + VariantTestUtil.testVariant(b.build(), v -> { + VariantTestUtil.checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT); + assertThat(v.numObjectElements()).isEqualTo(2); + // UTF-8 byte order: EF BF BF < F0 90 80 80, so the BMP key comes first. + assertThat(v.getFieldAtIndex(0).key).isEqualTo(bmpKey); + assertThat(v.getFieldAtIndex(1).key).isEqualTo(supplementaryKey); + assertThat(v.getFieldByKey(bmpKey).getLong()).isEqualTo(1); + assertThat(v.getFieldByKey(supplementaryKey).getLong()).isEqualTo(2); + }); + } + + /** + * A large object (>= BINARY_SEARCH_THRESHOLD) that mixes ASCII keys with U+FFFF and a + * supplementary-plane key, exercising the reader's binary-search path in + * {@link Variant#getFieldByKey}. The binary search must use the same UTF-8 byte ordering as the + * builder's sort; with a UTF-16 comparator on the read side, the supplementary key would be + * mis-navigated and not found. + */ + @Test + public void testLargeObjectBinarySearchWithSupplementaryKey() { + String bmpKey = "￿"; // UTF-8 EF BF BF + String supplementaryKey = "𐀀"; // UTF-8 F0 90 80 80 + + VariantBuilder b = new VariantBuilder(); + VariantObjectBuilder o = b.startObject(); + for (int i = 0; i < 40; i++) { // well above BINARY_SEARCH_THRESHOLD (32) + o.appendKey(String.format("a%03d", i)); + o.appendLong(i); + } + o.appendKey(bmpKey); + o.appendLong(998); + o.appendKey(supplementaryKey); + o.appendLong(999); + b.endObject(); + + VariantTestUtil.testVariant(b.build(), v -> { + assertThat(v.numObjectElements()).isEqualTo(42); + assertThat(v.getFieldByKey(bmpKey)).isNotNull(); + assertThat(v.getFieldByKey(bmpKey).getLong()).isEqualTo(998); + assertThat(v.getFieldByKey(supplementaryKey)).isNotNull(); + assertThat(v.getFieldByKey(supplementaryKey).getLong()).isEqualTo(999); + assertThat(v.getFieldByKey("a037").getLong()).isEqualTo(37); + }); + } + @Test public void testMixedObjectBuilder() { VariantBuilder b = new VariantBuilder();