From 61689aa0167f9b6a402546f00eaaeed4d043a620 Mon Sep 17 00:00:00 2001 From: Arturo Bernal Date: Fri, 21 Aug 2026 16:08:27 +0200 Subject: [PATCH 1/3] Initial commit --- ...InflatingDictionaryBrotliDataConsumer.java | 274 ++++++++ .../InflatingDictionaryZstdDataConsumer.java | 254 ++++++++ .../BasicCompressionDictionaryStore.java | 191 ++++++ .../compress/CompressionDictionary.java | 143 +++++ .../compress/CompressionDictionaryStore.java | 59 ++ .../http/entity/compress/ContentCoding.java | 10 +- .../async/CompressionDictionaryFreshness.java | 158 +++++ .../CompressionDictionaryHeaderSupport.java | 76 +++ .../async/CompressionDictionaryMatcher.java | 54 ++ ...ompressionDictionaryUrlPatternMatcher.java | 15 + .../async/ContentCompressionAsyncExec.java | 301 ++++++++- .../DefaultCompressionDictionaryMatcher.java | 175 ++++++ .../DictionaryCapturingAsyncDataConsumer.java | 171 +++++ .../impl/async/HttpAsyncClientBuilder.java | 26 +- .../http/impl/async/UseAsDictionary.java | 91 +++ .../AsyncClientCompressionDictionary.java | 218 +++++++ .../AsyncClientServerDictionaryBrotli.java | 591 ++++++++++++++++++ 17 files changed, 2798 insertions(+), 9 deletions(-) create mode 100644 httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/InflatingDictionaryBrotliDataConsumer.java create mode 100644 httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/InflatingDictionaryZstdDataConsumer.java create mode 100644 httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/BasicCompressionDictionaryStore.java create mode 100644 httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CompressionDictionary.java create mode 100644 httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CompressionDictionaryStore.java create mode 100644 httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryFreshness.java create mode 100644 httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryHeaderSupport.java create mode 100644 httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryMatcher.java create mode 100644 httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryUrlPatternMatcher.java create mode 100644 httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/DefaultCompressionDictionaryMatcher.java create mode 100644 httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/DictionaryCapturingAsyncDataConsumer.java create mode 100644 httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/UseAsDictionary.java create mode 100644 httpclient5/src/test/java/org/apache/hc/client5/http/examples/AsyncClientCompressionDictionary.java create mode 100644 httpclient5/src/test/java/org/apache/hc/client5/http/examples/AsyncClientServerDictionaryBrotli.java diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/InflatingDictionaryBrotliDataConsumer.java b/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/InflatingDictionaryBrotliDataConsumer.java new file mode 100644 index 0000000000..e79de8733c --- /dev/null +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/InflatingDictionaryBrotliDataConsumer.java @@ -0,0 +1,274 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.async.methods; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; + +import com.aayushatharva.brotli4j.decoder.DecoderJNI; + +import org.apache.hc.client5.http.entity.compress.CompressionDictionary; +import org.apache.hc.client5.http.entity.compress.CompressionDictionaryStore; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpException; +import org.apache.hc.core5.http.nio.AsyncDataConsumer; +import org.apache.hc.core5.http.nio.CapacityChannel; +import org.apache.hc.core5.util.Args; + +/** + * {@link AsyncDataConsumer} that decodes a Dictionary-Compressed Brotli ({@code dcb}) response + * on the fly and forwards the plain output to a downstream consumer. + *

+ * A {@code dcb} stream begins with a fixed header: the four-byte magic sequence + * {@code 0xFF 0x44 0x43 0x42} followed by the 32-byte SHA-256 hash of the dictionary the origin + * used to compress the body. The header is buffered until complete, the hash is looked up in the + * supplied {@link CompressionDictionaryStore}, and the matching {@link CompressionDictionary} is + * attached to the Brotli decoder as the shared dictionary before any compressed payload is + * decoded. If the hash is unknown, or the header does not validate, the stream is rejected. See + * the Compression Dictionary Transport specification for the dictionary negotiation and framing. + *

+ * This consumer is stateful and not thread-safe; a fresh instance is required per response. + * + * @since 5.7 + */ +public final class InflatingDictionaryBrotliDataConsumer implements AsyncDataConsumer { + + private static final byte[] MAGIC = { + (byte) 0xff, 0x44, 0x43, 0x42 + }; + + private static final int HASH_LENGTH = 32; + private static final int HEADER_LENGTH = MAGIC.length + HASH_LENGTH; + + private final AsyncDataConsumer downstream; + private final CompressionDictionaryStore store; + private final ByteBuffer header; + + private DecoderJNI.Wrapper decoder; + + /** + * Creates a consumer that decodes a {@code dcb} stream and forwards the decoded bytes to + * {@code downstream}. + * + * @param downstream the consumer that receives the decompressed content; must not be {@code null}. + * @param store the store consulted to resolve the dictionary referenced by the stream header; + * must not be {@code null}. + * @since 5.7 + */ + public InflatingDictionaryBrotliDataConsumer( + final AsyncDataConsumer downstream, + final CompressionDictionaryStore store) { + this.downstream = Args.notNull(downstream, "Downstream data consumer"); + this.store = Args.notNull(store, "Dictionary store"); + this.header = ByteBuffer.allocate(HEADER_LENGTH); + } + + /** + * Propagates capacity signalling to the transport. The downstream consumer's demand is relayed + * through a wrapper so that back-pressure it exerts on the decoded output is translated into + * capacity requests on the compressed input. + * + * @since 5.7 + */ + @Override + public void updateCapacity(final CapacityChannel capacityChannel) throws IOException { + downstream.updateCapacity(new InflatingCapacityChannel(capacityChannel)); + } + + /** + * Consumes a chunk of the compressed stream. Until the fixed header has been fully buffered and + * the dictionary resolved, incoming bytes feed header parsing and no output is produced; once the + * decoder is initialised the chunk is handed to it and any decoded bytes are pushed downstream. + * A single call may be a partial header, may straddle the header and payload boundary, or may be + * pure payload. + * + * @param src the next slice of compressed bytes; fully drained on return. + * @throws IOException if the header is malformed, the dictionary is unavailable, the stream is + * corrupt, or trailing bytes follow a completed stream. + * @since 5.7 + */ + @Override + public void consume(final ByteBuffer src) throws IOException { + if (decoder == null) { + consumeHeader(src); + if (decoder == null) { + return; + } + } + + while (src.hasRemaining()) { + if (decoder.getStatus() == DecoderJNI.Status.DONE) { + throw new IOException("Unexpected data after DCB stream"); + } + + final ByteBuffer in = decoder.getInputBuffer(); + in.clear(); + + final int xfer = Math.min(src.remaining(), in.remaining()); + final int lim = src.limit(); + src.limit(src.position() + xfer); + in.put(src); + src.limit(lim); + + decoder.push(xfer); + pump(); + } + } + + private void consumeHeader(final ByteBuffer src) throws IOException { + final int xfer = Math.min(src.remaining(), header.remaining()); + final int lim = src.limit(); + src.limit(src.position() + xfer); + header.put(src); + src.limit(lim); + + if (header.hasRemaining()) { + return; + } + + header.flip(); + + for (final byte expected : MAGIC) { + if (header.get() != expected) { + throw new IOException("Invalid DCB stream header"); + } + } + + final byte[] hash = new byte[HASH_LENGTH]; + header.get(hash); + + final CompressionDictionary dictionary = store.getByHash(hash); + if (dictionary == null || !dictionary.matchesHash(hash)) { + throw new IOException("Compression dictionary not available"); + } + + try { + decoder = new DecoderJNI.Wrapper(8 * 1024); + + final byte[] content = dictionary.getContent(); + final ByteBuffer dictionaryBuffer = ByteBuffer.allocateDirect(content.length); + dictionaryBuffer.put(content); + dictionaryBuffer.flip(); + + if (!decoder.attachDictionary(dictionaryBuffer)) { + decoder.destroy(); + decoder = null; + throw new IOException("Unable to attach Brotli dictionary"); + } + } catch (final IOException ex) { + throw ex; + } catch (final RuntimeException ex) { + throw new IOException("Unable to initialize DCB decoder", ex); + } + } + + private void pump() throws IOException { + for (; ; ) { + switch (decoder.getStatus()) { + case OK: + decoder.push(0); + break; + case NEEDS_MORE_OUTPUT: { + final ByteBuffer nativeBuf = decoder.pull(); + if (nativeBuf != null && nativeBuf.hasRemaining()) { + final ByteBuffer copy = ByteBuffer.allocateDirect(nativeBuf.remaining()); + copy.put(nativeBuf).flip(); + downstream.consume(copy); + } + break; + } + case NEEDS_MORE_INPUT: + if (decoder.hasOutput()) { + final ByteBuffer nativeBuf = decoder.pull(); + if (nativeBuf != null && nativeBuf.hasRemaining()) { + final ByteBuffer copy = ByteBuffer.allocateDirect(nativeBuf.remaining()); + copy.put(nativeBuf).flip(); + downstream.consume(copy); + break; + } + } + return; + case DONE: + if (decoder.hasOutput()) { + final ByteBuffer nativeBuf = decoder.pull(); + if (nativeBuf != null && nativeBuf.hasRemaining()) { + final ByteBuffer copy = ByteBuffer.allocateDirect(nativeBuf.remaining()); + copy.put(nativeBuf).flip(); + downstream.consume(copy); + break; + } + } + return; + default: + throw new IOException("DCB stream corrupted"); + } + } + } + + /** + * Finalises decoding at end of stream. The decoder is drained of any pending output and the + * completion is signalled to the downstream consumer. A stream that ends before the header is + * complete, or before the Brotli decoder reaches its terminal state, is treated as truncated. + * + * @param trailers the response trailers, forwarded unchanged to the downstream consumer. + * @throws IOException if the stream ends prematurely or the decoder has not fully consumed it. + * @since 5.7 + */ + @Override + public void streamEnd(final List trailers) throws IOException, HttpException { + if (header.hasRemaining() || decoder == null) { + throw new IOException("Truncated DCB stream header"); + } + + pump(); + + if (decoder.getStatus() != DecoderJNI.Status.DONE) { + throw new IOException("Truncated DCB stream"); + } + + downstream.streamEnd(trailers); + } + + /** + * Releases the native Brotli decoder and propagates the call to the downstream consumer. Safe to + * invoke whether or not the decoder was ever initialised; failures while destroying the native + * decoder are suppressed so that downstream release always runs. + * + * @since 5.7 + */ + @Override + public void releaseResources() { + if (decoder != null) { + try { + decoder.destroy(); + } catch (final Throwable ignore) { + } + } + downstream.releaseResources(); + } +} \ No newline at end of file diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/InflatingDictionaryZstdDataConsumer.java b/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/InflatingDictionaryZstdDataConsumer.java new file mode 100644 index 0000000000..08df28d4bf --- /dev/null +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/InflatingDictionaryZstdDataConsumer.java @@ -0,0 +1,254 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.async.methods; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import com.github.luben.zstd.ZstdDecompressCtx; + +import org.apache.hc.client5.http.entity.compress.CompressionDictionary; +import org.apache.hc.client5.http.entity.compress.CompressionDictionaryStore; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpException; +import org.apache.hc.core5.http.nio.AsyncDataConsumer; +import org.apache.hc.core5.http.nio.CapacityChannel; +import org.apache.hc.core5.util.Args; + +/** + * {@link AsyncDataConsumer} that inflates a Dictionary-Compressed Zstandard ({@code dcz}) + * response body on the fly and forwards the decoded bytes to a downstream consumer. + *

+ * The stream opens with the {@code dcz} framing prefix defined by Compression Dictionary Transport: an eight-byte + * magic sequence followed by the 32-byte hash of the dictionary the origin used to compress + * the body. The hash is looked up in the supplied {@link CompressionDictionaryStore} and the + * matching {@link CompressionDictionary} is loaded into the Zstandard decompression context + * before any payload is decoded. If the prefix is malformed, or no dictionary matching the + * advertised hash is available, decoding fails with an {@link IOException} so the exchange is + * not silently served undecoded content. + *

+ * Input arrives incrementally and is decoded in bounded direct buffers; output is delivered to + * the downstream consumer as it becomes available, honouring back-pressure by returning early + * whenever the downstream cannot accept the whole batch. Instances are single-use and not + * thread-safe: they follow the sequential {@code AsyncDataConsumer} callback contract. + * + * @since 5.7 + */ +public final class InflatingDictionaryZstdDataConsumer implements AsyncDataConsumer { + + private static final byte[] MAGIC = { + 0x5e, 0x2a, 0x4d, 0x18, 0x20, 0x00, 0x00, 0x00 + }; + + private static final int HASH_LENGTH = 32; + private static final int HEADER_LENGTH = MAGIC.length + HASH_LENGTH; + + private static final int IN_BUF = 64 * 1024; + private static final int OUT_BUF = 128 * 1024; + + private final AsyncDataConsumer downstream; + private final CompressionDictionaryStore store; + private final ByteBuffer header; + private final ZstdDecompressCtx dctx; + private final ByteBuffer inDirect; + private final ByteBuffer outDirect; + private final AtomicBoolean closed; + + private boolean initialized; + private boolean frameComplete; + + /** + * Creates a consumer that decodes a {@code dcz} body and forwards the inflated bytes. + * + * @param downstream the consumer that receives the decoded content; must not be {@code null}. + * @param store the dictionary store queried, by the hash carried in the {@code dcz} + * prefix, for the dictionary required to decode the body; must not be + * {@code null}. + * @since 5.7 + */ + public InflatingDictionaryZstdDataConsumer( + final AsyncDataConsumer downstream, + final CompressionDictionaryStore store) { + this.downstream = Args.notNull(downstream, "Downstream data consumer"); + this.store = Args.notNull(store, "Dictionary store"); + this.header = ByteBuffer.allocate(HEADER_LENGTH); + this.dctx = new ZstdDecompressCtx(); + this.inDirect = ByteBuffer.allocateDirect(IN_BUF); + this.outDirect = ByteBuffer.allocateDirect(OUT_BUF); + this.closed = new AtomicBoolean(false); + + inDirect.limit(0); + outDirect.limit(0); + } + + /** + * Propagates a capacity update to the downstream consumer. The channel is wrapped so that the + * capacity requested reflects compressed input rather than the larger inflated output, keeping + * back-pressure meaningful across the decoding step. + * + * @since 5.7 + */ + @Override + public void updateCapacity(final CapacityChannel capacityChannel) throws IOException { + downstream.updateCapacity(new InflatingCapacityChannel(capacityChannel)); + } + + /** + * Decodes the next chunk of the {@code dcz} body. The framing prefix is accumulated and + * validated across the leading calls; once the dictionary has been resolved and loaded, the + * remaining bytes are inflated and passed downstream. Calls made after the stream has ended or + * its resources have been released are ignored. + * + * @throws IOException if the framing prefix is invalid, the required dictionary is not + * available, or the compressed stream is corrupt. + * @since 5.7 + */ + @Override + public void consume(final ByteBuffer src) throws IOException { + if (closed.get()) { + return; + } + + if (!initialized) { + consumeHeader(src); + if (!initialized) { + return; + } + } + + try { + while (src.hasRemaining()) { + inDirect.compact(); + + final int take = Math.min(inDirect.remaining(), src.remaining()); + final int oldLimit = src.limit(); + src.limit(src.position() + take); + inDirect.put(src); + src.limit(oldLimit); + + inDirect.flip(); + + while (inDirect.hasRemaining()) { + outDirect.compact(); + + frameComplete = + dctx.decompressDirectByteBufferStream(outDirect, inDirect); + + outDirect.flip(); + + if (outDirect.hasRemaining()) { + downstream.consume(outDirect); + if (outDirect.hasRemaining()) { + return; + } + } else if (!inDirect.hasRemaining()) { + break; + } + } + } + } catch (final RuntimeException ex) { + throw new IOException("DCZ stream corrupted", ex); + } + } + + private void consumeHeader(final ByteBuffer src) throws IOException { + final int xfer = Math.min(src.remaining(), header.remaining()); + final int lim = src.limit(); + src.limit(src.position() + xfer); + header.put(src); + src.limit(lim); + + if (header.hasRemaining()) { + return; + } + + header.flip(); + + for (final byte expected : MAGIC) { + if (header.get() != expected) { + throw new IOException("Invalid DCZ stream header"); + } + } + + final byte[] hash = new byte[HASH_LENGTH]; + header.get(hash); + + final CompressionDictionary dictionary = store.getByHash(hash); + if (dictionary == null || !dictionary.matchesHash(hash)) { + throw new IOException("Compression dictionary not available"); + } + + try { + dctx.loadDict(dictionary.getContent()); + initialized = true; + } catch (final RuntimeException ex) { + throw new IOException("Unable to initialize DCZ decoder", ex); + } + } + + /** + * Finalises decoding once the origin has signalled end of stream. The header and the final + * Zstandard frame must both be complete; an incomplete header or a truncated frame is reported + * as an {@link IOException} rather than yielding partial content. On success the decompression + * context is closed and {@code streamEnd} is propagated downstream exactly once. + * + * @throws IOException if the {@code dcz} header or the compressed frame was truncated. + * @since 5.7 + */ + @Override + public void streamEnd(final List trailers) throws HttpException, IOException { + if (!initialized) { + throw new IOException("Truncated DCZ stream header"); + } + + if (!frameComplete) { + throw new IOException("Truncated DCZ stream"); + } + + if (closed.compareAndSet(false, true)) { + dctx.close(); + downstream.streamEnd(trailers); + } + } + + /** + * Releases the native Zstandard decompression context, if it has not already been closed by + * {@link #streamEnd(List)}, and releases the downstream consumer. Safe to call more than once + * and safe to call to abort an in-flight exchange. + * + * @since 5.7 + */ + @Override + public void releaseResources() { + if (closed.compareAndSet(false, true)) { + dctx.close(); + } + downstream.releaseResources(); + } +} \ No newline at end of file diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/BasicCompressionDictionaryStore.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/BasicCompressionDictionaryStore.java new file mode 100644 index 0000000000..f422a6a015 --- /dev/null +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/BasicCompressionDictionaryStore.java @@ -0,0 +1,191 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.entity.compress; + +import java.net.URI; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.hc.core5.annotation.Contract; +import org.apache.hc.core5.annotation.ThreadingBehavior; +import org.apache.hc.core5.util.Args; + +/** + * Basic in-memory implementation of {@link CompressionDictionaryStore}. + *

+ * Dictionaries are held in a bounded map keyed by the SHA-256 hash of their content. The store + * retains at most {@code maxEntries} dictionaries; once that bound is exceeded the oldest entries + * are evicted in insertion order until the store is back within capacity. Adding a dictionary whose + * hash already exists replaces the previous entry and moves it to the most-recently-added position, + * so it is evicted last. All mutating and reading operations are guarded by a lock, hence instances + * are safe for concurrent use. + * + * @since 5.7 + */ +@Contract(threading = ThreadingBehavior.SAFE) +public final class BasicCompressionDictionaryStore implements CompressionDictionaryStore { + + private static final int DEFAULT_MAX_ENTRIES = 64; + + private final int maxEntries; + private final Map dictionaries; + private final ReentrantLock lock; + + public BasicCompressionDictionaryStore() { + this(DEFAULT_MAX_ENTRIES); + } + + public BasicCompressionDictionaryStore(final int maxEntries) { + this.maxEntries = Args.positive(maxEntries, "Maximum entries"); + this.dictionaries = new LinkedHashMap<>(); + this.lock = new ReentrantLock(); + } + + @Override + public void add(final CompressionDictionary dictionary) { + Args.notNull(dictionary, "Dictionary"); + + lock.lock(); + try { + final String key = key( + dictionary.getSource(), + dictionary.getSha256()); + + dictionaries.remove(key); + dictionaries.put(key, dictionary); + + while (dictionaries.size() > maxEntries) { + final Iterator iterator = + dictionaries.keySet().iterator(); + if (iterator.hasNext()) { + iterator.next(); + iterator.remove(); + } + } + } finally { + lock.unlock(); + } + } + + @Override + public CompressionDictionary getByHash( + final URI origin, + final byte[] sha256) { + Args.notNull(origin, "Origin"); + Args.notNull(sha256, "SHA-256"); + + lock.lock(); + try { + return dictionaries.get(key(origin, sha256)); + } finally { + lock.unlock(); + } + } + + @Override + public List getByOrigin(final URI origin) { + Args.notNull(origin, "Origin"); + + lock.lock(); + try { + final List result = + new ArrayList<>(); + + for (final CompressionDictionary dictionary + : dictionaries.values()) { + if (sameOrigin(origin, dictionary.getSource())) { + result.add(dictionary); + } + } + + return result; + } finally { + lock.unlock(); + } + } + + @Override + public void clear() { + lock.lock(); + try { + dictionaries.clear(); + } finally { + lock.unlock(); + } + } + + private static String key( + final URI origin, + final byte[] sha256) { + return originKey(origin) + + '|' + + Base64.getEncoder().encodeToString(sha256); + } + + private static String originKey(final URI uri) { + return uri.getScheme().toLowerCase(Locale.ROOT) + + "://" + + uri.getHost().toLowerCase(Locale.ROOT) + + ':' + + effectivePort(uri); + } + + private static boolean sameOrigin( + final URI first, + final URI second) { + return equalsIgnoreCase(first.getScheme(), second.getScheme()) + && equalsIgnoreCase(first.getHost(), second.getHost()) + && effectivePort(first) == effectivePort(second); + } + + private static boolean equalsIgnoreCase( + final String first, + final String second) { + return first != null + && second != null + && first.equalsIgnoreCase(second); + } + + private static int effectivePort(final URI uri) { + if (uri.getPort() >= 0) { + return uri.getPort(); + } + if ("https".equalsIgnoreCase(uri.getScheme())) { + return 443; + } + if ("http".equalsIgnoreCase(uri.getScheme())) { + return 80; + } + return -1; + } +} \ No newline at end of file diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CompressionDictionary.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CompressionDictionary.java new file mode 100644 index 0000000000..dcbd64c03c --- /dev/null +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CompressionDictionary.java @@ -0,0 +1,143 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.entity.compress; + +import java.net.URI; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.apache.hc.core5.annotation.Contract; +import org.apache.hc.core5.annotation.ThreadingBehavior; +import org.apache.hc.core5.util.Args; + +/** + * An immutable compression dictionary used by Compression Dictionary Transport. + *

+ * A dictionary carries the raw bytes that seed a Dictionary-Compressed Brotli ({@code dcb}) or + * Dictionary-Compressed Zstandard ({@code dcz}) decoder, together with the metadata needed to + * negotiate its use with an origin. The {@code match} pattern advertised in the origin's + * {@code Use-As-Dictionary} response selects the requests a dictionary applies to; the SHA-256 + * hash is offered in the {@code Available-Dictionary} request header and echoed by the origin in + * {@code Dictionary-ID}, so that both peers agree on which dictionary decoded the response. + *

+ * Instances are value objects: the dictionary content is defensively copied on construction and + * on every access, and the SHA-256 hash is computed once from that copy, so the observable state + * never changes after construction. + * + * @since 5.7 + */ +@Contract(threading = ThreadingBehavior.IMMUTABLE) +public final class CompressionDictionary { + + private final byte[] content; + private final byte[] sha256; + private final URI source; + private final String match; + private final List matchDest; + private final String id; + private final String type; + private final Instant storedAt; + private final Instant validUntil; + + public CompressionDictionary( + final byte[] content, + final URI source, + final String match, + final List matchDest, + final String id, + final String type, + final Instant storedAt, + final Instant validUntil) { + this.content = Args.notNull(content, "Content").clone(); + this.sha256 = sha256(content); + this.source = Args.notNull(source, "Source"); + this.match = Args.notBlank(match, "Match"); + this.matchDest = matchDest != null + ? List.copyOf(matchDest) + : Collections.emptyList(); + this.id = id != null ? id : ""; + this.type = type != null ? type : "raw"; + this.storedAt = Args.notNull(storedAt, "Stored at"); + this.validUntil = Args.notNull(validUntil, "Valid until"); + } + + public byte[] getContent() { + return content.clone(); + } + + public byte[] getSha256() { + return sha256.clone(); + } + + public URI getSource() { + return source; + } + + public String getMatch() { + return match; + } + + public List getMatchDest() { + return matchDest; + } + + public String getId() { + return id; + } + + public String getType() { + return type; + } + + public Instant getStoredAt() { + return storedAt; + } + + public Instant getValidUntil() { + return validUntil; + } + + public boolean isFresh(final Instant now) { + return now.isBefore(validUntil); + } + + public boolean matchesHash(final byte[] hash) { + return MessageDigest.isEqual(sha256, hash); + } + + private static byte[] sha256(final byte[] content) { + try { + return MessageDigest.getInstance("SHA-256").digest(content); + } catch (final NoSuchAlgorithmException ex) { + throw new IllegalStateException(ex); + } + } +} \ No newline at end of file diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CompressionDictionaryStore.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CompressionDictionaryStore.java new file mode 100644 index 0000000000..13e7333191 --- /dev/null +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CompressionDictionaryStore.java @@ -0,0 +1,59 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.entity.compress; + +import java.net.URI; +import java.util.List; + +/** + * Repository of {@link CompressionDictionary} instances available to the client for + * decoding dictionary-compressed Brotli ({@code dcb}) and dictionary-compressed + * Zstandard ({@code dcz}) responses under Compression Dictionary Transport. + *

+ * Dictionaries are indexed both by their SHA-256 hash, which the origin echoes back in + * the {@code Dictionary-ID} header to identify the dictionary a response was encoded + * against, and by request origin, so that the client can select the candidates whose + * match pattern applies to an outbound request and advertise them through the + * {@code Available-Dictionary} header. + *

+ * Implementations are expected to be thread-safe, since a store is shared across + * concurrent exchanges. The store keeps entries as supplied; enforcing freshness against + * {@link CompressionDictionary#getValidUntil()} is the caller's responsibility. + * + * @since 5.7 + */ + +public interface CompressionDictionaryStore { + + void add(CompressionDictionary dictionary); + + CompressionDictionary getByHash(URI origin, byte[] sha256); + + List getByOrigin(URI origin); + + void clear(); +} \ No newline at end of file diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/ContentCoding.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/ContentCoding.java index 3a8f169777..f0fc3bbab2 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/ContentCoding.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/ContentCoding.java @@ -66,6 +66,14 @@ public enum ContentCoding { * Zstandard compression format. */ ZSTD("zstd"), + /** + * Dictionary-Compressed Brotli format. + */ + DCB("dcb"), + /** + * Dictionary-Compressed Zstandard format. + */ + DCZ("dcz"), /** * XZ compression format. */ @@ -131,4 +139,4 @@ public String token() { public static ContentCoding fromToken(final String token) { return token != null ? TOKEN_LOOKUP.get(token.toLowerCase(Locale.ROOT)) : null; } -} +} \ No newline at end of file diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryFreshness.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryFreshness.java new file mode 100644 index 0000000000..547362a3a1 --- /dev/null +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryFreshness.java @@ -0,0 +1,158 @@ +package org.apache.hc.client5.http.impl.async; + +import java.time.Duration; +import java.time.Instant; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; + +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpHeaders; +import org.apache.hc.core5.http.HttpResponse; + +final class CompressionDictionaryFreshness { + + private CompressionDictionaryFreshness() { + } + + static Instant determineValidUntil( + final HttpResponse response, + final Instant requestTime, + final Instant responseTime) { + + final CacheControl cacheControl = + parseCacheControl( + response.getFirstHeader(HttpHeaders.CACHE_CONTROL)); + + if (cacheControl.noStore || cacheControl.noCache) { + return null; + } + + final long freshnessLifetime; + + if (cacheControl.maxAge >= 0) { + freshnessLifetime = cacheControl.maxAge; + } else { + final Instant expires = + parseHttpDate(response.getFirstHeader(HttpHeaders.EXPIRES)); + final Instant date = + parseHttpDate(response.getFirstHeader(HttpHeaders.DATE)); + + if (expires == null || date == null) { + return null; + } + + freshnessLifetime = + Math.max(0, Duration.between(date, expires).getSeconds()); + } + + final Instant date = + parseHttpDate(response.getFirstHeader(HttpHeaders.DATE)); + + final long apparentAge = + date != null + ? Math.max( + 0, + Duration.between(date, responseTime) + .getSeconds()) + : 0; + + final long ageValue = + parseAge(response.getFirstHeader(HttpHeaders.AGE)); + + final long responseDelay = + Math.max( + 0, + Duration.between(requestTime, responseTime) + .getSeconds()); + + final long correctedAgeValue = + saturatedAdd(ageValue, responseDelay); + + final long correctedInitialAge = + Math.max(apparentAge, correctedAgeValue); + + final long remaining = + freshnessLifetime - correctedInitialAge; + + if (remaining <= 0) { + return null; + } + + return responseTime.plusSeconds(remaining); + } + + private static long parseAge(final Header header) { + if (header == null) { + return 0; + } + + try { + return Math.max(0, Long.parseLong(header.getValue().trim())); + } catch (final NumberFormatException ex) { + return 0; + } + } + + private static Instant parseHttpDate(final Header header) { + if (header == null) { + return null; + } + + try { + return ZonedDateTime.parse( + header.getValue(), + DateTimeFormatter.RFC_1123_DATE_TIME) + .toInstant(); + } catch (final RuntimeException ex) { + return null; + } + } + + private static CacheControl parseCacheControl(final Header header) { + final CacheControl result = new CacheControl(); + + if (header == null) { + return result; + } + + for (final String element : header.getValue().split(",")) { + final String directive = element.trim(); + + if ("no-store".equalsIgnoreCase(directive)) { + result.noStore = true; + } else if ("no-cache".equalsIgnoreCase(directive)) { + result.noCache = true; + } else if (directive.regionMatches( + true, 0, "max-age=", 0, 8)) { + try { + result.maxAge = + Math.max( + 0, + Long.parseLong( + directive.substring(8) + .trim())); + } catch (final NumberFormatException ignore) { + result.maxAge = -1; + } + } + } + + return result; + } + + private static long saturatedAdd( + final long first, + final long second) { + if (Long.MAX_VALUE - first < second) { + return Long.MAX_VALUE; + } + return first + second; + } + + private static final class CacheControl { + + private boolean noStore; + private boolean noCache; + private long maxAge = -1; + } +} \ No newline at end of file diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryHeaderSupport.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryHeaderSupport.java new file mode 100644 index 0000000000..7eb4a20378 --- /dev/null +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryHeaderSupport.java @@ -0,0 +1,76 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.impl.async; + +import java.util.Base64; + +import org.apache.hc.core5.util.Args; + +/** + * Encodes the request headers used to negotiate a + * {@link org.apache.hc.client5.http.entity.compress.CompressionDictionary} for Compression + * Dictionary Transport. The header values are HTTP Structured Field + * items: {@code Available-Dictionary} carries the dictionary hash as a Byte Sequence and + * {@code Dictionary-ID} carries the server-supplied identifier as a String. + *

+ * This class is a stateless collection of static helpers and holds no mutable state; it is therefore + * safe for concurrent use. It cannot be instantiated. + */ +final class CompressionDictionaryHeaderSupport { + + static final String USE_AS_DICTIONARY = "Use-As-Dictionary"; + static final String AVAILABLE_DICTIONARY = "Available-Dictionary"; + static final String DICTIONARY_ID = "Dictionary-ID"; + + private CompressionDictionaryHeaderSupport() { + } + + static String formatHash(final byte[] hash) { + return ":" + Base64.getEncoder().encodeToString(hash) + ":"; + } + + static String formatString(final String value) { + final StringBuilder buffer = + new StringBuilder(value.length() + 2); + + buffer.append('"'); + + for (int i = 0; i < value.length(); i++) { + final char ch = value.charAt(i); + + if (ch == '"' || ch == '\\') { + buffer.append('\\'); + } + + buffer.append(ch); + } + + buffer.append('"'); + + return buffer.toString(); + } +} \ No newline at end of file diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryMatcher.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryMatcher.java new file mode 100644 index 0000000000..973ffc56d7 --- /dev/null +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryMatcher.java @@ -0,0 +1,54 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.impl.async; + +import java.net.URI; +import java.util.Collection; + +import org.apache.hc.client5.http.entity.compress.CompressionDictionary; + +/** + * Selects the compression dictionary to advertise for an outgoing request, following the + * Compression Dictionary Transport matching rules. Implementations decide which stored {@link CompressionDictionary}, + * if any, is eligible for a given request URI; the winner is later announced to the origin through + * the {@code Available-Dictionary} header so it may return a {@code dcb} or {@code dcz} response. + *

+ * Eligibility is governed by the {@code Use-As-Dictionary} metadata carried on each dictionary: + * a candidate applies only when it is still fresh, shares the request's origin, and its stored + * URL pattern matches the request path. When several candidates qualify the implementation is + * expected to return the single most specific one. + *

+ * Implementations are expected to be stateless and safe for concurrent use. + */ + +interface CompressionDictionaryMatcher { + + CompressionDictionary match( + URI requestUri, + String requestDestination, + Collection dictionaries); +} \ No newline at end of file diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryUrlPatternMatcher.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryUrlPatternMatcher.java new file mode 100644 index 0000000000..33f4013c32 --- /dev/null +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryUrlPatternMatcher.java @@ -0,0 +1,15 @@ +package org.apache.hc.client5.http.impl.async; + +import java.net.URI; + +interface CompressionDictionaryUrlPatternMatcher { + + boolean isValid( + String pattern, + URI dictionaryUri); + + boolean matches( + String pattern, + URI dictionaryUri, + URI requestUri); +} \ No newline at end of file diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/ContentCompressionAsyncExec.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/ContentCompressionAsyncExec.java index cf29083429..db3c3e7ca4 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/ContentCompressionAsyncExec.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/ContentCompressionAsyncExec.java @@ -27,10 +27,15 @@ package org.apache.hc.client5.http.impl.async; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.time.DateTimeException; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Set; import java.util.function.UnaryOperator; @@ -39,8 +44,12 @@ import org.apache.hc.client5.http.async.AsyncExecChainHandler; import org.apache.hc.client5.http.async.methods.InflatingAsyncDataConsumer; import org.apache.hc.client5.http.async.methods.InflatingBrotliDataConsumer; +import org.apache.hc.client5.http.async.methods.InflatingDictionaryBrotliDataConsumer; +import org.apache.hc.client5.http.async.methods.InflatingDictionaryZstdDataConsumer; import org.apache.hc.client5.http.async.methods.InflatingGzipDataConsumer; import org.apache.hc.client5.http.async.methods.InflatingZstdDataConsumer; +import org.apache.hc.client5.http.entity.compress.CompressionDictionary; +import org.apache.hc.client5.http.entity.compress.CompressionDictionaryStore; import org.apache.hc.client5.http.entity.compress.ContentCoding; import org.apache.hc.client5.http.impl.Brotli4jRuntime; import org.apache.hc.client5.http.impl.ContentCodingSupport; @@ -50,10 +59,12 @@ import org.apache.hc.core5.annotation.Internal; import org.apache.hc.core5.annotation.ThreadingBehavior; import org.apache.hc.core5.http.EntityDetails; +import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpException; import org.apache.hc.core5.http.HttpHeaders; import org.apache.hc.core5.http.HttpRequest; import org.apache.hc.core5.http.HttpResponse; +import org.apache.hc.core5.http.ParseException; import org.apache.hc.core5.http.config.Lookup; import org.apache.hc.core5.http.config.RegistryBuilder; import org.apache.hc.core5.http.message.MessageSupport; @@ -65,20 +76,71 @@ @Internal public final class ContentCompressionAsyncExec implements AsyncExecChainHandler { + private static final int DEFAULT_MAX_DICTIONARY_SIZE = 16 * 1024 * 1024; + private final Lookup> decoders; private final List acceptTokens; + private final List dictionaryAcceptTokens; private final int maxCodecListLen; + private final CompressionDictionaryStore compressionDictionaryStore; + private final CompressionDictionaryMatcher compressionDictionaryMatcher; public ContentCompressionAsyncExec( final LinkedHashMap> decoderMap, - final int maxCodecListLen) { + final int maxCodecListLen, + final CompressionDictionaryStore compressionDictionaryStore) { Args.notEmpty(decoderMap, "Decoder map"); final RegistryBuilder> rb = RegistryBuilder.create(); decoderMap.forEach(rb::register); + + final List tokens = new ArrayList<>(); + decoderMap.keySet().forEach(token -> { + if (!ContentCoding.DCB.token().equalsIgnoreCase(token) + && !ContentCoding.DCZ.token().equalsIgnoreCase(token)) { + tokens.add(token); + } + }); + + final List dictionaryTokens = new ArrayList<>(); + if (compressionDictionaryStore != null) { + if (decoderMap.containsKey(ContentCoding.DCB.token())) { + dictionaryTokens.add(ContentCoding.DCB.token()); + } else if (Brotli4jRuntime.available()) { + rb.register(ContentCoding.DCB.token(), + d -> new InflatingDictionaryBrotliDataConsumer(d, compressionDictionaryStore)); + dictionaryTokens.add(ContentCoding.DCB.token()); + } + + if (decoderMap.containsKey(ContentCoding.DCZ.token())) { + dictionaryTokens.add(ContentCoding.DCZ.token()); + } else if (ZstdRuntime.available()) { + rb.register(ContentCoding.DCZ.token(), + d -> new InflatingDictionaryZstdDataConsumer(d, compressionDictionaryStore)); + dictionaryTokens.add(ContentCoding.DCZ.token()); + } + } + this.decoders = rb.build(); - this.acceptTokens = new ArrayList<>(decoderMap.keySet()); + this.acceptTokens = tokens; + this.dictionaryAcceptTokens = dictionaryTokens; this.maxCodecListLen = maxCodecListLen; + this.compressionDictionaryStore = compressionDictionaryStore; + this.compressionDictionaryMatcher = compressionDictionaryStore != null + ? new DefaultCompressionDictionaryMatcher() + : null; + } + + public ContentCompressionAsyncExec( + final LinkedHashMap> decoderMap, + final int maxCodecListLen) { + this(decoderMap, maxCodecListLen, null); + } + + public ContentCompressionAsyncExec( + final LinkedHashMap> decoderMap, + final CompressionDictionaryStore compressionDictionaryStore) { + this(decoderMap, ContentCodingSupport.MAX_CODEC_LIST_LEN, compressionDictionaryStore); } public ContentCompressionAsyncExec( @@ -89,7 +151,9 @@ public ContentCompressionAsyncExec( /** * Default: DEFLATE + GZIP (plus x-gzip alias). */ - public ContentCompressionAsyncExec(final int maxCodecListLen) { + public ContentCompressionAsyncExec( + final int maxCodecListLen, + final CompressionDictionaryStore compressionDictionaryStore) { final LinkedHashMap> map = new LinkedHashMap<>(); map.put(ContentCoding.DEFLATE.token(), d -> new InflatingAsyncDataConsumer(d, null)); map.put(ContentCoding.GZIP.token(), InflatingGzipDataConsumer::new); @@ -113,9 +177,38 @@ public ContentCompressionAsyncExec(final int maxCodecListLen) { tokens.add(ContentCoding.BROTLI.token()); } + final List dictionaryTokens = new ArrayList<>(); + if (compressionDictionaryStore != null) { + if (Brotli4jRuntime.available()) { + rb.register(ContentCoding.DCB.token(), + d -> new InflatingDictionaryBrotliDataConsumer(d, compressionDictionaryStore)); + dictionaryTokens.add(ContentCoding.DCB.token()); + } + + if (ZstdRuntime.available()) { + rb.register(ContentCoding.DCZ.token(), + d -> new InflatingDictionaryZstdDataConsumer(d, compressionDictionaryStore)); + dictionaryTokens.add(ContentCoding.DCZ.token()); + } + } + this.decoders = rb.build(); this.acceptTokens = tokens; + this.dictionaryAcceptTokens = dictionaryTokens; this.maxCodecListLen = maxCodecListLen; + this.compressionDictionaryStore = compressionDictionaryStore; + this.compressionDictionaryMatcher = compressionDictionaryStore != null + ? new DefaultCompressionDictionaryMatcher() + : null; + } + + public ContentCompressionAsyncExec(final int maxCodecListLen) { + this(maxCodecListLen, null); + } + + public ContentCompressionAsyncExec( + final CompressionDictionaryStore compressionDictionaryStore) { + this(ContentCodingSupport.MAX_CODEC_LIST_LEN, compressionDictionaryStore); } public ContentCompressionAsyncExec() { @@ -132,9 +225,34 @@ public void execute( final HttpClientContext ctx = scope != null ? scope.clientContext : HttpClientContext.create(); final boolean enabled = ctx.getRequestConfigOrDefault().isContentCompressionEnabled(); + final URI requestUri = resolveRequestUri(request, scope); + + final CompressionDictionary dictionary = enabled + ? findDictionary(request, requestUri) + : null; + + if (dictionary != null) { + request.addHeader( + CompressionDictionaryHeaderSupport.AVAILABLE_DICTIONARY, + CompressionDictionaryHeaderSupport.formatAvailableDictionary(dictionary.getSha256())); + + if (!dictionary.getId().isEmpty()) { + request.addHeader( + CompressionDictionaryHeaderSupport.DICTIONARY_ID, + CompressionDictionaryHeaderSupport.formatDictionaryId(dictionary.getId())); + } + } if (enabled && !request.containsHeader(HttpHeaders.ACCEPT_ENCODING)) { - request.addHeader(MessageSupport.headerOfTokens(HttpHeaders.ACCEPT_ENCODING, acceptTokens)); + if (dictionary != null && !dictionaryAcceptTokens.isEmpty()) { + final List tokens = new ArrayList<>( + acceptTokens.size() + dictionaryAcceptTokens.size()); + tokens.addAll(acceptTokens); + tokens.addAll(dictionaryAcceptTokens); + request.addHeader(MessageSupport.headerOfTokens(HttpHeaders.ACCEPT_ENCODING, tokens)); + } else { + request.addHeader(MessageSupport.headerOfTokens(HttpHeaders.ACCEPT_ENCODING, acceptTokens)); + } } chain.proceed(request, producer, scope, new AsyncExecCallback() { @@ -148,6 +266,17 @@ public AsyncDataConsumer handleResponse(final HttpResponse rsp, return cb.handleResponse(rsp, details); } + final UseAsDictionary useAsDictionary = + parseUseAsDictionary(rsp, requestUri); + + final Instant storedAt = useAsDictionary != null + ? Instant.now() + : null; + + final Instant validUntil = storedAt != null + ? getValidUntil(rsp, storedAt) + : null; + final List codecs = ContentCodingSupport.parseContentCodecs(details); ContentCodingSupport.validate(codecs, maxCodecListLen); if (!codecs.isEmpty()) { @@ -155,8 +284,28 @@ public AsyncDataConsumer handleResponse(final HttpResponse rsp, if (downstream == null) { return null; } + + if (useAsDictionary != null && validUntil != null) { + downstream = new DictionaryCapturingAsyncDataConsumer( + downstream, + compressionDictionaryStore, + requestUri, + useAsDictionary, + storedAt, + validUntil, + DEFAULT_MAX_DICTIONARY_SIZE); + } + for (int i = codecs.size() - 1; i >= 0; i--) { final String codec = codecs.get(i); + + if ((ContentCoding.DCB.token().equalsIgnoreCase(codec) + || ContentCoding.DCZ.token().equalsIgnoreCase(codec)) + && dictionary == null) { + throw new HttpException( + "Dictionary Content-Encoding without negotiated dictionary: " + codec); + } + final UnaryOperator op = decoders.lookup(codec); if (op != null) { downstream = op.apply(downstream); @@ -167,7 +316,18 @@ public AsyncDataConsumer handleResponse(final HttpResponse rsp, return downstream; } - return cb.handleResponse(rsp, details); + AsyncDataConsumer downstream = cb.handleResponse(rsp, details); + if (downstream != null && useAsDictionary != null && validUntil != null) { + downstream = new DictionaryCapturingAsyncDataConsumer( + downstream, + compressionDictionaryStore, + requestUri, + useAsDictionary, + storedAt, + validUntil, + DEFAULT_MAX_DICTIONARY_SIZE); + } + return downstream; } @Override @@ -188,6 +348,135 @@ public void failed(final Exception ex) { }); } + private CompressionDictionary findDictionary( + final HttpRequest request, + final URI requestUri) { + if (compressionDictionaryStore == null + || compressionDictionaryMatcher == null + || requestUri == null + || !"https".equalsIgnoreCase(requestUri.getScheme()) + || request.containsHeader(CompressionDictionaryHeaderSupport.AVAILABLE_DICTIONARY) + || request.containsHeader(CompressionDictionaryHeaderSupport.DICTIONARY_ID)) { + return null; + } + + return compressionDictionaryMatcher.match( + requestUri, + compressionDictionaryStore.getByOrigin(requestUri)); + } + + private UseAsDictionary parseUseAsDictionary( + final HttpResponse response, + final URI requestUri) { + if (compressionDictionaryStore == null + || requestUri == null + || !"https".equalsIgnoreCase(requestUri.getScheme())) { + return null; + } + + final Header header = + response.getFirstHeader(CompressionDictionaryHeaderSupport.USE_AS_DICTIONARY); + if (header == null) { + return null; + } + + try { + final UseAsDictionary useAsDictionary = UseAsDictionary.parse(header.getValue()); + return useAsDictionary.isSupported() ? useAsDictionary : null; + } catch (final ParseException ex) { + return null; + } + } + + private static Instant getValidUntil( + final HttpResponse response, + final Instant storedAt) { + long maxAge = -1; + + for (final Header header : response.getHeaders(HttpHeaders.CACHE_CONTROL)) { + final String[] directives = header.getValue().split(","); + for (final String value : directives) { + final String directive = value.trim(); + final String lowerCase = directive.toLowerCase(Locale.ROOT); + + if ("no-store".equals(lowerCase) || "no-cache".equals(lowerCase)) { + return null; + } + + if (lowerCase.startsWith("max-age=")) { + final String maxAgeValue = unquote( + directive.substring(directive.indexOf('=') + 1).trim()); + try { + maxAge = Long.parseLong(maxAgeValue); + } catch (final NumberFormatException ex) { + return null; + } + } + } + } + + if (maxAge <= 0) { + return null; + } + + long age = 0; + final Header ageHeader = response.getFirstHeader("Age"); + if (ageHeader != null) { + try { + age = Long.parseLong(ageHeader.getValue()); + } catch (final NumberFormatException ex) { + return null; + } + } + + final long remaining = maxAge - Math.max(0, age); + if (remaining <= 0) { + return null; + } + + try { + return storedAt.plusSeconds(remaining); + } catch (final DateTimeException | ArithmeticException ex) { + return null; + } + } + + private static String unquote(final String value) { + if (value.length() > 1 + && value.charAt(0) == '"' + && value.charAt(value.length() - 1) == '"') { + return value.substring(1, value.length() - 1); + } + return value; + } + + private static URI resolveRequestUri( + final HttpRequest request, + final AsyncExecChain.Scope scope) { + try { + if (scope != null) { + final URI originalUri = scope.originalRequest.getUri(); + if (originalUri.isAbsolute()) { + return originalUri; + } + } + + final URI requestUri = request.getUri(); + if (requestUri.isAbsolute()) { + return requestUri; + } + + if (scope != null) { + final URI baseUri = URI.create(scope.route.getTargetHost().toURI() + "/"); + return baseUri.resolve(requestUri); + } + + return null; + } catch (final URISyntaxException | IllegalArgumentException ex) { + return null; + } + } + private static EntityDetails wrapEntityDetails(final EntityDetails original) { return new EntityDetails() { @Override @@ -216,4 +505,4 @@ public Set getTrailerNames() { } }; } -} +} \ No newline at end of file diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/DefaultCompressionDictionaryMatcher.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/DefaultCompressionDictionaryMatcher.java new file mode 100644 index 0000000000..0e3ba8e2e5 --- /dev/null +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/DefaultCompressionDictionaryMatcher.java @@ -0,0 +1,175 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.impl.async; + +import java.net.URI; +import java.time.Clock; +import java.time.Instant; +import java.util.Collection; +import java.util.Comparator; + +import org.apache.hc.client5.http.entity.compress.CompressionDictionary; +import org.apache.hc.core5.util.Args; + +/** + * Default {@link CompressionDictionaryMatcher} that selects the compression dictionary + * to advertise for an outgoing request, following the matching and selection rules of + * Compression Dictionary Transport. + *

+ * A candidate is eligible only when it is still fresh at the current instant, shares the + * same origin as the request and its stored URL pattern matches the request path. Dictionary + * transport is restricted to secure origins, so a request whose scheme is not {@code https} + * never matches, and a candidate whose source is not an {@code https} URL is ignored. + *

+ * When several candidates are eligible the most specific one wins: the longest pattern is + * preferred, and ties are broken in favour of the most recently stored dictionary. The + * winning dictionary is the value the caller offers to the origin through the + * {@code Available-Dictionary} header. + *

+ * Instances are immutable and stateless apart from the injected {@link Clock}, hence thread + * safe and reusable across requests. The clock constructor exists for deterministic testing; + * production code uses the system UTC clock. + */ +final class DefaultCompressionDictionaryMatcher + implements CompressionDictionaryMatcher { + + private final Clock clock; + private final CompressionDictionaryUrlPatternMatcher urlPatternMatcher; + + DefaultCompressionDictionaryMatcher( + final CompressionDictionaryUrlPatternMatcher urlPatternMatcher) { + this(Clock.systemUTC(), urlPatternMatcher); + } + + DefaultCompressionDictionaryMatcher( + final Clock clock, + final CompressionDictionaryUrlPatternMatcher urlPatternMatcher) { + this.clock = Args.notNull(clock, "Clock"); + this.urlPatternMatcher = + Args.notNull(urlPatternMatcher, "URL pattern matcher"); + } + + @Override + public CompressionDictionary match( + final URI requestUri, + final String requestDestination, + final Collection dictionaries) { + + if (requestUri == null + || !"https".equalsIgnoreCase(requestUri.getScheme()) + || dictionaries == null + || dictionaries.isEmpty()) { + return null; + } + + final Instant now = clock.instant(); + + return dictionaries.stream() + .filter(dictionary -> dictionary.isFresh(now)) + .filter(dictionary -> "raw".equalsIgnoreCase( + dictionary.getType())) + .filter(dictionary -> sameOrigin( + dictionary.getSource(), requestUri)) + .filter(dictionary -> destinationMatches( + dictionary, requestDestination)) + .filter(dictionary -> urlPatternMatcher.matches( + dictionary.getMatch(), + dictionary.getSource(), + requestUri)) + .max( + Comparator + .comparingInt( + (CompressionDictionary dictionary) -> + destinationPrecedence( + dictionary, + requestDestination)) + .thenComparingInt( + dictionary -> + dictionary.getMatch().length()) + .thenComparing( + CompressionDictionary::getStoredAt)) + .orElse(null); + } + + private static int destinationPrecedence( + final CompressionDictionary dictionary, + final String requestDestination) { + + if (requestDestination == null + || dictionary.getMatchDest().isEmpty()) { + return 0; + } + + return dictionary.getMatchDest().contains(requestDestination) + ? 1 + : 0; + } + + private static boolean destinationMatches( + final CompressionDictionary dictionary, + final String requestDestination) { + + if (requestDestination == null) { + /* + * RFC 9842: clients that do not support request destinations + * MUST treat match-dest as an empty list. + */ + return true; + } + + return dictionary.getMatchDest().isEmpty() + || dictionary.getMatchDest().contains(requestDestination); + } + + private static boolean sameOrigin( + final URI first, + final URI second) { + + return equalsIgnoreCase(first.getScheme(), second.getScheme()) + && equalsIgnoreCase(first.getHost(), second.getHost()) + && effectivePort(first) == effectivePort(second); + } + + private static boolean equalsIgnoreCase( + final String first, + final String second) { + return first != null + && second != null + && first.equalsIgnoreCase(second); + } + + private static int effectivePort(final URI uri) { + if (uri.getPort() >= 0) { + return uri.getPort(); + } + return "https".equalsIgnoreCase(uri.getScheme()) + ? 443 + : "http".equalsIgnoreCase(uri.getScheme()) + ? 80 + : -1; + } +} \ No newline at end of file diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/DictionaryCapturingAsyncDataConsumer.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/DictionaryCapturingAsyncDataConsumer.java new file mode 100644 index 0000000000..5bace0cdd1 --- /dev/null +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/DictionaryCapturingAsyncDataConsumer.java @@ -0,0 +1,171 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.impl.async; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URI; +import java.nio.ByteBuffer; +import java.time.Instant; +import java.util.List; + +import org.apache.hc.client5.http.entity.compress.CompressionDictionary; +import org.apache.hc.client5.http.entity.compress.CompressionDictionaryStore; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpException; +import org.apache.hc.core5.http.nio.AsyncDataConsumer; +import org.apache.hc.core5.http.nio.CapacityChannel; +import org.apache.hc.core5.util.Args; + +/** + * {@link AsyncDataConsumer} decorator that captures a response body so it can be + * registered as a compression dictionary while the body is delivered to the wrapped + * consumer unchanged. + *

+ * When the origin marks a response with a {@code Use-As-Dictionary} directive, that + * response may be reused as a shared dictionary to decode future Dictionary-Compressed + * Brotli ({@code dcb}) or Dictionary-Compressed Zstandard ({@code dcz}) responses, as + * defined by Compression Dictionary Transport. This decorator sits between the message pipeline and the actual + * body consumer: every chunk is forwarded downstream first, then a copy of the bytes the + * downstream consumer accepted is accumulated into an in-memory buffer. On stream end the + * buffered bytes are handed to the {@link CompressionDictionaryStore} as a new + * {@link CompressionDictionary}, keyed by the request {@link URI} and the match pattern + * and identifier carried by the directive. + *

+ * Capture is best-effort and bounded. If the accumulated body would exceed {@code maxSize}, + * or if the exchange is aborted before it completes, the buffer is discarded and no + * dictionary is stored; the downstream consumer is unaffected in either case. A dictionary + * is only stored when the directive is {@link UseAsDictionary#isSupported() supported}. + *

+ * Instances are not thread-safe and, like any {@link AsyncDataConsumer}, expect their + * callbacks to be invoked by a single I/O thread for the lifetime of one message exchange. + */ +final class DictionaryCapturingAsyncDataConsumer implements AsyncDataConsumer { + + private final AsyncDataConsumer downstream; + private final CompressionDictionaryStore store; + private final URI source; + private final UseAsDictionary directive; + private final Instant storedAt; + private final Instant validUntil; + private final int maxSize; + private final ByteArrayOutputStream buffer; + + private boolean discarded; + + DictionaryCapturingAsyncDataConsumer( + final AsyncDataConsumer downstream, + final CompressionDictionaryStore store, + final URI source, + final UseAsDictionary directive, + final Instant storedAt, + final Instant validUntil, + final int maxSize) { + this.downstream = Args.notNull(downstream, "Downstream data consumer"); + this.store = Args.notNull(store, "Dictionary store"); + this.source = Args.notNull(source, "Source"); + this.directive = Args.notNull(directive, "Directive"); + this.storedAt = Args.notNull(storedAt, "Stored at"); + this.validUntil = Args.notNull(validUntil, "Valid until"); + this.maxSize = Args.positive(maxSize, "Maximum dictionary size"); + this.buffer = new ByteArrayOutputStream(Math.min(maxSize, 8192)); + } + + @Override + public void updateCapacity(final CapacityChannel capacityChannel) throws IOException { + downstream.updateCapacity(capacityChannel); + } + + /** + * Forwards the chunk downstream and, unless capture has been discarded, appends a copy + * of the bytes the downstream consumer actually accepted to the capture buffer. Only the + * range from the entry position to the position left by {@code downstream} is captured, + * so partially consumed buffers are handled correctly. Capture is abandoned for the rest + * of the exchange once the buffered size would exceed {@code maxSize}. + */ + @Override + public void consume(final ByteBuffer src) throws IOException { + final int start = src.position(); + + downstream.consume(src); + + if (!discarded) { + final int consumed = src.position() - start; + if (consumed > 0) { + if (buffer.size() + consumed > maxSize) { + discarded = true; + buffer.reset(); + return; + } + + final ByteBuffer copy = src.duplicate(); + copy.position(start); + copy.limit(start + consumed); + + final byte[] bytes = new byte[consumed]; + copy.get(bytes); + buffer.write(bytes, 0, bytes.length); + } + } + } + + /** + * Registers the captured body as a compression dictionary when capture completed intact + * and the directive is supported, then signals stream end to the downstream consumer. + * The downstream consumer is always notified, whether or not a dictionary was stored. + */ + @Override + public void streamEnd( + final List trailers) + throws HttpException, IOException { + + if (!discarded && directive.isSupported()) { + store.add(new CompressionDictionary( + buffer.toByteArray(), + source, + directive.getMatch(), + directive.getMatchDest(), + directive.getId(), + directive.getType(), + storedAt, + validUntil)); + } + + downstream.streamEnd(trailers); + } + + /** + * Discards any captured bytes and releases the downstream consumer. Marking capture as + * discarded ensures no dictionary is stored for an exchange torn down before completion. + */ + @Override + public void releaseResources() { + discarded = true; + buffer.reset(); + downstream.releaseResources(); + } +} \ No newline at end of file diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/HttpAsyncClientBuilder.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/HttpAsyncClientBuilder.java index d010ac8617..6054e2d449 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/HttpAsyncClientBuilder.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/HttpAsyncClientBuilder.java @@ -53,6 +53,7 @@ import org.apache.hc.client5.http.cookie.BasicCookieStore; import org.apache.hc.client5.http.cookie.CookieSpecFactory; import org.apache.hc.client5.http.cookie.CookieStore; +import org.apache.hc.client5.http.entity.compress.CompressionDictionaryStore; import org.apache.hc.client5.http.impl.ChainElement; import org.apache.hc.client5.http.impl.CookieSpecSupport; import org.apache.hc.client5.http.impl.DefaultAuthenticationStrategy; @@ -262,6 +263,8 @@ private ExecInterceptorEntry( */ private LinkedHashMap> contentDecoderMap; + private CompressionDictionaryStore compressionDictionaryStore; + /** * When {@code true} the client skips all transparent response decompression. */ @@ -883,6 +886,25 @@ public HttpAsyncClientBuilder setContentDecoderMap( return this; } + /** + * Sets the compression dictionary store used for Compression Dictionary + * Transport as defined by RFC 9842. + *

+ * When configured, matching dictionaries can be advertised and used for + * {@code dcb} and {@code dcz} response decompression. + *

+ * + * @param compressionDictionaryStore the compression dictionary store, + * or {@code null} to disable dictionary transport + * @return {@code this} builder instance + * @since 5.7 + */ + public final HttpAsyncClientBuilder setCompressionDictionaryStore( + final CompressionDictionaryStore compressionDictionaryStore) { + this.compressionDictionaryStore = compressionDictionaryStore; + return this; + } + /** * Disables transparent response decompression for the client produced by * this builder. @@ -1117,11 +1139,11 @@ public CloseableHttpAsyncClient build() { if (!contentCompressionDisabled) { if (contentDecoderMap != null && !contentDecoderMap.isEmpty()) { execChainDefinition.addFirst( - new ContentCompressionAsyncExec(contentDecoderMap), + new ContentCompressionAsyncExec(contentDecoderMap, compressionDictionaryStore), ChainElement.COMPRESS.name()); } else { execChainDefinition.addFirst( - new ContentCompressionAsyncExec(), + new ContentCompressionAsyncExec(compressionDictionaryStore), ChainElement.COMPRESS.name()); } } diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/UseAsDictionary.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/UseAsDictionary.java new file mode 100644 index 0000000000..cc810380ef --- /dev/null +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/UseAsDictionary.java @@ -0,0 +1,91 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.impl.async; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; + +import org.apache.hc.core5.http.ParseException; +import org.apache.hc.core5.util.Args; + +/** + * Parsed representation of the {@code Use-As-Dictionary} response header defined by Compression + * Dictionary Transport. The origin returns this header alongside a resource it + * offers as a future compression dictionary; it is a Structured Field Dictionary whose members + * describe how the resource may later be matched and referenced. This type retains the members + * the client acts on: {@code match} (a URL Pattern, mandatory) and {@code id} (an optional + * opaque identifier echoed back in the {@code Dictionary-ID} request header), together with the + * {@code type} that names the dictionary format. + *

+ * Only the {@code raw} dictionary type is recognised; other values leave {@link #isSupported()} + * {@code false} so the caller can ignore an offer it cannot honour rather than fail. Instances + * are immutable and therefore safe to share between threads. + */ +final class UseAsDictionary { + + static final String RAW = "raw"; + + private final String match; + private final List matchDest; + private final String id; + private final String type; + + UseAsDictionary( + final String match, + final List matchDest, + final String id, + final String type) { + this.match = match; + this.matchDest = matchDest != null + ? Collections.unmodifiableList(new ArrayList<>(matchDest)) + : Collections.emptyList(); + this.id = id != null ? id : ""; + this.type = type != null ? type : RAW; + } + + String getMatch() { + return match; + } + + List getMatchDest() { + return matchDest; + } + + String getId() { + return id; + } + + String getType() { + return type; + } + + boolean isSupported() { + return RAW.equalsIgnoreCase(type); + } +} \ No newline at end of file diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/examples/AsyncClientCompressionDictionary.java b/httpclient5/src/test/java/org/apache/hc/client5/http/examples/AsyncClientCompressionDictionary.java new file mode 100644 index 0000000000..d92dba3624 --- /dev/null +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/examples/AsyncClientCompressionDictionary.java @@ -0,0 +1,218 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.examples; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Future; + +import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; +import org.apache.hc.client5.http.async.methods.SimpleRequestBuilder; +import org.apache.hc.client5.http.entity.compress.BasicCompressionDictionaryStore; +import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; +import org.apache.hc.client5.http.impl.async.HttpAsyncClients; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpHeaders; +import org.apache.hc.core5.http.HttpResponse; +import org.apache.hc.core5.http.Message; +import org.apache.hc.core5.http.nio.entity.BasicAsyncEntityConsumer; +import org.apache.hc.core5.http.nio.support.BasicRequestProducer; +import org.apache.hc.core5.http.nio.support.BasicResponseConsumer; + +/** + * Async client example for RFC 9842 Compression Dictionary Transport. + *

+ * The example uses {@code canicompress.com}, which publishes a new JavaScript + * bundle every minute. The previous bundle is used as a compression dictionary + * for the current bundle. + *

+ * The client: + *

    + *
  1. Fetches the two most recent bundle URLs from the manifest.
    2. + *
  2. Fetches the previous bundle and stores it as a compression dictionary.
    3. + *
  3. Fetches the current bundle using the negotiated dictionary.
    4. + *
  4. Transparently decompresses the {@code dcz} response.
    5. + *
+ */ +public final class AsyncClientCompressionDictionary { + + private static final String ORIGIN = "https://canicompress.com"; + private static final URI MANIFEST_URI = URI.create(ORIGIN + "/manifest.json"); + + public static void main(final String[] args) throws Exception { + final BasicCompressionDictionaryStore dictionaryStore = + new BasicCompressionDictionaryStore(); + + try (final CloseableHttpAsyncClient client = HttpAsyncClients.custom() + .setCompressionDictionaryStore(dictionaryStore) + .build()) { + + client.start(); + + final Message manifestResponse = + execute(client, MANIFEST_URI); + + final String manifest = new String( + manifestResponse.getBody(), + StandardCharsets.UTF_8); + + final List paths = extractDeployPaths(manifest); + if (paths.size() < 2) { + throw new IllegalStateException( + "Unable to find two recent deploys in manifest"); + } + + final long cacheBuster = System.currentTimeMillis(); + + final URI dictionaryUri = URI.create( + ORIGIN + paths.get(1) + "?t=" + cacheBuster + "a"); + + final URI resourceUri = URI.create( + ORIGIN + paths.get(0) + "?t=" + cacheBuster + "b"); + + System.out.println("Dictionary:"); + System.out.println(" " + dictionaryUri); + + System.out.println(); + System.out.println("Resource:"); + System.out.println(" " + resourceUri); + + System.out.println(); + System.out.println("Fetching dictionary..."); + + final Message dictionaryResponse = + execute(client, dictionaryUri); + + final HttpResponse dictionaryHead = dictionaryResponse.getHead(); + + System.out.println("Status : " + dictionaryHead.getCode()); + + final Header useAsDictionary = + dictionaryHead.getFirstHeader("Use-As-Dictionary"); + + System.out.println("Use-As-Dictionary : " + + (useAsDictionary != null + ? useAsDictionary.getValue() + : "(none)")); + + System.out.println("Dictionary bytes : " + + (dictionaryResponse.getBody() != null + ? dictionaryResponse.getBody().length + : 0)); + + System.out.println("Stored dictionaries: " + + dictionaryStore.getByOrigin(dictionaryUri).size()); + + System.out.println(); + System.out.println("Fetching current resource..."); + + final Message resourceResponse = + execute(client, resourceUri); + + final HttpResponse resourceHead = resourceResponse.getHead(); + + System.out.println("Status : " + resourceHead.getCode()); + + final Header contentEncoding = + resourceHead.getFirstHeader(HttpHeaders.CONTENT_ENCODING); + + System.out.println("Content-Encoding : " + + (contentEncoding != null + ? contentEncoding.getValue() + : "(none)")); + + final byte[] body = resourceResponse.getBody() != null + ? resourceResponse.getBody() + : new byte[0]; + + System.out.println("Decoded bytes : " + body.length); + + final String text = new String(body, StandardCharsets.UTF_8); + + System.out.println("Response prefix : " + + text.substring(0, Math.min(text.length(), 120)) + .replace('\n', ' ')); + } + } + + private static Message execute( + final CloseableHttpAsyncClient client, + final URI uri) throws Exception { + + final SimpleHttpRequest request = SimpleRequestBuilder.get(uri) + .build(); + + final Future> future = client.execute( + new BasicRequestProducer(request, null), + new BasicResponseConsumer<>(new BasicAsyncEntityConsumer()), + null); + + return future.get(); + } + + private static List extractDeployPaths(final String manifest) { + final List paths = new ArrayList<>(); + + int offset = manifest.indexOf("\"recent_deploys\""); + if (offset < 0) { + return paths; + } + + while (true) { + final int name = manifest.indexOf("\"path\"", offset); + if (name < 0) { + break; + } + + final int colon = manifest.indexOf(':', name); + if (colon < 0) { + break; + } + + final int start = manifest.indexOf('"', colon + 1); + if (start < 0) { + break; + } + + final int end = manifest.indexOf('"', start + 1); + if (end < 0) { + break; + } + + final String path = manifest.substring(start + 1, end); + if (!paths.contains(path)) { + paths.add(path); + } + + offset = end + 1; + } + + return paths; + } +} \ No newline at end of file diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/examples/AsyncClientServerDictionaryBrotli.java b/httpclient5/src/test/java/org/apache/hc/client5/http/examples/AsyncClientServerDictionaryBrotli.java new file mode 100644 index 0000000000..f3ab55ecc7 --- /dev/null +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/examples/AsyncClientServerDictionaryBrotli.java @@ -0,0 +1,591 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.examples; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.InetAddress; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.KeyFactory; +import java.security.KeyStore; +import java.security.MessageDigest; +import java.security.PrivateKey; +import java.security.cert.Certificate; +import java.security.cert.CertificateFactory; +import java.security.spec.PKCS8EncodedKeySpec; +import java.util.Base64; +import java.util.concurrent.Future; + +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; + +import com.aayushatharva.brotli4j.Brotli4jLoader; +import com.aayushatharva.brotli4j.encoder.BrotliOutputStream; +import com.aayushatharva.brotli4j.encoder.Encoder; +import com.aayushatharva.brotli4j.encoder.PreparedDictionary; + +import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; +import org.apache.hc.client5.http.async.methods.SimpleRequestBuilder; +import org.apache.hc.client5.http.entity.compress.BasicCompressionDictionaryStore; +import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; +import org.apache.hc.client5.http.impl.async.HttpAsyncClients; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.client5.http.ssl.DefaultClientTlsStrategy; +import org.apache.hc.core5.http.ClassicHttpRequest; +import org.apache.hc.core5.http.ClassicHttpResponse; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpHeaders; +import org.apache.hc.core5.http.HttpResponse; +import org.apache.hc.core5.http.HttpStatus; +import org.apache.hc.core5.http.Message; +import org.apache.hc.core5.http.impl.bootstrap.HttpServer; +import org.apache.hc.core5.http.impl.bootstrap.ServerBootstrap; +import org.apache.hc.core5.http.io.HttpRequestHandler; +import org.apache.hc.core5.http.io.entity.ByteArrayEntity; +import org.apache.hc.core5.http.nio.entity.BasicAsyncEntityConsumer; +import org.apache.hc.core5.http.nio.support.BasicRequestProducer; +import org.apache.hc.core5.http.nio.support.BasicResponseConsumer; +import org.apache.hc.core5.http.protocol.HttpContext; +import org.apache.hc.core5.io.CloseMode; + +/** + * End-to-end RFC 9842 Dictionary-Compressed Brotli ({@code dcb}) example. + *

+ * A local HTTPS server first serves a resource marked with + * {@code Use-As-Dictionary}. The asynchronous client stores that response + * as a compression dictionary. + *

+ * A subsequent matching request advertises the dictionary through + * {@code Available-Dictionary}. The server then returns a real + * Dictionary-Compressed Brotli response and the client transparently + * decompresses it. + *

+ * The TLS certificate embedded in this example is for localhost testing only. + */ +public final class AsyncClientServerDictionaryBrotli { + + private static final String DCB = "dcb"; + + private static final byte[] DCB_MAGIC = { + (byte) 0xff, 0x44, 0x43, 0x42 + }; + + private static final byte[] DICTIONARY = ( + "{" + + "\"application\":\"Apache HttpClient\"," + + "\"feature\":\"Compression Dictionary Transport\"," + + "\"version\":1," + + "\"message\":\"This document is used as a shared compression dictionary.\"," + + "\"description\":\"Apache HttpClient supports transparent asynchronous " + + "HTTP content decompression using gzip, deflate, Brotli and Zstandard.\"" + + "}" + ).getBytes(StandardCharsets.UTF_8); + + private static final byte[] RESOURCE = ( + "{" + + "\"application\":\"Apache HttpClient\"," + + "\"feature\":\"Compression Dictionary Transport\"," + + "\"version\":2," + + "\"message\":\"This document is compressed using the previous response " + + "as a shared compression dictionary.\"," + + "\"description\":\"Apache HttpClient supports transparent asynchronous " + + "HTTP content decompression using gzip, deflate, Brotli and Zstandard.\"" + + "}" + ).getBytes(StandardCharsets.UTF_8); + + /* + * Test-only localhost certificate. + */ + private static final String CERTIFICATE = + "-----BEGIN CERTIFICATE-----\n" + + "MIIDJTCCAg2gAwIBAgIUcY/j5gVTlIfg/yW+kjv4Pg7q8mYwDQYJKoZIhvcNAQEL\n" + + "BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI2MDgyMTExMTE1M1oXDTM2MDgx\n" + + "ODExMTE1M1owFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF\n" + + "AAOCAQ8AMIIBCgKCAQEApiAxQTbyBOrja4j73l2RqCDRODtg7+6rziV2T31UPhNF\n" + + "L3gcn6ApTJdp9roR89ndYHvN/OD49n3eryD5AEf/KcuMuHsFYUFodmUXhLk54ndl\n" + + "REw3iDl/4DTFcaRzRgAK/yVtlp6mYe6pRiEv7oR9oBXb7g3VzPvYpJ6ZenX1xuiz\n" + + "t5CDpZj4lmFoUHGxbcj02UOl4CCXTSqYIM6ibt9Y6EjdxxEzz9DfCkIw1z9lRpp5\n" + + "wkcvauZTokWQO9kmYQPsaHYwcWVYd7ahXo8d67sgDMXSiyqvUYgcog298iC8K+o0\n" + + "ZaAOxQqfFj+ZAamUVHwUlJPa7WLP17ev8EQDfBmi1wIDAQABo28wbTAdBgNVHQ4E\n" + + "FgQUESRWkI3fjUV4Yck000U1MlsPx1IwHwYDVR0jBBgwFoAUESRWkI3fjUV4Yck0\n" + + "00U1MlsPx1IwDwYDVR0TAQH/BAUwAwEB/zAaBgNVHREEEzARgglsb2NhbGhvc3SH\n" + + "BH8AAAEwDQYJKoZIhvcNAQELBQADggEBAHbtKlv5rZq3pu3yFj/apIIQyapQo2Gn\n" + + "8Qhw6IRFEZ+SYP/DVxYpEY8TlJy+XxSkSeDqpgGdYKG4Ljt762KF95rsxUAg+IcC\n" + + "6s/wDSpQ9GMTdGRGSeGd/hR1O0MuqCiIBijYK+MjutKZOOvMSsdE0wjhUCoWNIOS\n" + + "AlMzBJL1Np5dDs8Dqne/tNA8MZ/Dsx9gmo4Fv1JuIHWTCjG87CpZB78+G+QOhJWF\n" + + "6ubYTaNX/V/GW+cFX7XlF5iWy1o9TlBHMmbNos/za3FIRIEc45vrK4sM/0U1P8cJ\n" + + "eNR5rHEXdQX7C0AEHs2Xs29D0k7egK9AqoDWGihzTcwGItFwIf1+ZbI=\n" + + "-----END CERTIFICATE-----"; + + private static final String PRIVATE_KEY = + "-----BEGIN PRIVATE KEY-----\n" + + "MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCmIDFBNvIE6uNr\n" + + "iPveXZGoINE4O2Dv7qvOJXZPfVQ+E0UveByfoClMl2n2uhHz2d1ge8384Pj2fd6v\n" + + "IPkAR/8py4y4ewVhQWh2ZReEuTnid2VETDeIOX/gNMVxpHNGAAr/JW2WnqZh7qlG\n" + + "IS/uhH2gFdvuDdXM+9iknpl6dfXG6LO3kIOlmPiWYWhQcbFtyPTZQ6XgIJdNKpgg\n" + + "zqJu31joSN3HETPP0N8KQjDXP2VGmnnCRy9q5lOiRZA72SZhA+xodjBxZVh3tqFe\n" + + "jx3ruyAMxdKLKq9RiByiDb3yILwr6jRloA7FCp8WP5kBqZRUfBSUk9rtYs/Xt6/w\n" + + "RAN8GaLXAgMBAAECggEAMRb2NxUrczSNu2shMlZoAkygRoOVY5Edh68eROL+D9HV\n" + + "8e8GVk0XpyBfGZ9mSq6ocihjeERqjTwon4uYyPJ9fjY+AQ2pS1Husn2w83Fgn4E0\n" + + "lXgIOOL03KX7ald0EM1Wcor21TlQZUQHFUgdR9gy3ylWcgP4l7gcDpknNT7CP+Jt\n" + + "rkNsifYFES2/lRYyFDbZBqV4Qjc2L+iik50qM+rj5B7DtjTns97+/Ck/ke1vGMyY\n" + + "waaqrOg0MH2O/LWXfHaqAUfEiq0AfQRqnqPKAofNbCTlFSEVKrH21bpRzF8kfNaR\n" + + "n62KVWSYN+7WbXAdmyoIiDw8McXW/0zFKppAoOWkAQKBgQDWwNxUyQz8XtNRoU6l\n" + + "aW0Z/lDxhNlDHwoRxTRKtnaLhaNJO/UoF2HGu/LDAiS29xUsyULyE3ViSVc/TgF9\n" + + "k74ou+KqfIOjNKN2xJ+NvDmfz4am7TmjM9LggPZii8yIMu9BXUJMOmDgkecm2Dzb\n" + + "EUbfTw0YcD2xpDl5jaU5V+QD1wKBgQDGCGBWV3n+ZUF2AkLc7OVek5XKA3VTZg8m\n" + + "CmN1qQ+ThoO94TXS+VtO1Rarhfz6Nhdggdder4VxuLgthmHtkQtox90GBE0VjJ0Z\n" + + "RTkGrUY6NJjpVhtsdZmB7gzrlXtmO0yp0oXdyXe6fc0jjsf5YmjYfy1/wk57wB7K\n" + + "jXUo6CN5AQKBgA+z2mh4qvJpHJqDaPS/WLLl3ZVLWXeG9X2HJeOwo8pf4yifsbVU\n" + + "wFl/tKh9p6GZP3se3D5HHfYp1q9STNmZy/W+hzxgDmAIoUs15VS/xpbg3b+m6Of+\n" + + "ChVQWLOr9TCgSM5Gu2pHen3xLS2x8gEyqjP528NFsb0jfPBeYw5mVs3RAoGBALlf\n" + + "9e5dDJGa72AsVbLA/yU9OiZUfmuHSf7uEpR9oVsTvBbuzpejXFm7FvGRB3KhV9i7\n" + + "MoQsAdqmc6IJ/XmJIQkArmGHfTEC47xYFD2vzeGGgu1J8Xnhy8TYtbeBwnW8ZNND\n" + + "gpROl4k3YeQ7L+6+tC6VPl4t4ZHuEeTB7j5Qr4QBAoGBAKpm9gLtvNYiCRvQjE2x\n" + + "vspx6yBa4z9E9Zzd98jXwIgWmbeN2oLuWXbHej4aBNCTlcoJhOof5wOlEUQhyjf9\n" + + "584Yj4SGVyXFWk3LOBJvr4iHfnm8uNxZxCZBqIwAZsK4e/02ExuT7Qpvh3kkn435\n" + + "wBYccyXfapxyL80kzfYp7bqb\n" + + "-----END PRIVATE KEY-----"; + + static { + Brotli4jLoader.ensureAvailability(); + } + + public static void main(final String[] args) throws Exception { + final Certificate certificate = loadCertificate(); + final SSLContext serverSslContext = + createServerSslContext(certificate, loadPrivateKey()); + final SSLContext clientSslContext = + createClientSslContext(certificate); + + final byte[] dictionaryHash = + MessageDigest.getInstance("SHA-256").digest(DICTIONARY); + + final HttpServer server = ServerBootstrap.bootstrap() + .setLocalAddress(InetAddress.getLoopbackAddress()) + .setListenerPort(0) + .setCanonicalHostName("localhost") + .setSslContext(serverSslContext) + .register("/dictionary", new DictionaryHandler()) + .register("/resource", new ResourceHandler(dictionaryHash)) + .create(); + + server.start(); + + final int port = server.getLocalPort(); + + final URI dictionaryUri = + URI.create("https://localhost:" + port + "/dictionary"); + + final URI resourceUri = + URI.create("https://localhost:" + port + "/resource"); + + final BasicCompressionDictionaryStore dictionaryStore = + new BasicCompressionDictionaryStore(); + + final PoolingAsyncClientConnectionManager connectionManager = + PoolingAsyncClientConnectionManagerBuilder.create() + .setTlsStrategy(new DefaultClientTlsStrategy(clientSslContext)) + .build(); + + try (final CloseableHttpAsyncClient client = HttpAsyncClients.custom() + .setConnectionManager(connectionManager) + .setCompressionDictionaryStore(dictionaryStore) + .build()) { + + client.start(); + + System.out.println("Fetching dictionary:"); + System.out.println(" " + dictionaryUri); + + final Message dictionaryResponse = + execute(client, dictionaryUri); + + final HttpResponse dictionaryHead = + dictionaryResponse.getHead(); + + final Header useAsDictionary = + dictionaryHead.getFirstHeader("Use-As-Dictionary"); + + System.out.println("Status : " + + dictionaryHead.getCode()); + + System.out.println("Use-As-Dictionary : " + + (useAsDictionary != null + ? useAsDictionary.getValue() + : "(none)")); + + System.out.println("Dictionary bytes : " + + dictionaryResponse.getBody().length); + + System.out.println("Stored dictionaries: " + + dictionaryStore.getByOrigin(dictionaryUri).size()); + + System.out.println(); + System.out.println("Fetching DCB resource:"); + System.out.println(" " + resourceUri); + + final Message resourceResponse = + execute(client, resourceUri); + + final HttpResponse resourceHead = + resourceResponse.getHead(); + + final Header contentEncoding = + resourceHead.getFirstHeader(HttpHeaders.CONTENT_ENCODING); + + final byte[] decoded = resourceResponse.getBody(); + + System.out.println("Status : " + + resourceHead.getCode()); + + System.out.println("Content-Encoding : " + + (contentEncoding != null + ? contentEncoding.getValue() + : "(none)")); + + System.out.println("Decoded bytes : " + decoded.length); + + System.out.println("Response (plain) : " + + new String(decoded, StandardCharsets.UTF_8)); + + if (!MessageDigest.isEqual(RESOURCE, decoded)) { + throw new IllegalStateException( + "Decoded DCB response does not match original resource"); + } + + if (contentEncoding == null + || !DCB.equalsIgnoreCase(contentEncoding.getValue())) { + throw new IllegalStateException( + "Server did not return a DCB response"); + } + + System.out.println(); + System.out.println("DCB round-trip successful."); + } finally { + server.close(CloseMode.GRACEFUL); + } + } + + private static Message execute( + final CloseableHttpAsyncClient client, + final URI uri) throws Exception { + + final SimpleHttpRequest request = + SimpleRequestBuilder.get(uri).build(); + + final Future> future = + client.execute( + new BasicRequestProducer(request, null), + new BasicResponseConsumer<>( + new BasicAsyncEntityConsumer()), + null); + + return future.get(); + } + + private static final class DictionaryHandler + implements HttpRequestHandler { + + @Override + public void handle( + final ClassicHttpRequest request, + final ClassicHttpResponse response, + final HttpContext context) { + + response.setCode(HttpStatus.SC_OK); + + response.addHeader( + HttpHeaders.CACHE_CONTROL, + "max-age=3600"); + + response.addHeader( + "Use-As-Dictionary", + "match=\"/resource\", id=\"local-dcb-v1\""); + + response.setEntity( + new ByteArrayEntity( + DICTIONARY, + ContentType.APPLICATION_JSON)); + } + } + + private static final class ResourceHandler + implements HttpRequestHandler { + + private final byte[] dictionaryHash; + private final String availableDictionary; + + ResourceHandler(final byte[] dictionaryHash) { + this.dictionaryHash = dictionaryHash.clone(); + this.availableDictionary = + ":" + Base64.getEncoder() + .encodeToString(dictionaryHash) + ":"; + } + + @Override + public void handle( + final ClassicHttpRequest request, + final ClassicHttpResponse response, + final HttpContext context) throws IOException { + + final Header available = + request.getFirstHeader("Available-Dictionary"); + + final Header acceptEncoding = + request.getFirstHeader(HttpHeaders.ACCEPT_ENCODING); + + System.out.println(); + System.out.println("Server received:"); + System.out.println("Available-Dictionary: " + + (available != null + ? available.getValue() + : "(none)")); + System.out.println("Accept-Encoding : " + + (acceptEncoding != null + ? acceptEncoding.getValue() + : "(none)")); + + final boolean dictionaryMatches = + available != null + && availableDictionary.equals( + available.getValue()); + + final boolean acceptsDcb = + acceptEncoding != null + && containsToken( + acceptEncoding.getValue(), DCB); + + if (!dictionaryMatches || !acceptsDcb) { + response.setCode(HttpStatus.SC_OK); + response.setEntity( + new ByteArrayEntity( + RESOURCE, + ContentType.APPLICATION_JSON)); + return; + } + + final byte[] compressed = + createDcb( + DICTIONARY, + dictionaryHash, + RESOURCE); + + response.setCode(HttpStatus.SC_OK); + response.addHeader( + HttpHeaders.CONTENT_ENCODING, + DCB); + + response.addHeader( + HttpHeaders.VARY, + "Accept-Encoding, Available-Dictionary"); + + response.setEntity( + new ByteArrayEntity( + compressed, + ContentType.APPLICATION_OCTET_STREAM)); + } + } + + private static byte[] createDcb( + final byte[] dictionary, + final byte[] dictionaryHash, + final byte[] content) throws IOException { + + final ByteBuffer dictionaryBuffer = + ByteBuffer.allocateDirect(dictionary.length); + + dictionaryBuffer.put(dictionary); + dictionaryBuffer.flip(); + + /* + * Shared dictionary type 0 is a raw LZ77 prefix dictionary. + */ + final PreparedDictionary preparedDictionary = + Encoder.prepareDictionary(dictionaryBuffer, 0); + + final ByteArrayOutputStream compressed = + new ByteArrayOutputStream(); + + try { + final Encoder.Parameters parameters = + Encoder.Parameters.create( + 6, + 24, + Encoder.Mode.TEXT); + + try (final BrotliOutputStream out = + new BrotliOutputStream( + compressed, + parameters)) { + + out.attachDictionary(preparedDictionary); + out.write(content); + } + } finally { + if (preparedDictionary instanceof AutoCloseable) { + try { + ((AutoCloseable) preparedDictionary).close(); + } catch (final Exception ex) { + throw new IOException( + "Unable to release Brotli dictionary", + ex); + } + } + } + + final ByteArrayOutputStream dcb = + new ByteArrayOutputStream( + DCB_MAGIC.length + + dictionaryHash.length + + compressed.size()); + + dcb.write(DCB_MAGIC); + dcb.write(dictionaryHash); + compressed.writeTo(dcb); + + return dcb.toByteArray(); + } + + private static boolean containsToken( + final String value, + final String token) { + + final String[] tokens = value.split(","); + + for (final String current : tokens) { + if (token.equalsIgnoreCase(current.trim())) { + return true; + } + } + + return false; + } + + private static Certificate loadCertificate() + throws Exception { + + final String value = CERTIFICATE + .replace("-----BEGIN CERTIFICATE-----", "") + .replace("-----END CERTIFICATE-----", "") + .replaceAll("\\s", ""); + + final byte[] encoded = + Base64.getDecoder().decode(value); + + final CertificateFactory factory = + CertificateFactory.getInstance("X.509"); + + return factory.generateCertificate( + new ByteArrayInputStream(encoded)); + } + + private static PrivateKey loadPrivateKey() + throws Exception { + + final String value = PRIVATE_KEY + .replace("-----BEGIN PRIVATE KEY-----", "") + .replace("-----END PRIVATE KEY-----", "") + .replaceAll("\\s", ""); + + final byte[] encoded = + Base64.getDecoder().decode(value); + + return KeyFactory.getInstance("RSA") + .generatePrivate( + new PKCS8EncodedKeySpec(encoded)); + } + + private static SSLContext createServerSslContext( + final Certificate certificate, + final PrivateKey privateKey) throws Exception { + + final char[] password = "changeit".toCharArray(); + + final KeyStore keyStore = + KeyStore.getInstance(KeyStore.getDefaultType()); + + keyStore.load(null, null); + + keyStore.setKeyEntry( + "localhost", + privateKey, + password, + new Certificate[]{certificate}); + + final KeyManagerFactory keyManagerFactory = + KeyManagerFactory.getInstance( + KeyManagerFactory.getDefaultAlgorithm()); + + keyManagerFactory.init(keyStore, password); + + final SSLContext sslContext = + SSLContext.getInstance("TLS"); + + sslContext.init( + keyManagerFactory.getKeyManagers(), + null, + null); + + return sslContext; + } + + private static SSLContext createClientSslContext( + final Certificate certificate) throws Exception { + + final KeyStore trustStore = + KeyStore.getInstance(KeyStore.getDefaultType()); + + trustStore.load(null, null); + + trustStore.setCertificateEntry( + "localhost", + certificate); + + final TrustManagerFactory trustManagerFactory = + TrustManagerFactory.getInstance( + TrustManagerFactory.getDefaultAlgorithm()); + + trustManagerFactory.init(trustStore); + + final SSLContext sslContext = + SSLContext.getInstance("TLS"); + + sslContext.init( + null, + trustManagerFactory.getTrustManagers(), + null); + + return sslContext; + } +} \ No newline at end of file From 939dfa205128ebc814f4323e0158f046e0df8f21 Mon Sep 17 00:00:00 2001 From: Arturo Bernal Date: Fri, 21 Aug 2026 16:26:41 +0200 Subject: [PATCH 2/3] add test --- ...InflatingDictionaryBrotliDataConsumer.java | 247 +++++++++++++ ...stInflatingDictionaryZstdDataConsumer.java | 275 ++++++++++++++ .../TestBasicCompressionDictionaryStore.java | 251 +++++++++++++ .../compress/TestCompressionDictionary.java | 197 ++++++++++ ...estCompressionDictionaryHeaderSupport.java | 103 ++++++ ...ContentCompressionAsyncExecDictionary.java | 343 ++++++++++++++++++ ...stDefaultCompressionDictionaryMatcher.java | 280 ++++++++++++++ ...tDictionaryCapturingAsyncDataConsumer.java | 266 ++++++++++++++ .../http/impl/async/TestUseAsDictionary.java | 210 +++++++++++ 9 files changed, 2172 insertions(+) create mode 100644 httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/TestInflatingDictionaryBrotliDataConsumer.java create mode 100644 httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/TestInflatingDictionaryZstdDataConsumer.java create mode 100644 httpclient5/src/test/java/org/apache/hc/client5/http/entity/compress/TestBasicCompressionDictionaryStore.java create mode 100644 httpclient5/src/test/java/org/apache/hc/client5/http/entity/compress/TestCompressionDictionary.java create mode 100644 httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestCompressionDictionaryHeaderSupport.java create mode 100644 httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestContentCompressionAsyncExecDictionary.java create mode 100644 httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestDefaultCompressionDictionaryMatcher.java create mode 100644 httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestDictionaryCapturingAsyncDataConsumer.java create mode 100644 httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestUseAsDictionary.java diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/TestInflatingDictionaryBrotliDataConsumer.java b/httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/TestInflatingDictionaryBrotliDataConsumer.java new file mode 100644 index 0000000000..e59b37118e --- /dev/null +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/TestInflatingDictionaryBrotliDataConsumer.java @@ -0,0 +1,247 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.async.methods; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.net.URI; +import java.nio.ByteBuffer; +import java.time.Instant; +import java.util.Collections; + +import org.apache.hc.client5.http.entity.compress.BasicCompressionDictionaryStore; +import org.apache.hc.client5.http.entity.compress.CompressionDictionary; +import org.apache.hc.client5.http.entity.compress.CompressionDictionaryStore; +import org.apache.hc.core5.http.nio.AsyncDataConsumer; +import org.junit.jupiter.api.Test; + +/** + * Native-free unit tests for {@link InflatingDictionaryBrotliDataConsumer}. + *

+ * These tests exercise only the header-parsing and dictionary-lookup logic, + * all of which runs before the lazy Brotli {@code DecoderJNI.Wrapper} is + * created, so no native Brotli library is required. + */ +class TestInflatingDictionaryBrotliDataConsumer { + + private static final byte[] MAGIC = { + (byte) 0xff, 0x44, 0x43, 0x42 + }; + + private static final int HASH_LENGTH = 32; + private static final int HEADER_LENGTH = MAGIC.length + HASH_LENGTH; + + private static CompressionDictionary dictionary(final byte[] content) { + final Instant storedAt = Instant.parse("2020-01-01T00:00:00Z"); + final Instant validUntil = Instant.parse("2030-01-01T00:00:00Z"); + return new CompressionDictionary( + content, + URI.create("https://example.com/dict"), + "/*", + "dict-1", + storedAt, + validUntil); + } + + private static byte[] header(final byte[] magic, final byte[] hash) { + final byte[] out = new byte[HEADER_LENGTH]; + System.arraycopy(magic, 0, out, 0, magic.length); + System.arraycopy(hash, 0, out, magic.length, HASH_LENGTH); + return out; + } + + @Test + void invalidMagicThrowsFromConsumeSingleBuffer() { + final AsyncDataConsumer downstream = mock(AsyncDataConsumer.class); + final CompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final InflatingDictionaryBrotliDataConsumer consumer = + new InflatingDictionaryBrotliDataConsumer(downstream, store); + + final byte[] bogusMagic = {0x00, 0x44, 0x43, 0x42}; + final byte[] hash = new byte[HASH_LENGTH]; + final ByteBuffer src = ByteBuffer.wrap(header(bogusMagic, hash)); + + final IOException ex = assertThrows(IOException.class, () -> consumer.consume(src)); + assertTrue(ex.getMessage().contains("Invalid DCB stream header"), ex.getMessage()); + verifyNoInteractions(downstream); + } + + @Test + void invalidMagicThrowsWhenHeaderSplitAcrossTwoBuffers() { + final AsyncDataConsumer downstream = mock(AsyncDataConsumer.class); + final CompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final InflatingDictionaryBrotliDataConsumer consumer = + new InflatingDictionaryBrotliDataConsumer(downstream, store); + + final byte[] bogusMagic = {0x00, 0x44, 0x43, 0x42}; + final byte[] hash = new byte[HASH_LENGTH]; + final byte[] full = header(bogusMagic, hash); + + // First buffer only carries a partial header; must not throw yet. + final ByteBuffer first = ByteBuffer.wrap(full, 0, 2).slice(); + assertDoesNotThrowConsume(consumer, first); + + // Completing the header triggers validation. + final ByteBuffer second = ByteBuffer.wrap(full, 2, full.length - 2).slice(); + final IOException ex = assertThrows(IOException.class, () -> consumer.consume(second)); + assertTrue(ex.getMessage().contains("Invalid DCB stream header"), ex.getMessage()); + verifyNoInteractions(downstream); + } + + @Test + void unknownDictionaryThrowsFromConsume() { + final AsyncDataConsumer downstream = mock(AsyncDataConsumer.class); + final CompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final InflatingDictionaryBrotliDataConsumer consumer = + new InflatingDictionaryBrotliDataConsumer(downstream, store); + + // Valid MAGIC, but the store is empty so getByHash returns null. + final byte[] hash = new byte[HASH_LENGTH]; + for (int i = 0; i < HASH_LENGTH; i++) { + hash[i] = (byte) i; + } + final ByteBuffer src = ByteBuffer.wrap(header(MAGIC, hash)); + + final IOException ex = assertThrows(IOException.class, () -> consumer.consume(src)); + assertTrue(ex.getMessage().contains("Compression dictionary not available"), ex.getMessage()); + verifyNoInteractions(downstream); + } + + @Test + void unknownDictionaryThrowsWhenHeaderSplitAcrossTwoBuffers() { + final AsyncDataConsumer downstream = mock(AsyncDataConsumer.class); + final CompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final InflatingDictionaryBrotliDataConsumer consumer = + new InflatingDictionaryBrotliDataConsumer(downstream, store); + + final byte[] hash = new byte[HASH_LENGTH]; + final byte[] full = header(MAGIC, hash); + + final ByteBuffer first = ByteBuffer.wrap(full, 0, 20).slice(); + assertDoesNotThrowConsume(consumer, first); + + final ByteBuffer second = ByteBuffer.wrap(full, 20, full.length - 20).slice(); + final IOException ex = assertThrows(IOException.class, () -> consumer.consume(second)); + assertTrue(ex.getMessage().contains("Compression dictionary not available"), ex.getMessage()); + verifyNoInteractions(downstream); + } + + @Test + void knownHashPresentAndMatchingPassesLookupAndReachesDecoderInit() { + // A real store keyed by the dictionary's own SHA-256; the header hash + // matches, so the lookup succeeds and control proceeds to lazy decoder + // creation. Without the native library that step throws, but the lookup + // guard itself must be cleared (message must NOT be about availability). + final AsyncDataConsumer downstream = mock(AsyncDataConsumer.class); + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final CompressionDictionary dict = dictionary(new byte[]{'h', 'e', 'l', 'l', 'o'}); + store.add(dict); + + final InflatingDictionaryBrotliDataConsumer consumer = + new InflatingDictionaryBrotliDataConsumer(downstream, store); + + final ByteBuffer src = ByteBuffer.wrap(header(MAGIC, dict.getSha256())); + + try { + consumer.consume(src); + // If the native library is present, header consumed without error. + } catch (final IOException ex) { + // If native init fails, it must not be the availability guard. + assertTrue(!ex.getMessage().contains("Compression dictionary not available"), ex.getMessage()); + } catch (final Throwable linkError) { + // UnsatisfiedLinkError / NoClassDefFoundError when native lib absent. + assertTrue(true); + } + } + + @Test + void hashPresentButNotMatchingThrows() { + // Store returns a dictionary for any hash, but its SHA-256 differs from + // the header hash, so matchesHash() fails. + final AsyncDataConsumer downstream = mock(AsyncDataConsumer.class); + final CompressionDictionary dict = dictionary(new byte[]{1, 2, 3, 4}); + final CompressionDictionaryStore store = mock(CompressionDictionaryStore.class); + when(store.getByHash(org.mockito.ArgumentMatchers.any())).thenReturn(dict); + + final InflatingDictionaryBrotliDataConsumer consumer = + new InflatingDictionaryBrotliDataConsumer(downstream, store); + + // Header hash of all-zeros will not equal the dictionary's real SHA-256. + final byte[] hash = new byte[HASH_LENGTH]; + final ByteBuffer src = ByteBuffer.wrap(header(MAGIC, hash)); + + final IOException ex = assertThrows(IOException.class, () -> consumer.consume(src)); + assertTrue(ex.getMessage().contains("Compression dictionary not available"), ex.getMessage()); + verifyNoInteractions(downstream); + } + + @Test + void truncatedHeaderThenStreamEndThrows() { + final AsyncDataConsumer downstream = mock(AsyncDataConsumer.class); + final CompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final InflatingDictionaryBrotliDataConsumer consumer = + new InflatingDictionaryBrotliDataConsumer(downstream, store); + + // Fewer than 36 bytes: header stays incomplete, decoder stays null. + final byte[] partial = new byte[10]; + System.arraycopy(MAGIC, 0, partial, 0, MAGIC.length); + final ByteBuffer src = ByteBuffer.wrap(partial); + assertDoesNotThrowConsume(consumer, src); + + final IOException ex = assertThrows(IOException.class, + () -> consumer.streamEnd(Collections.emptyList())); + assertTrue(ex.getMessage().contains("Truncated DCB stream header"), ex.getMessage()); + verifyNoInteractions(downstream); + } + + @Test + void streamEndWithNoDataThrowsTruncatedHeader() { + final AsyncDataConsumer downstream = mock(AsyncDataConsumer.class); + final CompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final InflatingDictionaryBrotliDataConsumer consumer = + new InflatingDictionaryBrotliDataConsumer(downstream, store); + + final IOException ex = assertThrows(IOException.class, + () -> consumer.streamEnd(Collections.emptyList())); + assertTrue(ex.getMessage().contains("Truncated DCB stream header"), ex.getMessage()); + verifyNoInteractions(downstream); + } + + private static void assertDoesNotThrowConsume( + final InflatingDictionaryBrotliDataConsumer consumer, final ByteBuffer src) { + try { + consumer.consume(src); + } catch (final IOException ex) { + throw new AssertionError("Unexpected IOException on partial header", ex); + } + } +} diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/TestInflatingDictionaryZstdDataConsumer.java b/httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/TestInflatingDictionaryZstdDataConsumer.java new file mode 100644 index 0000000000..10f6a2ea35 --- /dev/null +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/TestInflatingDictionaryZstdDataConsumer.java @@ -0,0 +1,275 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.async.methods; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Collections; +import java.util.List; + +import com.github.luben.zstd.ZstdCompressCtx; + +import org.apache.hc.client5.http.entity.compress.BasicCompressionDictionaryStore; +import org.apache.hc.client5.http.entity.compress.CompressionDictionary; +import org.apache.hc.client5.http.impl.ZstdRuntime; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpException; +import org.apache.hc.core5.http.nio.AsyncDataConsumer; +import org.apache.hc.core5.http.nio.CapacityChannel; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; + +class TestInflatingDictionaryZstdDataConsumer { + + private static final byte[] MAGIC = { + 0x5e, 0x2a, 0x4d, 0x18, 0x20, 0x00, 0x00, 0x00 + }; + + private static final int HASH_LENGTH = 32; + + /** + * Fake downstream consumer that accumulates all consumed bytes. + */ + private static final class AccumulatingConsumer implements AsyncDataConsumer { + + private final ByteArrayOutputStream out = new ByteArrayOutputStream(); + private boolean ended; + + @Override + public void updateCapacity(final CapacityChannel capacityChannel) throws IOException { + capacityChannel.update(Integer.MAX_VALUE); + } + + @Override + public void consume(final ByteBuffer src) throws IOException { + while (src.hasRemaining()) { + out.write(src.get()); + } + } + + @Override + public void streamEnd(final List trailers) throws HttpException, IOException { + ended = true; + } + + @Override + public void releaseResources() { + } + + byte[] toByteArray() { + return out.toByteArray(); + } + + boolean isEnded() { + return ended; + } + } + + private static CompressionDictionary dictionaryOf(final byte[] content) { + return new CompressionDictionary( + content, + URI.create("https://example.com/"), + "/*", + "dict-1", + Instant.parse("2026-01-01T00:00:00Z"), + Instant.parse("2099-01-01T00:00:00Z")); + } + + private static byte[] concat(final byte[]... parts) { + int total = 0; + for (final byte[] part : parts) { + total += part.length; + } + final byte[] result = new byte[total]; + int offset = 0; + for (final byte[] part : parts) { + System.arraycopy(part, 0, result, offset, part.length); + offset += part.length; + } + return result; + } + + private static byte[] compress(final byte[] dictionaryContent, final byte[] payload) { + try (ZstdCompressCtx cctx = new ZstdCompressCtx()) { + cctx.loadDict(dictionaryContent); + return cctx.compress(payload); + } + } + + @Test + void invalidMagicRaisesIOException() { + Assumptions.assumeTrue(ZstdRuntime.available()); + + final AccumulatingConsumer downstream = new AccumulatingConsumer(); + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final InflatingDictionaryZstdDataConsumer consumer = + new InflatingDictionaryZstdDataConsumer(downstream, store); + + final byte[] header = new byte[MAGIC.length + HASH_LENGTH]; + // deliberately wrong first byte + header[0] = 0x00; + + final ByteBuffer src = ByteBuffer.wrap(header); + final IOException ex = Assertions.assertThrows(IOException.class, () -> consumer.consume(src)); + Assertions.assertTrue(ex.getMessage().contains("Invalid DCZ stream header"), ex.getMessage()); + + consumer.releaseResources(); + } + + @Test + void unknownDictionaryHashRaisesIOException() { + Assumptions.assumeTrue(ZstdRuntime.available()); + + final AccumulatingConsumer downstream = new AccumulatingConsumer(); + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final InflatingDictionaryZstdDataConsumer consumer = + new InflatingDictionaryZstdDataConsumer(downstream, store); + + final byte[] hash = new byte[HASH_LENGTH]; + for (int i = 0; i < hash.length; i++) { + hash[i] = (byte) i; + } + final byte[] header = concat(MAGIC, hash); + + final ByteBuffer src = ByteBuffer.wrap(header); + final IOException ex = Assertions.assertThrows(IOException.class, () -> consumer.consume(src)); + Assertions.assertEquals("Compression dictionary not available", ex.getMessage()); + + consumer.releaseResources(); + } + + @Test + void truncatedHeaderThenStreamEndRaisesIOException() { + Assumptions.assumeTrue(ZstdRuntime.available()); + + final AccumulatingConsumer downstream = new AccumulatingConsumer(); + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final InflatingDictionaryZstdDataConsumer consumer = + new InflatingDictionaryZstdDataConsumer(downstream, store); + + // feed only part of the magic; header stays incomplete, so not initialized + final ByteBuffer src = ByteBuffer.wrap(new byte[] {MAGIC[0], MAGIC[1], MAGIC[2]}); + Assertions.assertDoesNotThrow(() -> consumer.consume(src)); + + final IOException ex = Assertions.assertThrows(IOException.class, + () -> consumer.streamEnd(Collections.

emptyList())); + Assertions.assertEquals("Truncated DCZ stream header", ex.getMessage()); + + consumer.releaseResources(); + } + + @Test + void headerSplitAcrossBuffersRaisesUnknownDictionary() { + Assumptions.assumeTrue(ZstdRuntime.available()); + + final AccumulatingConsumer downstream = new AccumulatingConsumer(); + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final InflatingDictionaryZstdDataConsumer consumer = + new InflatingDictionaryZstdDataConsumer(downstream, store); + + final byte[] hash = new byte[HASH_LENGTH]; + final byte[] header = concat(MAGIC, hash); + + // split header across two ByteBuffers; the valid MAGIC still validates, + // then the (all-zero, absent) dictionary hash is rejected. + final ByteBuffer first = ByteBuffer.wrap(header, 0, 5); + final ByteBuffer second = ByteBuffer.wrap(header, 5, header.length - 5); + + Assertions.assertDoesNotThrow(() -> consumer.consume(first)); + final IOException ex = Assertions.assertThrows(IOException.class, () -> consumer.consume(second)); + Assertions.assertEquals("Compression dictionary not available", ex.getMessage()); + + consumer.releaseResources(); + } + + @Test + void roundTripSingleBuffer() throws Exception { + Assumptions.assumeTrue(ZstdRuntime.available()); + + final byte[] dictionaryContent = "the quick brown fox jumps over the lazy dog".getBytes(StandardCharsets.UTF_8); + final byte[] payload = ("the quick brown fox jumps over the lazy dog " + + "and then the quick brown fox runs away").getBytes(StandardCharsets.UTF_8); + + final CompressionDictionary dictionary = dictionaryOf(dictionaryContent); + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + store.add(dictionary); + + final byte[] frame = compress(dictionaryContent, payload); + final byte[] stream = concat(MAGIC, dictionary.getSha256(), frame); + + final AccumulatingConsumer downstream = new AccumulatingConsumer(); + final InflatingDictionaryZstdDataConsumer consumer = + new InflatingDictionaryZstdDataConsumer(downstream, store); + + consumer.consume(ByteBuffer.wrap(stream)); + consumer.streamEnd(Collections.
emptyList()); + + Assertions.assertArrayEquals(payload, downstream.toByteArray()); + Assertions.assertTrue(downstream.isEnded()); + + consumer.releaseResources(); + } + + @Test + void roundTripSplitAcrossManyBuffers() throws Exception { + Assumptions.assumeTrue(ZstdRuntime.available()); + + final byte[] dictionaryContent = "dictionary payload sample content".getBytes(StandardCharsets.UTF_8); + final StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 200; i++) { + sb.append("dictionary payload sample content line ").append(i).append('\n'); + } + final byte[] payload = sb.toString().getBytes(StandardCharsets.UTF_8); + + final CompressionDictionary dictionary = dictionaryOf(dictionaryContent); + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + store.add(dictionary); + + final byte[] frame = compress(dictionaryContent, payload); + final byte[] stream = concat(MAGIC, dictionary.getSha256(), frame); + + final AccumulatingConsumer downstream = new AccumulatingConsumer(); + final InflatingDictionaryZstdDataConsumer consumer = + new InflatingDictionaryZstdDataConsumer(downstream, store); + + // feed one byte at a time to exercise header/frame splitting + for (int i = 0; i < stream.length; i++) { + consumer.consume(ByteBuffer.wrap(stream, i, 1)); + } + consumer.streamEnd(Collections.
emptyList()); + + Assertions.assertArrayEquals(payload, downstream.toByteArray()); + Assertions.assertTrue(downstream.isEnded()); + + consumer.releaseResources(); + } +} diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/entity/compress/TestBasicCompressionDictionaryStore.java b/httpclient5/src/test/java/org/apache/hc/client5/http/entity/compress/TestBasicCompressionDictionaryStore.java new file mode 100644 index 0000000000..d888322e84 --- /dev/null +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/entity/compress/TestBasicCompressionDictionaryStore.java @@ -0,0 +1,251 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.entity.compress; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URI; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.jupiter.api.Test; + +class TestBasicCompressionDictionaryStore { + + private static final Instant STORED_AT = Instant.parse("2020-01-01T00:00:00Z"); + private static final Instant VALID_UNTIL = Instant.parse("2030-01-01T00:00:00Z"); + + private static CompressionDictionary dictionary(final byte[] content, final URI source) { + return new CompressionDictionary(content, source, "/path", "id", STORED_AT, VALID_UNTIL); + } + + private static CompressionDictionary dictionary(final int n) { + return dictionary(new byte[]{(byte) n}, URI.create("https://example.com/")); + } + + @Test + void addNullThrows() { + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + assertThrows(NullPointerException.class, () -> store.add(null)); + } + + @Test + void getByHashNullThrows() { + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + assertThrows(NullPointerException.class, () -> store.getByHash(null)); + } + + @Test + void getByOriginNullThrows() { + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + assertThrows(NullPointerException.class, () -> store.getByOrigin(null)); + } + + @Test + void nonPositiveMaxEntriesThrows() { + assertThrows(IllegalArgumentException.class, () -> new BasicCompressionDictionaryStore(0)); + assertThrows(IllegalArgumentException.class, () -> new BasicCompressionDictionaryStore(-1)); + } + + @Test + void defaultConstructorWorks() { + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final CompressionDictionary dict = dictionary(1); + store.add(dict); + assertSame(dict, store.getByHash(dict.getSha256())); + } + + @Test + void addThenGetByHashReturnsDictionary() { + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final CompressionDictionary dict = dictionary(42); + store.add(dict); + assertSame(dict, store.getByHash(dict.getSha256())); + } + + @Test + void getByHashUnknownReturnsNull() { + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + store.add(dictionary(1)); + final CompressionDictionary other = dictionary(99); + assertNull(store.getByHash(other.getSha256())); + } + + @Test + void sameHashTwiceKeepsSingleEntry() { + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final URI source = URI.create("https://example.com/"); + final CompressionDictionary first = dictionary(new byte[]{7}, source); + final CompressionDictionary second = dictionary(new byte[]{7}, source); + store.add(first); + store.add(second); + assertSame(second, store.getByHash(first.getSha256())); + assertEquals(1, store.getByOrigin(source).size()); + } + + @Test + void evictionRemovesOldest() { + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(2); + final CompressionDictionary first = dictionary(1); + final CompressionDictionary second = dictionary(2); + final CompressionDictionary third = dictionary(3); + store.add(first); + store.add(second); + store.add(third); + assertNull(store.getByHash(first.getSha256())); + assertSame(second, store.getByHash(second.getSha256())); + assertSame(third, store.getByHash(third.getSha256())); + } + + @Test + void getByOriginFiltersByScheme() { + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final CompressionDictionary https = dictionary(new byte[]{1}, URI.create("https://example.com/")); + final CompressionDictionary http = dictionary(new byte[]{2}, URI.create("http://example.com/")); + store.add(https); + store.add(http); + final List result = store.getByOrigin(URI.create("https://example.com/other")); + assertEquals(1, result.size()); + assertSame(https, result.get(0)); + } + + @Test + void getByOriginFiltersByHost() { + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final CompressionDictionary a = dictionary(new byte[]{1}, URI.create("https://a.example.com/")); + final CompressionDictionary b = dictionary(new byte[]{2}, URI.create("https://b.example.com/")); + store.add(a); + store.add(b); + final List result = store.getByOrigin(URI.create("https://b.example.com/")); + assertEquals(1, result.size()); + assertSame(b, result.get(0)); + } + + @Test + void getByOriginTreatsHttpsDefaultPortAsEqual() { + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final CompressionDictionary implicit = dictionary(new byte[]{1}, URI.create("https://example.com/")); + store.add(implicit); + final List result = store.getByOrigin(URI.create("https://example.com:443/")); + assertEquals(1, result.size()); + assertSame(implicit, result.get(0)); + } + + @Test + void getByOriginTreatsHttpDefaultPortAsEqual() { + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final CompressionDictionary explicit = dictionary(new byte[]{1}, URI.create("http://example.com:80/")); + store.add(explicit); + final List result = store.getByOrigin(URI.create("http://example.com/")); + assertEquals(1, result.size()); + assertSame(explicit, result.get(0)); + } + + @Test + void getByOriginDistinguishesNonDefaultPort() { + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final CompressionDictionary def = dictionary(new byte[]{1}, URI.create("https://example.com/")); + final CompressionDictionary alt = dictionary(new byte[]{2}, URI.create("https://example.com:8443/")); + store.add(def); + store.add(alt); + assertEquals(1, store.getByOrigin(URI.create("https://example.com:8443/")).size()); + assertSame(alt, store.getByOrigin(URI.create("https://example.com:8443/")).get(0)); + assertSame(def, store.getByOrigin(URI.create("https://example.com/")).get(0)); + } + + @Test + void getByOriginNoMatchReturnsEmpty() { + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + store.add(dictionary(new byte[]{1}, URI.create("https://example.com/"))); + final List result = store.getByOrigin(URI.create("https://other.example.com/")); + assertTrue(result.isEmpty()); + } + + @Test + void clearEmptiesStore() { + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final CompressionDictionary dict = dictionary(1); + store.add(dict); + store.clear(); + assertNull(store.getByHash(dict.getSha256())); + assertTrue(store.getByOrigin(URI.create("https://example.com/")).isEmpty()); + } + + @Test + void concurrentAddsStayWithinMaxEntries() throws InterruptedException { + final int maxEntries = 16; + final int threads = 32; + final int perThread = 50; + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(maxEntries); + final ExecutorService executor = Executors.newFixedThreadPool(threads); + final CountDownLatch start = new CountDownLatch(1); + final CountDownLatch done = new CountDownLatch(threads); + final AtomicReference failure = new AtomicReference<>(); + try { + for (int t = 0; t < threads; t++) { + final int base = t; + executor.execute(() -> { + try { + start.await(); + for (int i = 0; i < perThread; i++) { + final byte[] content = new byte[]{(byte) base, (byte) i}; + store.add(dictionary(content, URI.create("https://example.com/"))); + } + } catch (final Throwable ex) { + failure.compareAndSet(null, ex); + } finally { + done.countDown(); + } + }); + } + start.countDown(); + assertTrue(done.await(30, TimeUnit.SECONDS), "threads did not finish in time"); + } finally { + executor.shutdownNow(); + } + assertNull(failure.get(), "concurrent add threw: " + failure.get()); + assertTrue(store.getByOrigin(URI.create("https://example.com/")).size() <= maxEntries); + } + + @Test + void getByHashReturnsNonNullAfterAdd() { + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final CompressionDictionary dict = dictionary(5); + store.add(dict); + assertNotNull(store.getByHash(dict.getSha256())); + } +} diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/entity/compress/TestCompressionDictionary.java b/httpclient5/src/test/java/org/apache/hc/client5/http/entity/compress/TestCompressionDictionary.java new file mode 100644 index 0000000000..6c0520145d --- /dev/null +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/entity/compress/TestCompressionDictionary.java @@ -0,0 +1,197 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.entity.compress; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Instant; + +import org.junit.jupiter.api.Test; + +class TestCompressionDictionary { + + private static final byte[] CONTENT = "the quick brown fox".getBytes(StandardCharsets.UTF_8); + private static final URI SOURCE = URI.create("https://example.com/dict.bin"); + private static final String MATCH = "/*"; + private static final String ID = "dict-1"; + private static final Instant STORED_AT = Instant.parse("2026-01-01T00:00:00Z"); + private static final Instant VALID_UNTIL = Instant.parse("2026-02-01T00:00:00Z"); + + private static CompressionDictionary newDictionary() { + return new CompressionDictionary(CONTENT, SOURCE, MATCH, ID, STORED_AT, VALID_UNTIL); + } + + private static byte[] sha256(final byte[] content) { + try { + return MessageDigest.getInstance("SHA-256").digest(content); + } catch (final NoSuchAlgorithmException ex) { + throw new IllegalStateException(ex); + } + } + + @Test + void constructorRejectsNullContent() { + assertThrows(NullPointerException.class, () -> + new CompressionDictionary(null, SOURCE, MATCH, ID, STORED_AT, VALID_UNTIL)); + } + + @Test + void constructorRejectsNullSource() { + assertThrows(NullPointerException.class, () -> + new CompressionDictionary(CONTENT, null, MATCH, ID, STORED_AT, VALID_UNTIL)); + } + + @Test + void constructorRejectsNullMatch() { + assertThrows(NullPointerException.class, () -> + new CompressionDictionary(CONTENT, SOURCE, null, ID, STORED_AT, VALID_UNTIL)); + } + + @Test + void constructorRejectsBlankMatch() { + assertThrows(IllegalArgumentException.class, () -> + new CompressionDictionary(CONTENT, SOURCE, " ", ID, STORED_AT, VALID_UNTIL)); + } + + @Test + void constructorRejectsNullStoredAt() { + assertThrows(NullPointerException.class, () -> + new CompressionDictionary(CONTENT, SOURCE, MATCH, ID, null, VALID_UNTIL)); + } + + @Test + void constructorRejectsNullValidUntil() { + assertThrows(NullPointerException.class, () -> + new CompressionDictionary(CONTENT, SOURCE, MATCH, ID, STORED_AT, null)); + } + + @Test + void nullIdBecomesEmptyString() { + final CompressionDictionary dict = + new CompressionDictionary(CONTENT, SOURCE, MATCH, null, STORED_AT, VALID_UNTIL); + assertEquals("", dict.getId()); + } + + @Test + void gettersReturnPassedValues() { + final CompressionDictionary dict = newDictionary(); + assertEquals(SOURCE, dict.getSource()); + assertEquals(MATCH, dict.getMatch()); + assertEquals(ID, dict.getId()); + assertEquals(STORED_AT, dict.getStoredAt()); + assertEquals(VALID_UNTIL, dict.getValidUntil()); + assertArrayEquals(CONTENT, dict.getContent()); + } + + @Test + void constructorClonesContent() { + final byte[] mutable = CONTENT.clone(); + final CompressionDictionary dict = + new CompressionDictionary(mutable, SOURCE, MATCH, ID, STORED_AT, VALID_UNTIL); + mutable[0] = (byte) (mutable[0] ^ 0xFF); + assertArrayEquals(CONTENT, dict.getContent()); + } + + @Test + void getContentReturnsDefensiveCopy() { + final CompressionDictionary dict = newDictionary(); + final byte[] first = dict.getContent(); + first[0] = (byte) (first[0] ^ 0xFF); + assertArrayEquals(CONTENT, dict.getContent()); + } + + @Test + void getContentReturnsDistinctArrays() { + final CompressionDictionary dict = newDictionary(); + final byte[] first = dict.getContent(); + final byte[] second = dict.getContent(); + assertNotSame(first, second); + assertArrayEquals(first, second); + } + + @Test + void getSha256ReturnsDefensiveCopy() { + final CompressionDictionary dict = newDictionary(); + final byte[] first = dict.getSha256(); + first[0] = (byte) (first[0] ^ 0xFF); + assertArrayEquals(sha256(CONTENT), dict.getSha256()); + } + + @Test + void getSha256ReturnsDistinctArrays() { + final CompressionDictionary dict = newDictionary(); + final byte[] first = dict.getSha256(); + final byte[] second = dict.getSha256(); + assertNotSame(first, second); + assertArrayEquals(first, second); + } + + @Test + void sha256MatchesFreshlyComputedDigest() { + final CompressionDictionary dict = newDictionary(); + assertArrayEquals(sha256(CONTENT), dict.getSha256()); + } + + @Test + void matchesHashTrueForSha256() { + final CompressionDictionary dict = newDictionary(); + assertTrue(dict.matchesHash(sha256(CONTENT))); + } + + @Test + void matchesHashFalseForOtherHash() { + final CompressionDictionary dict = newDictionary(); + assertFalse(dict.matchesHash(sha256("different".getBytes(StandardCharsets.UTF_8)))); + } + + @Test + void isFreshBeforeValidUntil() { + final CompressionDictionary dict = newDictionary(); + assertTrue(dict.isFresh(VALID_UNTIL.minusSeconds(1))); + } + + @Test + void isFreshAtValidUntil() { + final CompressionDictionary dict = newDictionary(); + assertFalse(dict.isFresh(VALID_UNTIL)); + } + + @Test + void isFreshAfterValidUntil() { + final CompressionDictionary dict = newDictionary(); + assertFalse(dict.isFresh(VALID_UNTIL.plusSeconds(1))); + } +} diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestCompressionDictionaryHeaderSupport.java b/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestCompressionDictionaryHeaderSupport.java new file mode 100644 index 0000000000..c9561aaa74 --- /dev/null +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestCompressionDictionaryHeaderSupport.java @@ -0,0 +1,103 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.impl.async; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +import org.junit.jupiter.api.Test; + +class TestCompressionDictionaryHeaderSupport { + + @Test + void testHeaderNameConstants() { + assertEquals("Use-As-Dictionary", CompressionDictionaryHeaderSupport.USE_AS_DICTIONARY); + assertEquals("Available-Dictionary", CompressionDictionaryHeaderSupport.AVAILABLE_DICTIONARY); + assertEquals("Dictionary-ID", CompressionDictionaryHeaderSupport.DICTIONARY_ID); + } + + @Test + void testFormatAvailableDictionaryKnownInput() { + final byte[] sha256 = "abc".getBytes(StandardCharsets.US_ASCII); + assertEquals(":YWJj:", CompressionDictionaryHeaderSupport.formatAvailableDictionary(sha256)); + } + + @Test + void testFormatAvailableDictionaryMatchesBase64() { + final byte[] sha256 = new byte[32]; + for (int i = 0; i < sha256.length; i++) { + sha256[i] = (byte) i; + } + final String expected = ":" + Base64.getEncoder().encodeToString(sha256) + ":"; + assertEquals(expected, CompressionDictionaryHeaderSupport.formatAvailableDictionary(sha256)); + } + + @Test + void testFormatAvailableDictionaryEmpty() { + assertEquals("::", CompressionDictionaryHeaderSupport.formatAvailableDictionary(new byte[0])); + } + + @Test + void testFormatAvailableDictionaryNull() { + assertThrows(NullPointerException.class, + () -> CompressionDictionaryHeaderSupport.formatAvailableDictionary(null)); + } + + @Test + void testFormatDictionaryIdPlain() { + assertEquals("\"foo\"", CompressionDictionaryHeaderSupport.formatDictionaryId("foo")); + } + + @Test + void testFormatDictionaryIdEmpty() { + assertEquals("\"\"", CompressionDictionaryHeaderSupport.formatDictionaryId("")); + } + + @Test + void testFormatDictionaryIdEscapesQuotes() { + assertEquals("\"a\\\"b\"", CompressionDictionaryHeaderSupport.formatDictionaryId("a\"b")); + } + + @Test + void testFormatDictionaryIdEscapesBackslash() { + assertEquals("\"a\\\\b\"", CompressionDictionaryHeaderSupport.formatDictionaryId("a\\b")); + } + + @Test + void testFormatDictionaryIdEscapesBoth() { + assertEquals("\"\\\\\\\"\"", CompressionDictionaryHeaderSupport.formatDictionaryId("\\\"")); + } + + @Test + void testFormatDictionaryIdNull() { + assertThrows(NullPointerException.class, + () -> CompressionDictionaryHeaderSupport.formatDictionaryId(null)); + } +} diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestContentCompressionAsyncExecDictionary.java b/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestContentCompressionAsyncExecDictionary.java new file mode 100644 index 0000000000..15a585a7f8 --- /dev/null +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestContentCompressionAsyncExecDictionary.java @@ -0,0 +1,343 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.impl.async; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Base64; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.hc.client5.http.HttpRoute; +import org.apache.hc.client5.http.async.AsyncExecCallback; +import org.apache.hc.client5.http.async.AsyncExecChain; +import org.apache.hc.client5.http.async.AsyncExecRuntime; +import org.apache.hc.client5.http.entity.compress.BasicCompressionDictionaryStore; +import org.apache.hc.client5.http.entity.compress.CompressionDictionary; +import org.apache.hc.client5.http.entity.compress.CompressionDictionaryStore; +import org.apache.hc.client5.http.impl.Brotli4jRuntime; +import org.apache.hc.client5.http.impl.ZstdRuntime; +import org.apache.hc.client5.http.protocol.HttpClientContext; +import org.apache.hc.core5.concurrent.CancellableDependency; +import org.apache.hc.core5.http.EntityDetails; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpException; +import org.apache.hc.core5.http.HttpHeaders; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.HttpRequest; +import org.apache.hc.core5.http.HttpResponse; +import org.apache.hc.core5.http.Method; +import org.apache.hc.core5.http.message.BasicHttpRequest; +import org.apache.hc.core5.http.message.BasicHttpResponse; +import org.apache.hc.core5.http.nio.AsyncDataConsumer; +import org.apache.hc.core5.http.nio.AsyncEntityProducer; +import org.apache.hc.core5.http.nio.CapacityChannel; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +class TestContentCompressionAsyncExecDictionary { + + private static final String ORIGIN = "https://example.com"; + private static final String AVAILABLE_DICTIONARY = "Available-Dictionary"; + private static final String DICTIONARY_ID = "Dictionary-ID"; + private static final String USE_AS_DICTIONARY = "Use-As-Dictionary"; + + @Mock + private AsyncExecChain execChain; + @Mock + private AsyncEntityProducer entityProducer; + @Mock + private AsyncExecCallback originalCb; + @Mock + private AsyncExecRuntime execRuntime; + @Mock + private CancellableDependency dependency; + + private HttpClientContext context; + private AsyncExecChain.Scope scope; + private BasicCompressionDictionaryStore store; + private ContentCompressionAsyncExec impl; + + @BeforeEach + void init() { + MockitoAnnotations.openMocks(this); + + final HttpHost target = new HttpHost("https", "example.com", 443); + final HttpRequest originalRequest = new BasicHttpRequest(Method.GET, "/"); + context = HttpClientContext.create(); + + scope = new AsyncExecChain.Scope( + "test", + new HttpRoute(target), + originalRequest, + dependency, + context, + execRuntime, + null, + new AtomicInteger()); + + store = new BasicCompressionDictionaryStore(); + impl = new ContentCompressionAsyncExec((CompressionDictionaryStore) store); + } + + private AsyncExecCallback executeAndCapture(final HttpRequest request) throws Exception { + final ArgumentCaptor cap = ArgumentCaptor.forClass(AsyncExecCallback.class); + doNothing().when(execChain).proceed(eq(request), eq(entityProducer), eq(scope), cap.capture()); + impl.execute(request, entityProducer, scope, execChain, originalCb); + return cap.getValue(); + } + + private static CompressionDictionary freshDictionary(final String match, final String id) { + final Instant now = Instant.now(); + return new CompressionDictionary( + "dictionary-content".getBytes(StandardCharsets.UTF_8), + URI.create(ORIGIN), + match, + id, + now.minusSeconds(60), + now.plusSeconds(3600)); + } + + private static String expectedAvailableDictionary(final CompressionDictionary dictionary) { + return ":" + Base64.getEncoder().encodeToString(dictionary.getSha256()) + ":"; + } + + @Test + void testFreshDictionaryAddsAvailableDictionaryHeader() throws Exception { + final CompressionDictionary dictionary = freshDictionary("/*", "dict-1"); + store.add(dictionary); + + final HttpRequest request = new BasicHttpRequest(Method.GET, URI.create(ORIGIN + "/page.html")); + executeAndCapture(request); + + assertTrue(request.containsHeader(AVAILABLE_DICTIONARY)); + assertEquals(expectedAvailableDictionary(dictionary), + request.getFirstHeader(AVAILABLE_DICTIONARY).getValue()); + } + + @Test + void testFreshDictionaryAddsDictionaryIdHeaderWhenIdPresent() throws Exception { + store.add(freshDictionary("/*", "dict-1")); + + final HttpRequest request = new BasicHttpRequest(Method.GET, URI.create(ORIGIN + "/page.html")); + executeAndCapture(request); + + assertTrue(request.containsHeader(DICTIONARY_ID)); + assertEquals("\"dict-1\"", request.getFirstHeader(DICTIONARY_ID).getValue()); + } + + @Test + void testDictionaryIdHeaderOmittedWhenIdEmpty() throws Exception { + store.add(freshDictionary("/*", "")); + + final HttpRequest request = new BasicHttpRequest(Method.GET, URI.create(ORIGIN + "/page.html")); + executeAndCapture(request); + + assertTrue(request.containsHeader(AVAILABLE_DICTIONARY)); + assertFalse(request.containsHeader(DICTIONARY_ID)); + } + + @Test + void testAcceptEncodingIncludesDictionaryTokensWhenAvailable() throws Exception { + store.add(freshDictionary("/*", "dict-1")); + + final HttpRequest request = new BasicHttpRequest(Method.GET, URI.create(ORIGIN + "/page.html")); + executeAndCapture(request); + + assertTrue(request.containsHeader(HttpHeaders.ACCEPT_ENCODING)); + final String acceptEncoding = request.getFirstHeader(HttpHeaders.ACCEPT_ENCODING).getValue(); + + // dcb / dcz tokens are only offered when the matching native runtime is present + assertEquals(Brotli4jRuntime.available(), tokenPresent(acceptEncoding, "dcb")); + assertEquals(ZstdRuntime.available(), tokenPresent(acceptEncoding, "dcz")); + } + + private static boolean tokenPresent(final String headerValue, final String token) { + for (final String part : headerValue.split(",")) { + if (token.equalsIgnoreCase(part.trim())) { + return true; + } + } + return false; + } + + @Test + void testNoDictionaryWhenRequestAlreadyHasAvailableDictionary() throws Exception { + store.add(freshDictionary("/*", "dict-1")); + + final HttpRequest request = new BasicHttpRequest(Method.GET, URI.create(ORIGIN + "/page.html")); + request.addHeader(AVAILABLE_DICTIONARY, ":preset:"); + executeAndCapture(request); + + // the exec must not add a second Available-Dictionary header nor a Dictionary-ID header + assertEquals(1, request.getHeaders(AVAILABLE_DICTIONARY).length); + assertEquals(":preset:", request.getFirstHeader(AVAILABLE_DICTIONARY).getValue()); + assertFalse(request.containsHeader(DICTIONARY_ID)); + + final String acceptEncoding = request.getFirstHeader(HttpHeaders.ACCEPT_ENCODING).getValue(); + assertFalse(tokenPresent(acceptEncoding, "dcb")); + assertFalse(tokenPresent(acceptEncoding, "dcz")); + } + + @Test + void testNoDictionaryWhenRequestUriNotHttps() throws Exception { + store.add(freshDictionary("/*", "dict-1")); + + final HttpRequest request = new BasicHttpRequest(Method.GET, URI.create("http://example.com/page.html")); + executeAndCapture(request); + + assertFalse(request.containsHeader(AVAILABLE_DICTIONARY)); + assertFalse(request.containsHeader(DICTIONARY_ID)); + + final String acceptEncoding = request.getFirstHeader(HttpHeaders.ACCEPT_ENCODING).getValue(); + assertFalse(tokenPresent(acceptEncoding, "dcb")); + assertFalse(tokenPresent(acceptEncoding, "dcz")); + } + + @Test + void testDcbContentEncodingRejectedWithoutNegotiatedDictionary() throws Exception { + // empty store -> no dictionary negotiated for the request + final HttpRequest request = new BasicHttpRequest(Method.GET, URI.create(ORIGIN + "/page.html")); + final AsyncExecCallback cb = executeAndCapture(request); + + final HttpResponse rsp = new BasicHttpResponse(200, "OK"); + final EntityDetails details = mock(EntityDetails.class); + when(details.getContentEncoding()).thenReturn("dcb"); + when(originalCb.handleResponse(same(rsp), any(EntityDetails.class))) + .thenReturn(mock(AsyncDataConsumer.class)); + + assertThrows(HttpException.class, () -> cb.handleResponse(rsp, details)); + } + + @Test + void testDczContentEncodingRejectedWithoutNegotiatedDictionary() throws Exception { + final HttpRequest request = new BasicHttpRequest(Method.GET, URI.create(ORIGIN + "/page.html")); + final AsyncExecCallback cb = executeAndCapture(request); + + final HttpResponse rsp = new BasicHttpResponse(200, "OK"); + final EntityDetails details = mock(EntityDetails.class); + when(details.getContentEncoding()).thenReturn("dcz"); + when(originalCb.handleResponse(same(rsp), any(EntityDetails.class))) + .thenReturn(mock(AsyncDataConsumer.class)); + + assertThrows(HttpException.class, () -> cb.handleResponse(rsp, details)); + } + + @Test + void testUseAsDictionaryWrapsDownstreamAndCapturesBody() throws Exception { + final HttpRequest request = new BasicHttpRequest(Method.GET, URI.create(ORIGIN + "/page.html")); + final AsyncExecCallback cb = executeAndCapture(request); + + final HttpResponse rsp = new BasicHttpResponse(200, "OK"); + rsp.addHeader(USE_AS_DICTIONARY, "match=\"/page.html\""); + rsp.addHeader(HttpHeaders.CACHE_CONTROL, "max-age=3600"); + + final EntityDetails details = mock(EntityDetails.class); + when(details.getContentEncoding()).thenReturn(null); + + final DrainingConsumer downstream = new DrainingConsumer(); + when(originalCb.handleResponse(same(rsp), any(EntityDetails.class))).thenReturn(downstream); + + final AsyncDataConsumer wrapped = cb.handleResponse(rsp, details); + + assertNotNull(wrapped); + assertTrue(wrapped instanceof DictionaryCapturingAsyncDataConsumer); + + final byte[] body = "hello dictionary".getBytes(StandardCharsets.UTF_8); + wrapped.consume(ByteBuffer.wrap(body)); + wrapped.streamEnd(null); + + assertTrue(downstream.ended); + + final List stored = store.getByOrigin(URI.create(ORIGIN + "/page.html")); + assertEquals(1, stored.size()); + assertEquals("/page.html", stored.get(0).getMatch()); + assertArrayEquals(body, stored.get(0).getContent()); + } + + @Test + void testUseAsDictionaryNotWrappedWithoutFreshnessDirective() throws Exception { + final HttpRequest request = new BasicHttpRequest(Method.GET, URI.create(ORIGIN + "/page.html")); + final AsyncExecCallback cb = executeAndCapture(request); + + final HttpResponse rsp = new BasicHttpResponse(200, "OK"); + // Use-As-Dictionary present but no max-age -> validUntil is null -> no capture wrapping + rsp.addHeader(USE_AS_DICTIONARY, "match=\"/page.html\""); + + final EntityDetails details = mock(EntityDetails.class); + when(details.getContentEncoding()).thenReturn(null); + + final DrainingConsumer downstream = new DrainingConsumer(); + when(originalCb.handleResponse(same(rsp), any(EntityDetails.class))).thenReturn(downstream); + + final AsyncDataConsumer wrapped = cb.handleResponse(rsp, details); + + assertSame(downstream, wrapped); + } + + private static final class DrainingConsumer implements AsyncDataConsumer { + + private boolean ended; + + @Override + public void updateCapacity(final CapacityChannel capacityChannel) { + } + + @Override + public void consume(final ByteBuffer src) { + src.position(src.limit()); + } + + @Override + public void streamEnd(final List trailers) { + ended = true; + } + + @Override + public void releaseResources() { + } + } +} diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestDefaultCompressionDictionaryMatcher.java b/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestDefaultCompressionDictionaryMatcher.java new file mode 100644 index 0000000000..82611fe8dc --- /dev/null +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestDefaultCompressionDictionaryMatcher.java @@ -0,0 +1,280 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.impl.async; + +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.net.URI; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; + +import org.apache.hc.client5.http.entity.compress.CompressionDictionary; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class TestDefaultCompressionDictionaryMatcher { + + private static final Instant NOW = Instant.parse("2026-08-21T12:00:00Z"); + + private DefaultCompressionDictionaryMatcher matcher; + + @BeforeEach + void setUp() { + matcher = new DefaultCompressionDictionaryMatcher(Clock.fixed(NOW, ZoneOffset.UTC)); + } + + private static CompressionDictionary dictionary( + final String source, + final String match, + final Instant storedAt, + final Instant validUntil) { + return new CompressionDictionary( + new byte[] {1, 2, 3}, + URI.create(source), + match, + "id", + storedAt, + validUntil); + } + + /** + * A dictionary rooted at https://example.com, valid for one hour from the fixed clock. + */ + private static CompressionDictionary fresh(final String match) { + return dictionary( + "https://example.com/", + match, + NOW.minus(Duration.ofMinutes(10)), + NOW.plus(Duration.ofHours(1))); + } + + private static Collection list(final CompressionDictionary... dictionaries) { + return Arrays.asList(dictionaries); + } + + @Test + void nullRequestUriThrows() { + assertThrows(NullPointerException.class, + () -> matcher.match(null, Collections.emptyList())); + } + + @Test + void nullDictionariesThrows() { + assertThrows(NullPointerException.class, + () -> matcher.match(URI.create("https://example.com/app"), null)); + } + + @Test + void nonHttpsRequestUriReturnsNull() { + final CompressionDictionary dictionary = fresh("/*"); + assertNull(matcher.match(URI.create("http://example.com/app"), list(dictionary))); + } + + @Test + void httpsSchemeIsCaseInsensitive() { + final CompressionDictionary dictionary = dictionary( + "HTTPS://example.com/", + "/app/*", + NOW.minus(Duration.ofMinutes(1)), + NOW.plus(Duration.ofHours(1))); + assertSame(dictionary, + matcher.match(URI.create("HTTPS://example.com/app/main.js"), list(dictionary))); + } + + @Test + void nonHttpsDictionarySourceIsNotMatched() { + final CompressionDictionary dictionary = dictionary( + "http://example.com/", + "/*", + NOW.minus(Duration.ofMinutes(1)), + NOW.plus(Duration.ofHours(1))); + assertNull(matcher.match(URI.create("https://example.com/app"), list(dictionary))); + } + + @Test + void differentHostIsNotMatched() { + final CompressionDictionary dictionary = dictionary( + "https://other.com/", + "/*", + NOW.minus(Duration.ofMinutes(1)), + NOW.plus(Duration.ofHours(1))); + assertNull(matcher.match(URI.create("https://example.com/app"), list(dictionary))); + } + + @Test + void differentPortIsNotMatched() { + final CompressionDictionary dictionary = dictionary( + "https://example.com:8443/", + "/*", + NOW.minus(Duration.ofMinutes(1)), + NOW.plus(Duration.ofHours(1))); + assertNull(matcher.match(URI.create("https://example.com/app"), list(dictionary))); + } + + @Test + void defaultHttpsPortMatchesExplicit443() { + final CompressionDictionary dictionary = dictionary( + "https://example.com/", + "/*", + NOW.minus(Duration.ofMinutes(1)), + NOW.plus(Duration.ofHours(1))); + assertSame(dictionary, + matcher.match(URI.create("https://example.com:443/app"), list(dictionary))); + } + + @Test + void expiredDictionaryIsExcluded() { + final CompressionDictionary dictionary = dictionary( + "https://example.com/", + "/*", + NOW.minus(Duration.ofHours(2)), + NOW.minus(Duration.ofMinutes(1))); + assertNull(matcher.match(URI.create("https://example.com/app"), list(dictionary))); + } + + @Test + void dictionaryExpiringAtTheClockInstantIsExcluded() { + // isFresh uses instant.isBefore(validUntil); validUntil == now is not fresh. + final CompressionDictionary dictionary = dictionary( + "https://example.com/", + "/*", + NOW.minus(Duration.ofHours(2)), + NOW); + assertNull(matcher.match(URI.create("https://example.com/app"), list(dictionary))); + } + + @Test + void patternMustStartWithSlash() { + final CompressionDictionary dictionary = fresh("app/*"); + assertNull(matcher.match(URI.create("https://example.com/app/x"), list(dictionary))); + } + + @Test + void patternWithQueryCharNeverMatches() { + final CompressionDictionary dictionary = fresh("/app?x"); + assertNull(matcher.match(URI.create("https://example.com/app?x"), list(dictionary))); + } + + @Test + void patternWithFragmentCharNeverMatches() { + final CompressionDictionary dictionary = fresh("/app#x"); + assertNull(matcher.match(URI.create("https://example.com/app"), list(dictionary))); + } + + @Test + void patternWithParenthesesNeverMatches() { + assertNull(matcher.match(URI.create("https://example.com/app"), list(fresh("/app(")))); + assertNull(matcher.match(URI.create("https://example.com/app"), list(fresh("/app)")))); + } + + @Test + void patternWithBracesNeverMatches() { + assertNull(matcher.match(URI.create("https://example.com/app"), list(fresh("/app{")))); + assertNull(matcher.match(URI.create("https://example.com/app"), list(fresh("/app}")))); + } + + @Test + void patternWithColonNeverMatches() { + final CompressionDictionary dictionary = fresh("/app:x"); + assertNull(matcher.match(URI.create("https://example.com/app"), list(dictionary))); + } + + @Test + void globPrefixMatchesSubPath() { + final CompressionDictionary dictionary = fresh("/app/*"); + assertSame(dictionary, + matcher.match(URI.create("https://example.com/app/main.js"), list(dictionary))); + } + + @Test + void globPrefixDoesNotMatchDifferentPrefix() { + final CompressionDictionary dictionary = fresh("/app/*"); + assertNull(matcher.match(URI.create("https://example.com/static/main.js"), list(dictionary))); + } + + @Test + void rootGlobMatchesAnyPath() { + final CompressionDictionary dictionary = fresh("/*"); + assertSame(dictionary, + matcher.match(URI.create("https://example.com/anything/deep/path"), list(dictionary))); + } + + @Test + void exactPatternMatchesExactPath() { + final CompressionDictionary dictionary = fresh("/app/main.js"); + assertSame(dictionary, + matcher.match(URI.create("https://example.com/app/main.js"), list(dictionary))); + } + + @Test + void exactPatternDoesNotMatchLongerPath() { + final CompressionDictionary dictionary = fresh("/app/main.js"); + assertNull(matcher.match(URI.create("https://example.com/app/main.js.map"), list(dictionary))); + } + + @Test + void emptyCollectionReturnsNull() { + assertNull(matcher.match(URI.create("https://example.com/app"), + Collections.emptyList())); + } + + @Test + void longestMatchWins() { + final CompressionDictionary broad = fresh("/*"); + final CompressionDictionary narrow = fresh("/app/*"); + assertSame(narrow, + matcher.match(URI.create("https://example.com/app/main.js"), list(broad, narrow))); + // order independent + assertSame(narrow, + matcher.match(URI.create("https://example.com/app/main.js"), list(narrow, broad))); + } + + @Test + void equalLengthLaterStoredAtWins() { + final CompressionDictionary older = dictionary( + "https://example.com/", + "/app/*", + NOW.minus(Duration.ofHours(2)), + NOW.plus(Duration.ofHours(1))); + final CompressionDictionary newer = dictionary( + "https://example.com/", + "/app/*", + NOW.minus(Duration.ofMinutes(5)), + NOW.plus(Duration.ofHours(1))); + assertSame(newer, + matcher.match(URI.create("https://example.com/app/x"), list(older, newer))); + assertSame(newer, + matcher.match(URI.create("https://example.com/app/x"), list(newer, older))); + } +} diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestDictionaryCapturingAsyncDataConsumer.java b/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestDictionaryCapturingAsyncDataConsumer.java new file mode 100644 index 0000000000..205d336105 --- /dev/null +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestDictionaryCapturingAsyncDataConsumer.java @@ -0,0 +1,266 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.impl.async; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.hc.client5.http.entity.compress.BasicCompressionDictionaryStore; +import org.apache.hc.client5.http.entity.compress.CompressionDictionary; +import org.apache.hc.client5.http.entity.compress.CompressionDictionaryStore; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpException; +import org.apache.hc.core5.http.nio.AsyncDataConsumer; +import org.apache.hc.core5.http.nio.CapacityChannel; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class TestDictionaryCapturingAsyncDataConsumer { + + private static final URI SOURCE = URI.create("https://example.com/resource"); + private static final Instant STORED_AT = Instant.parse("2026-08-21T10:00:00Z"); + private static final Instant VALID_UNTIL = Instant.parse("2026-08-22T10:00:00Z"); + + private RecordingDataConsumer downstream; + private CompressionDictionaryStore store; + + @BeforeEach + void setUp() { + downstream = new RecordingDataConsumer(); + store = new BasicCompressionDictionaryStore(); + } + + private static UseAsDictionary rawDirective() throws Exception { + return UseAsDictionary.parse("match=\"/resource\", id=\"dict-1\""); + } + + private static UseAsDictionary unsupportedDirective() throws Exception { + return UseAsDictionary.parse("match=\"/resource\", id=\"dict-1\", type=other"); + } + + private DictionaryCapturingAsyncDataConsumer consumer(final int maxSize) throws Exception { + return new DictionaryCapturingAsyncDataConsumer( + downstream, store, SOURCE, rawDirective(), STORED_AT, VALID_UNTIL, maxSize); + } + + @Test + void testConstructorRejectsNullDownstream() throws Exception { + final UseAsDictionary directive = rawDirective(); + assertThrows(NullPointerException.class, () -> new DictionaryCapturingAsyncDataConsumer( + null, store, SOURCE, directive, STORED_AT, VALID_UNTIL, 1024)); + } + + @Test + void testConstructorRejectsNullStore() throws Exception { + final UseAsDictionary directive = rawDirective(); + assertThrows(NullPointerException.class, () -> new DictionaryCapturingAsyncDataConsumer( + downstream, null, SOURCE, directive, STORED_AT, VALID_UNTIL, 1024)); + } + + @Test + void testConstructorRejectsNullSource() throws Exception { + final UseAsDictionary directive = rawDirective(); + assertThrows(NullPointerException.class, () -> new DictionaryCapturingAsyncDataConsumer( + downstream, store, null, directive, STORED_AT, VALID_UNTIL, 1024)); + } + + @Test + void testConstructorRejectsNullDirective() { + assertThrows(NullPointerException.class, () -> new DictionaryCapturingAsyncDataConsumer( + downstream, store, SOURCE, null, STORED_AT, VALID_UNTIL, 1024)); + } + + @Test + void testConstructorRejectsNullStoredAt() throws Exception { + final UseAsDictionary directive = rawDirective(); + assertThrows(NullPointerException.class, () -> new DictionaryCapturingAsyncDataConsumer( + downstream, store, SOURCE, directive, null, VALID_UNTIL, 1024)); + } + + @Test + void testConstructorRejectsNullValidUntil() throws Exception { + final UseAsDictionary directive = rawDirective(); + assertThrows(NullPointerException.class, () -> new DictionaryCapturingAsyncDataConsumer( + downstream, store, SOURCE, directive, STORED_AT, null, 1024)); + } + + @Test + void testConstructorRejectsNonPositiveMaxSize() throws Exception { + final UseAsDictionary directive = rawDirective(); + assertThrows(IllegalArgumentException.class, () -> new DictionaryCapturingAsyncDataConsumer( + downstream, store, SOURCE, directive, STORED_AT, VALID_UNTIL, 0)); + assertThrows(IllegalArgumentException.class, () -> new DictionaryCapturingAsyncDataConsumer( + downstream, store, SOURCE, directive, STORED_AT, VALID_UNTIL, -1)); + } + + @Test + void testUpdateCapacityDelegates() throws Exception { + final AsyncDataConsumer spy = mock(AsyncDataConsumer.class); + final DictionaryCapturingAsyncDataConsumer mocked = new DictionaryCapturingAsyncDataConsumer( + spy, store, SOURCE, rawDirective(), STORED_AT, VALID_UNTIL, 1024); + final CapacityChannel channel = mock(CapacityChannel.class); + mocked.updateCapacity(channel); + verify(spy).updateCapacity(channel); + } + + @Test + void testConsumeDelegatesAndCaptures() throws Exception { + final DictionaryCapturingAsyncDataConsumer c = consumer(1024); + final byte[] payload = "hello world".getBytes(StandardCharsets.UTF_8); + final ByteBuffer src = ByteBuffer.wrap(payload); + + c.consume(src); + + assertArrayEquals(payload, downstream.consumedBytes()); + assertEquals(0, src.remaining()); + } + + @Test + void testStreamEndStoresDictionaryMatchingInputs() throws Exception { + final DictionaryCapturingAsyncDataConsumer c = consumer(1024); + final byte[] part1 = "abc".getBytes(StandardCharsets.UTF_8); + final byte[] part2 = "defgh".getBytes(StandardCharsets.UTF_8); + + c.consume(ByteBuffer.wrap(part1)); + c.consume(ByteBuffer.wrap(part2)); + c.streamEnd(Collections.
emptyList()); + + assertTrue(downstream.streamEnded()); + + final List stored = store.getByOrigin(SOURCE); + assertEquals(1, stored.size()); + final CompressionDictionary dictionary = stored.get(0); + assertArrayEquals("abcdefgh".getBytes(StandardCharsets.UTF_8), dictionary.getContent()); + assertEquals("/resource", dictionary.getMatch()); + assertEquals("dict-1", dictionary.getId()); + assertEquals(SOURCE, dictionary.getSource()); + assertEquals(STORED_AT, dictionary.getStoredAt()); + assertEquals(VALID_UNTIL, dictionary.getValidUntil()); + } + + @Test + void testExceedingMaxSizeDiscardsButDelegatesAllData() throws Exception { + final DictionaryCapturingAsyncDataConsumer c = consumer(4); + final byte[] part1 = "abc".getBytes(StandardCharsets.UTF_8); + final byte[] part2 = "defgh".getBytes(StandardCharsets.UTF_8); + + c.consume(ByteBuffer.wrap(part1)); + c.consume(ByteBuffer.wrap(part2)); + c.streamEnd(Collections.
emptyList()); + + assertTrue(store.getByOrigin(SOURCE).isEmpty()); + assertArrayEquals("abcdefgh".getBytes(StandardCharsets.UTF_8), downstream.consumedBytes()); + assertTrue(downstream.streamEnded()); + } + + @Test + void testUnsupportedDirectiveStoresNothing() throws Exception { + final DictionaryCapturingAsyncDataConsumer c = new DictionaryCapturingAsyncDataConsumer( + downstream, store, SOURCE, unsupportedDirective(), STORED_AT, VALID_UNTIL, 1024); + final byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); + + c.consume(ByteBuffer.wrap(payload)); + c.streamEnd(Collections.
emptyList()); + + assertTrue(store.getByOrigin(SOURCE).isEmpty()); + assertArrayEquals(payload, downstream.consumedBytes()); + assertTrue(downstream.streamEnded()); + } + + @Test + void testReleaseResourcesDelegatesAndPreventsStore() throws Exception { + final DictionaryCapturingAsyncDataConsumer c = consumer(1024); + final byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); + + c.consume(ByteBuffer.wrap(payload)); + c.releaseResources(); + c.streamEnd(Collections.
emptyList()); + + assertTrue(downstream.released()); + assertTrue(store.getByOrigin(SOURCE).isEmpty()); + } + + /** + * Fake downstream that drains (advances the position of) the buffer it is given, so the + * wrapper's {@code src.position()} delta reflects the bytes actually consumed. + */ + private static final class RecordingDataConsumer implements AsyncDataConsumer { + + private final ByteArrayOutputStream recorded = new ByteArrayOutputStream(); + private final List capacityChannels = new ArrayList<>(); + private boolean streamEnded; + private boolean released; + + @Override + public void updateCapacity(final CapacityChannel capacityChannel) throws IOException { + capacityChannels.add(capacityChannel); + } + + @Override + public void consume(final ByteBuffer src) throws IOException { + while (src.hasRemaining()) { + recorded.write(src.get()); + } + } + + @Override + public void streamEnd(final List trailers) throws HttpException, IOException { + streamEnded = true; + } + + @Override + public void releaseResources() { + released = true; + } + + byte[] consumedBytes() { + return recorded.toByteArray(); + } + + boolean streamEnded() { + return streamEnded; + } + + boolean released() { + return released; + } + } +} diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestUseAsDictionary.java b/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestUseAsDictionary.java new file mode 100644 index 0000000000..1b629361f4 --- /dev/null +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestUseAsDictionary.java @@ -0,0 +1,210 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.impl.async; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.hc.core5.http.ParseException; +import org.junit.jupiter.api.Test; + +class TestUseAsDictionary { + + private static String repeat(final char ch, final int count) { + final StringBuilder builder = new StringBuilder(count); + for (int i = 0; i < count; i++) { + builder.append(ch); + } + return builder.toString(); + } + + @Test + void parseWellFormedValue() throws ParseException { + final UseAsDictionary dictionary = UseAsDictionary.parse("match=\"/app/*\", id=\"v1\", type=raw"); + assertEquals("/app/*", dictionary.getMatch()); + assertEquals("v1", dictionary.getId()); + assertTrue(dictionary.isSupported()); + } + + @Test + void defaultTypeIsRawWhenAbsent() throws ParseException { + final UseAsDictionary dictionary = UseAsDictionary.parse("match=\"/app/*\", id=\"v1\""); + assertEquals("/app/*", dictionary.getMatch()); + assertEquals("v1", dictionary.getId()); + assertTrue(dictionary.isSupported()); + } + + @Test + void typeOtherIsNotSupported() throws ParseException { + final UseAsDictionary dictionary = UseAsDictionary.parse("match=\"/app/*\", type=other"); + assertEquals("/app/*", dictionary.getMatch()); + assertFalse(dictionary.isSupported()); + } + + @Test + void typeRawIsCaseInsensitive() throws ParseException { + final UseAsDictionary dictionary = UseAsDictionary.parse("match=\"/app/*\", type=RAW"); + assertTrue(dictionary.isSupported()); + } + + @Test + void idDefaultsToEmpty() throws ParseException { + final UseAsDictionary dictionary = UseAsDictionary.parse("match=\"/app/*\""); + assertEquals("", dictionary.getId()); + assertTrue(dictionary.isSupported()); + } + + @Test + void missingMatchThrows() { + assertThrows(ParseException.class, () -> UseAsDictionary.parse("id=\"v1\", type=raw")); + } + + @Test + void emptyMatchStringThrows() { + assertThrows(ParseException.class, () -> UseAsDictionary.parse("match=\"\"")); + } + + @Test + void nullValueThrows() { + assertThrows(NullPointerException.class, () -> UseAsDictionary.parse(null)); + } + + @Test + void blankValueThrows() { + assertThrows(IllegalArgumentException.class, () -> UseAsDictionary.parse(" ")); + } + + @Test + void emptyValueThrows() { + assertThrows(IllegalArgumentException.class, () -> UseAsDictionary.parse("")); + } + + @Test + void idExactly1024CharactersIsAccepted() throws ParseException { + final String id = repeat('a', 1024); + final UseAsDictionary dictionary = UseAsDictionary.parse("match=\"/x\", id=\"" + id + "\""); + assertEquals(1024, dictionary.getId().length()); + } + + @Test + void idExceeding1024CharactersThrows() { + final String id = repeat('a', 1025); + assertThrows(ParseException.class, () -> UseAsDictionary.parse("match=\"/x\", id=\"" + id + "\"")); + } + + @Test + void escapedQuoteInsideStringIsUnescaped() throws ParseException { + final UseAsDictionary dictionary = UseAsDictionary.parse("match=\"a\\\"b\""); + assertEquals("a\"b", dictionary.getMatch()); + } + + @Test + void escapedBackslashInsideStringIsUnescaped() throws ParseException { + final UseAsDictionary dictionary = UseAsDictionary.parse("match=\"a\\\\b\""); + assertEquals("a\\b", dictionary.getMatch()); + } + + @Test + void invalidEscapeSequenceThrows() { + assertThrows(ParseException.class, () -> UseAsDictionary.parse("match=\"a\\b\"")); + } + + @Test + void controlCharacterInsideStringThrows() { + assertThrows(ParseException.class, () -> UseAsDictionary.parse("match=\"a\tb\"")); + } + + @Test + void nonAsciiCharacterInsideStringThrows() { + final String value = "match=\"" + 'é' + "\""; + assertThrows(ParseException.class, () -> UseAsDictionary.parse(value)); + } + + @Test + void unquotedMatchThrows() { + assertThrows(ParseException.class, () -> UseAsDictionary.parse("match=raw")); + } + + @Test + void unbalancedQuoteThrows() { + assertThrows(ParseException.class, () -> UseAsDictionary.parse("match=\"abc")); + } + + @Test + void unbalancedOpenParenThrows() { + assertThrows(ParseException.class, () -> UseAsDictionary.parse("match=\"/x\", dummy=(a")); + } + + @Test + void unbalancedCloseParenThrows() { + assertThrows(ParseException.class, () -> UseAsDictionary.parse("match=\"/x\", dummy=a)")); + } + + @Test + void commaInsideQuotesIsNotSeparator() throws ParseException { + final UseAsDictionary dictionary = UseAsDictionary.parse("match=\"a,b\", id=\"c\""); + assertEquals("a,b", dictionary.getMatch()); + assertEquals("c", dictionary.getId()); + } + + @Test + void commaInsideParensIsNotSeparator() throws ParseException { + final UseAsDictionary dictionary = UseAsDictionary.parse("dummy=(a, b), match=\"/x\", type=raw"); + assertEquals("/x", dictionary.getMatch()); + assertTrue(dictionary.isSupported()); + } + + @Test + void memberWithoutEqualsIsIgnored() throws ParseException { + final UseAsDictionary dictionary = UseAsDictionary.parse("foo, match=\"/x\""); + assertEquals("/x", dictionary.getMatch()); + } + + @Test + void parametersAfterSemicolonAreIgnored() throws ParseException { + final UseAsDictionary dictionary = UseAsDictionary.parse("match=\"/x\", type=raw;foo=bar"); + assertTrue(dictionary.isSupported()); + } + + @Test + void parametersAfterSemicolonAreIgnoredForUnsupportedType() throws ParseException { + final UseAsDictionary dictionary = UseAsDictionary.parse("match=\"/x\", type=other;foo=bar"); + assertFalse(dictionary.isSupported()); + } + + @Test + void emptyTypeTokenThrows() { + assertThrows(ParseException.class, () -> UseAsDictionary.parse("match=\"/x\", type=")); + } + + @Test + void invalidTypeTokenCharacterThrows() { + assertThrows(ParseException.class, () -> UseAsDictionary.parse("match=\"/x\", type=a b")); + } +} From 28b875a3d751141bd0e7c3a98e7997d9707bdeba Mon Sep 17 00:00:00 2001 From: Arturo Bernal Date: Sat, 22 Aug 2026 11:31:30 +0200 Subject: [PATCH 3/3] update logic --- ...InflatingDictionaryBrotliDataConsumer.java | 97 +- .../InflatingDictionaryZstdDataConsumer.java | 54 +- .../BasicCompressionDictionaryStore.java | 159 +++- .../compress/CompressionDictionary.java | 203 +++- .../compress/CompressionDictionaryStore.java | 67 +- .../CompressionDictionaryCookieStore.java | 94 ++ .../async/CompressionDictionaryFreshness.java | 263 ++++-- .../CompressionDictionaryHeaderSupport.java | 71 +- .../async/CompressionDictionaryMatcher.java | 15 + ...ompressionDictionaryUrlPatternMatcher.java | 879 +++++++++++++++++- .../async/ContentCompressionAsyncExec.java | 160 ++-- .../DefaultCompressionDictionaryMatcher.java | 95 +- .../DictionaryCapturingAsyncDataConsumer.java | 51 +- .../impl/async/HttpAsyncClientBuilder.java | 10 +- .../http/impl/async/UseAsDictionary.java | 616 +++++++++++- ...InflatingDictionaryBrotliDataConsumer.java | 176 +++- ...stInflatingDictionaryZstdDataConsumer.java | 35 +- .../TestBasicCompressionDictionaryStore.java | 128 +-- .../compress/TestCompressionDictionary.java | 38 + .../AsyncClientCompressionDictionary.java | 15 +- .../AsyncClientServerDictionaryBrotli.java | 13 +- .../TestCompressionDictionaryCookieStore.java | 69 ++ .../TestCompressionDictionaryFreshness.java | 120 +++ ...ContentCompressionAsyncExecDictionary.java | 103 +- ...stDefaultCompressionDictionaryMatcher.java | 180 +++- ...tDictionaryCapturingAsyncDataConsumer.java | 55 +- .../http/impl/async/TestUseAsDictionary.java | 62 +- 27 files changed, 3350 insertions(+), 478 deletions(-) create mode 100644 httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryCookieStore.java create mode 100644 httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestCompressionDictionaryCookieStore.java create mode 100644 httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestCompressionDictionaryFreshness.java diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/InflatingDictionaryBrotliDataConsumer.java b/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/InflatingDictionaryBrotliDataConsumer.java index e79de8733c..0fb9fc8155 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/InflatingDictionaryBrotliDataConsumer.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/InflatingDictionaryBrotliDataConsumer.java @@ -33,7 +33,6 @@ import com.aayushatharva.brotli4j.decoder.DecoderJNI; import org.apache.hc.client5.http.entity.compress.CompressionDictionary; -import org.apache.hc.client5.http.entity.compress.CompressionDictionaryStore; import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpException; import org.apache.hc.core5.http.nio.AsyncDataConsumer; @@ -46,10 +45,11 @@ *

* A {@code dcb} stream begins with a fixed header: the four-byte magic sequence * {@code 0xFF 0x44 0x43 0x42} followed by the 32-byte SHA-256 hash of the dictionary the origin - * used to compress the body. The header is buffered until complete, the hash is looked up in the - * supplied {@link CompressionDictionaryStore}, and the matching {@link CompressionDictionary} is - * attached to the Brotli decoder as the shared dictionary before any compressed payload is - * decoded. If the hash is unknown, or the header does not validate, the stream is rejected. See + * used to compress the body. The header is buffered until complete and checked against the + * supplied {@link CompressionDictionary}, which is attached to the Brotli decoder as the shared + * dictionary before any compressed payload is + * decoded. The header hash must identify the exact dictionary negotiated for this exchange; + * otherwise the stream is rejected. See * the Compression Dictionary Transport specification for the dictionary negotiation and framing. *

* This consumer is stateful and not thread-safe; a fresh instance is required per response. @@ -66,25 +66,25 @@ public final class InflatingDictionaryBrotliDataConsumer implements AsyncDataCon private static final int HEADER_LENGTH = MAGIC.length + HASH_LENGTH; private final AsyncDataConsumer downstream; - private final CompressionDictionaryStore store; + private final CompressionDictionary dictionary; private final ByteBuffer header; private DecoderJNI.Wrapper decoder; + private ByteBuffer pendingOutput; /** * Creates a consumer that decodes a {@code dcb} stream and forwards the decoded bytes to * {@code downstream}. * * @param downstream the consumer that receives the decompressed content; must not be {@code null}. - * @param store the store consulted to resolve the dictionary referenced by the stream header; - * must not be {@code null}. + * @param dictionary the exact dictionary advertised for this exchange; must not be {@code null}. * @since 5.7 */ public InflatingDictionaryBrotliDataConsumer( final AsyncDataConsumer downstream, - final CompressionDictionaryStore store) { + final CompressionDictionary dictionary) { this.downstream = Args.notNull(downstream, "Downstream data consumer"); - this.store = Args.notNull(store, "Dictionary store"); + this.dictionary = Args.notNull(dictionary, "Compression dictionary"); this.header = ByteBuffer.allocate(HEADER_LENGTH); } @@ -120,6 +120,9 @@ public void consume(final ByteBuffer src) throws IOException { return; } } + if (!drainPendingOutput()) { + return; + } while (src.hasRemaining()) { if (decoder.getStatus() == DecoderJNI.Status.DONE) { @@ -162,9 +165,8 @@ private void consumeHeader(final ByteBuffer src) throws IOException { final byte[] hash = new byte[HASH_LENGTH]; header.get(hash); - final CompressionDictionary dictionary = store.getByHash(hash); - if (dictionary == null || !dictionary.matchesHash(hash)) { - throw new IOException("Compression dictionary not available"); + if (!dictionary.matchesHash(hash)) { + throw new IOException("DCB stream does not use the negotiated dictionary"); } try { @@ -188,6 +190,9 @@ private void consumeHeader(final ByteBuffer src) throws IOException { } private void pump() throws IOException { + if (!drainPendingOutput()) { + return; + } for (; ; ) { switch (decoder.getStatus()) { case OK: @@ -195,10 +200,8 @@ private void pump() throws IOException { break; case NEEDS_MORE_OUTPUT: { final ByteBuffer nativeBuf = decoder.pull(); - if (nativeBuf != null && nativeBuf.hasRemaining()) { - final ByteBuffer copy = ByteBuffer.allocateDirect(nativeBuf.remaining()); - copy.put(nativeBuf).flip(); - downstream.consume(copy); + if (!deliver(nativeBuf)) { + return; } break; } @@ -206,9 +209,9 @@ private void pump() throws IOException { if (decoder.hasOutput()) { final ByteBuffer nativeBuf = decoder.pull(); if (nativeBuf != null && nativeBuf.hasRemaining()) { - final ByteBuffer copy = ByteBuffer.allocateDirect(nativeBuf.remaining()); - copy.put(nativeBuf).flip(); - downstream.consume(copy); + if (!deliver(nativeBuf)) { + return; + } break; } } @@ -217,9 +220,9 @@ private void pump() throws IOException { if (decoder.hasOutput()) { final ByteBuffer nativeBuf = decoder.pull(); if (nativeBuf != null && nativeBuf.hasRemaining()) { - final ByteBuffer copy = ByteBuffer.allocateDirect(nativeBuf.remaining()); - copy.put(nativeBuf).flip(); - downstream.consume(copy); + if (!deliver(nativeBuf)) { + return; + } break; } } @@ -230,6 +233,38 @@ private void pump() throws IOException { } } + private boolean deliver(final ByteBuffer source) throws IOException { + if (source != null && source.hasRemaining()) { + pendingOutput = ByteBuffer.allocateDirect(source.remaining()); + pendingOutput.put(source).flip(); + } + return drainPendingOutput(); + } + + private boolean drainPendingOutput() throws IOException { + if (pendingOutput != null) { + downstream.consume(pendingOutput); + if (!pendingOutput.hasRemaining()) { + pendingOutput = null; + } + } + return pendingOutput == null; + } + + private void drainDecoderAtEnd() throws IOException { + do { + pump(); + while (pendingOutput != null) { + final int position = pendingOutput.position(); + drainPendingOutput(); + if (pendingOutput != null && pendingOutput.position() == position) { + throw new IOException("Unable to deliver decoded DCB data"); + } + } + } while (decoder.getStatus() == DecoderJNI.Status.NEEDS_MORE_OUTPUT + || decoder.hasOutput()); + } + /** * Finalises decoding at end of stream. The decoder is drained of any pending output and the * completion is signalled to the downstream consumer. A stream that ends before the header is @@ -245,7 +280,16 @@ public void streamEnd(final List trailers) throws IOException, throw new IOException("Truncated DCB stream header"); } - pump(); + drainDecoderAtEnd(); + + if (decoder.getStatus() == DecoderJNI.Status.NEEDS_MORE_INPUT) { + try { + decoder.push(0); + drainDecoderAtEnd(); + } catch (final RuntimeException ex) { + throw new IOException("DCB stream corrupted", ex); + } + } if (decoder.getStatus() != DecoderJNI.Status.DONE) { throw new IOException("Truncated DCB stream"); @@ -267,8 +311,11 @@ public void releaseResources() { try { decoder.destroy(); } catch (final Throwable ignore) { + } finally { + decoder = null; } } + pendingOutput = null; downstream.releaseResources(); } -} \ No newline at end of file +} diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/InflatingDictionaryZstdDataConsumer.java b/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/InflatingDictionaryZstdDataConsumer.java index 08df28d4bf..adf7af8517 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/InflatingDictionaryZstdDataConsumer.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/async/methods/InflatingDictionaryZstdDataConsumer.java @@ -34,7 +34,6 @@ import com.github.luben.zstd.ZstdDecompressCtx; import org.apache.hc.client5.http.entity.compress.CompressionDictionary; -import org.apache.hc.client5.http.entity.compress.CompressionDictionaryStore; import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpException; import org.apache.hc.core5.http.nio.AsyncDataConsumer; @@ -47,8 +46,8 @@ *

* The stream opens with the {@code dcz} framing prefix defined by Compression Dictionary Transport: an eight-byte * magic sequence followed by the 32-byte hash of the dictionary the origin used to compress - * the body. The hash is looked up in the supplied {@link CompressionDictionaryStore} and the - * matching {@link CompressionDictionary} is loaded into the Zstandard decompression context + * the body. The hash is checked against the supplied {@link CompressionDictionary}, which is + * loaded into the Zstandard decompression context * before any payload is decoded. If the prefix is malformed, or no dictionary matching the * advertised hash is available, decoding fails with an {@link IOException} so the exchange is * not silently served undecoded content. @@ -73,7 +72,7 @@ public final class InflatingDictionaryZstdDataConsumer implements AsyncDataConsu private static final int OUT_BUF = 128 * 1024; private final AsyncDataConsumer downstream; - private final CompressionDictionaryStore store; + private final CompressionDictionary dictionary; private final ByteBuffer header; private final ZstdDecompressCtx dctx; private final ByteBuffer inDirect; @@ -87,16 +86,14 @@ public final class InflatingDictionaryZstdDataConsumer implements AsyncDataConsu * Creates a consumer that decodes a {@code dcz} body and forwards the inflated bytes. * * @param downstream the consumer that receives the decoded content; must not be {@code null}. - * @param store the dictionary store queried, by the hash carried in the {@code dcz} - * prefix, for the dictionary required to decode the body; must not be - * {@code null}. + * @param dictionary the exact dictionary advertised for this exchange; must not be {@code null}. * @since 5.7 */ public InflatingDictionaryZstdDataConsumer( final AsyncDataConsumer downstream, - final CompressionDictionaryStore store) { + final CompressionDictionary dictionary) { this.downstream = Args.notNull(downstream, "Downstream data consumer"); - this.store = Args.notNull(store, "Dictionary store"); + this.dictionary = Args.notNull(dictionary, "Compression dictionary"); this.header = ByteBuffer.allocate(HEADER_LENGTH); this.dctx = new ZstdDecompressCtx(); this.inDirect = ByteBuffer.allocateDirect(IN_BUF); @@ -199,9 +196,8 @@ private void consumeHeader(final ByteBuffer src) throws IOException { final byte[] hash = new byte[HASH_LENGTH]; header.get(hash); - final CompressionDictionary dictionary = store.getByHash(hash); - if (dictionary == null || !dictionary.matchesHash(hash)) { - throw new IOException("Compression dictionary not available"); + if (!dictionary.matchesHash(hash)) { + throw new IOException("DCZ stream does not use the negotiated dictionary"); } try { @@ -212,6 +208,36 @@ private void consumeHeader(final ByteBuffer src) throws IOException { } } + private void finishBufferedInput() throws IOException { + try { + drainOutput(); + while (inDirect.hasRemaining()) { + final int inputPosition = inDirect.position(); + outDirect.compact(); + frameComplete = dctx.decompressDirectByteBufferStream(outDirect, inDirect); + outDirect.flip(); + drainOutput(); + if (inDirect.position() == inputPosition) { + throw new IOException("DCZ stream made no progress"); + } + } + } catch (final IOException ex) { + throw ex; + } catch (final RuntimeException ex) { + throw new IOException("DCZ stream corrupted", ex); + } + } + + private void drainOutput() throws IOException { + while (outDirect.hasRemaining()) { + final int position = outDirect.position(); + downstream.consume(outDirect); + if (outDirect.position() == position) { + throw new IOException("Unable to deliver decoded DCZ data"); + } + } + } + /** * Finalises decoding once the origin has signalled end of stream. The header and the final * Zstandard frame must both be complete; an incomplete header or a truncated frame is reported @@ -227,6 +253,8 @@ public void streamEnd(final List trailers) throws HttpExceptio throw new IOException("Truncated DCZ stream header"); } + finishBufferedInput(); + if (!frameComplete) { throw new IOException("Truncated DCZ stream"); } @@ -251,4 +279,4 @@ public void releaseResources() { } downstream.releaseResources(); } -} \ No newline at end of file +} diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/BasicCompressionDictionaryStore.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/BasicCompressionDictionaryStore.java index f422a6a015..4c2ec3a3d0 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/BasicCompressionDictionaryStore.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/BasicCompressionDictionaryStore.java @@ -36,6 +36,7 @@ import java.util.Map; import java.util.concurrent.locks.ReentrantLock; +import org.apache.hc.client5.http.cookie.CookieStore; import org.apache.hc.core5.annotation.Contract; import org.apache.hc.core5.annotation.ThreadingBehavior; import org.apache.hc.core5.util.Args; @@ -43,12 +44,15 @@ /** * Basic in-memory implementation of {@link CompressionDictionaryStore}. *

- * Dictionaries are held in a bounded map keyed by the SHA-256 hash of their content. The store - * retains at most {@code maxEntries} dictionaries; once that bound is exceeded the oldest entries - * are evicted in insertion order until the store is back within capacity. Adding a dictionary whose - * hash already exists replaces the previous entry and moves it to the most-recently-added position, - * so it is evicted last. All mutating and reading operations are guarded by a lock, hence instances - * are safe for concurrent use. + * Dictionaries are held in a bounded map keyed by cookie partition, request origin and the SHA-256 + * hash of the dictionary content, so identical content stored in different privacy partitions or + * against different origins yields distinct entries. The origin is normalised to its scheme, host + * and effective port, {@code 443} being assumed for {@code https} when no explicit port is present. The + * store retains at most {@code maxEntries} dictionaries; once that bound is exceeded the eldest + * entries are evicted in insertion order until the store is back within capacity. Adding a + * dictionary whose key already exists replaces the previous entry and moves it to the + * most-recently-added position, so it is evicted last. All access is guarded by a + * {@link ReentrantLock}, hence instances are safe for concurrent use. * * @since 5.7 */ @@ -58,26 +62,53 @@ public final class BasicCompressionDictionaryStore implements CompressionDiction private static final int DEFAULT_MAX_ENTRIES = 64; private final int maxEntries; - private final Map dictionaries; + private final Map dictionaries; private final ReentrantLock lock; + /** + * Creates a store bounded to the default of {@value #DEFAULT_MAX_ENTRIES} dictionaries. + * + * @since 5.7 + */ public BasicCompressionDictionaryStore() { this(DEFAULT_MAX_ENTRIES); } + /** + * Creates a store bounded to the given number of dictionaries. + * + * @param maxEntries the maximum number of dictionaries to retain; must be positive. + * @throws IllegalArgumentException if {@code maxEntries} is not positive. + * @since 5.7 + */ public BasicCompressionDictionaryStore(final int maxEntries) { this.maxEntries = Args.positive(maxEntries, "Maximum entries"); this.dictionaries = new LinkedHashMap<>(); this.lock = new ReentrantLock(); } + /** + * Stores the dictionary under the compound key formed from the privacy partition, its + * {@link CompressionDictionary#getSource() source} origin and SHA-256 hash. A pre-existing entry + * with the same key is replaced and promoted to the most-recently-added position; the store then + * evicts its eldest entries until it no longer exceeds {@code maxEntries}. + * + * @param partition the cookie storage partition; must not be {@code null}. + * @param dictionary the dictionary to store; must not be {@code null}. + * @throws NullPointerException if {@code partition} or {@code dictionary} is {@code null}. + * @since 5.7 + */ @Override - public void add(final CompressionDictionary dictionary) { + public void add( + final CookieStore partition, + final CompressionDictionary dictionary) { + Args.notNull(partition, "Cookie partition"); Args.notNull(dictionary, "Dictionary"); lock.lock(); try { - final String key = key( + final Key key = key( + partition, dictionary.getSource(), dictionary.getSha256()); @@ -85,7 +116,7 @@ public void add(final CompressionDictionary dictionary) { dictionaries.put(key, dictionary); while (dictionaries.size() > maxEntries) { - final Iterator iterator = + final Iterator iterator = dictionaries.keySet().iterator(); if (iterator.hasNext()) { iterator.next(); @@ -97,23 +128,51 @@ public void add(final CompressionDictionary dictionary) { } } + /** + * Returns the dictionary previously stored for the given origin and SHA-256 hash. This resolves a + * single candidate by the exact hash a response advertises, for example the hash carried in the + * {@code Available-Dictionary} negotiation. + * + * @param partition the cookie storage partition; must not be {@code null}. + * @param origin the request origin the dictionary was stored against; must not be {@code null}. + * @param sha256 the SHA-256 hash of the dictionary content; must not be {@code null}. + * @return the matching dictionary, or {@code null} if none is stored under that key. + * @throws NullPointerException if {@code partition}, {@code origin} or {@code sha256} is {@code null}. + * @since 5.7 + */ @Override public CompressionDictionary getByHash( + final CookieStore partition, final URI origin, final byte[] sha256) { + Args.notNull(partition, "Cookie partition"); Args.notNull(origin, "Origin"); Args.notNull(sha256, "SHA-256"); lock.lock(); try { - return dictionaries.get(key(origin, sha256)); + return dictionaries.get(key(partition, origin, sha256)); } finally { lock.unlock(); } } + /** + * Returns all dictionaries whose source shares the scheme, host and effective port of the given + * origin, in insertion order. This is the set of candidates a client may advertise for an outbound + * request to that origin, before further filtering by match pattern or freshness. + * + * @param partition the cookie storage partition; must not be {@code null}. + * @param origin the request origin to match; must not be {@code null}. + * @return a newly allocated, modifiable list of matching dictionaries, empty if none match. + * @throws NullPointerException if {@code partition} or {@code origin} is {@code null}. + * @since 5.7 + */ @Override - public List getByOrigin(final URI origin) { + public List getByOrigin( + final CookieStore partition, + final URI origin) { + Args.notNull(partition, "Cookie partition"); Args.notNull(origin, "Origin"); lock.lock(); @@ -121,10 +180,11 @@ public List getByOrigin(final URI origin) { final List result = new ArrayList<>(); - for (final CompressionDictionary dictionary - : dictionaries.values()) { - if (sameOrigin(origin, dictionary.getSource())) { - result.add(dictionary); + for (final Map.Entry entry + : dictionaries.entrySet()) { + if (entry.getKey().partition == partition + && sameOrigin(origin, entry.getValue().getSource())) { + result.add(entry.getValue()); } } @@ -134,6 +194,11 @@ public List getByOrigin(final URI origin) { } } + /** + * Removes all stored dictionaries, returning the store to an empty state. + * + * @since 5.7 + */ @Override public void clear() { lock.lock(); @@ -144,12 +209,27 @@ public void clear() { } } - private static String key( + @Override + public void clear(final CookieStore partition) { + Args.notNull(partition, "Cookie partition"); + lock.lock(); + try { + final Iterator iterator = dictionaries.keySet().iterator(); + while (iterator.hasNext()) { + if (iterator.next().partition == partition) { + iterator.remove(); + } + } + } finally { + lock.unlock(); + } + } + + private static Key key( + final CookieStore partition, final URI origin, final byte[] sha256) { - return originKey(origin) - + '|' - + Base64.getEncoder().encodeToString(sha256); + return new Key(partition, originKey(origin), Base64.getEncoder().encodeToString(sha256)); } private static String originKey(final URI uri) { @@ -183,9 +263,40 @@ private static int effectivePort(final URI uri) { if ("https".equalsIgnoreCase(uri.getScheme())) { return 443; } - if ("http".equalsIgnoreCase(uri.getScheme())) { - return 80; - } return -1; } -} \ No newline at end of file + + private static final class Key { + private final CookieStore partition; + private final String origin; + private final String hash; + + Key(final CookieStore partition, final String origin, final String hash) { + this.partition = partition; + this.origin = origin; + this.hash = hash; + } + + @Override + public int hashCode() { + int result = System.identityHashCode(partition); + result = 31 * result + origin.hashCode(); + result = 31 * result + hash.hashCode(); + return result; + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof Key)) { + return false; + } + final Key that = (Key) obj; + return partition == that.partition + && origin.equals(that.origin) + && hash.equals(that.hash); + } + } +} diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CompressionDictionary.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CompressionDictionary.java index dcbd64c03c..87360e9010 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CompressionDictionary.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CompressionDictionary.java @@ -30,7 +30,7 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.Instant; -import java.util.Arrays; +import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -45,8 +45,8 @@ * Dictionary-Compressed Zstandard ({@code dcz}) decoder, together with the metadata needed to * negotiate its use with an origin. The {@code match} pattern advertised in the origin's * {@code Use-As-Dictionary} response selects the requests a dictionary applies to; the SHA-256 - * hash is offered in the {@code Available-Dictionary} request header and echoed by the origin in - * {@code Dictionary-ID}, so that both peers agree on which dictionary decoded the response. + * hash is offered in the {@code Available-Dictionary} request header. The optional server supplied + * identifier is echoed independently in the {@code Dictionary-ID} request header. *

* Instances are value objects: the dictionary content is defensively copied on construction and * on every access, and the SHA-256 hash is computed once from that copy, so the observable state @@ -67,6 +67,48 @@ public final class CompressionDictionary { private final Instant storedAt; private final Instant validUntil; + /** + * Creates a dictionary with no {@code match-dest} restriction and the default {@code raw} format. + * + * @param content the raw dictionary bytes; copied defensively. + * @param source the URI the dictionary was fetched from. + * @param match the {@code match} URL pattern that selects the requests the dictionary applies to. + * @param id the opaque server identifier echoed in the {@code Dictionary-ID} request header, + * or {@code null} for none. + * @param storedAt the instant the dictionary was stored. + * @param validUntil the instant at which the dictionary stops being fresh; freshness is a half-open + * window ending strictly before this instant. + * @since 5.7 + */ + public CompressionDictionary( + final byte[] content, + final URI source, + final String match, + final String id, + final Instant storedAt, + final Instant validUntil) { + this(content, source, match, Collections.emptyList(), id, "raw", storedAt, validUntil); + } + + /** + * Creates a dictionary with the full set of negotiation metadata. + * + * @param content the raw dictionary bytes; copied defensively. + * @param source the URI the dictionary was fetched from. + * @param match the {@code match} URL pattern that selects the requests the dictionary applies to. + * @param matchDest the optional {@code match-dest} destinations that further constrain which requests + * the dictionary applies to; copied defensively, {@code null} is treated as empty. + * @param id the opaque server identifier echoed in the {@code Dictionary-ID} request header, + * or {@code null} for none. + * @param type the format token; only {@code raw} is honoured, {@code null} defaults to {@code raw}. + * @param storedAt the instant the dictionary was stored. + * @param validUntil the instant at which the dictionary stops being fresh. + * @throws NullPointerException if {@code content}, {@code source}, {@code match}, {@code storedAt} + * or {@code validUntil} is {@code null}. + * @throws IllegalArgumentException if {@code match} is blank or a {@code matchDest} + * element is {@code null}. + * @since 5.7 + */ public CompressionDictionary( final byte[] content, final URI source, @@ -77,60 +119,134 @@ public CompressionDictionary( final Instant storedAt, final Instant validUntil) { this.content = Args.notNull(content, "Content").clone(); - this.sha256 = sha256(content); - this.source = Args.notNull(source, "Source"); - this.match = Args.notBlank(match, "Match"); - this.matchDest = matchDest != null - ? List.copyOf(matchDest) - : Collections.emptyList(); - this.id = id != null ? id : ""; - this.type = type != null ? type : "raw"; + this.sha256 = sha256(this.content); + this.source = validateSource(Args.notNull(source, "Source")); + this.match = validateString(Args.notBlank(match, "Match"), "Match", Integer.MAX_VALUE); + this.matchDest = copyMatchDest(matchDest); + this.id = validateString(id != null ? id : "", "ID", 1024); + this.type = validateToken(type != null ? type : "raw"); this.storedAt = Args.notNull(storedAt, "Stored at"); this.validUntil = Args.notNull(validUntil, "Valid until"); } + /** + * Returns a defensive copy of the raw dictionary bytes that seed the {@code dcb} or {@code dcz} + * decoder. + * + * @return a fresh copy of the dictionary content. + * @since 5.7 + */ public byte[] getContent() { return content.clone(); } + /** + * Returns a defensive copy of the SHA-256 digest of the dictionary content. The digest is offered + * to the origin in the {@code Available-Dictionary} request header. + * + * @return a fresh copy of the SHA-256 digest. + * @since 5.7 + */ public byte[] getSha256() { return sha256.clone(); } + /** + * Returns the URI the dictionary was fetched from. + * + * @return the source URI. + * @since 5.7 + */ public URI getSource() { return source; } + /** + * Returns the {@code match} URL pattern advertised in the origin's {@code Use-As-Dictionary} + * response that selects the requests this dictionary applies to. + * + * @return the match pattern. + * @since 5.7 + */ public String getMatch() { return match; } + /** + * Returns the optional {@code match-dest} destinations that further constrain which requests the + * dictionary applies to. The list is unmodifiable and empty when no destinations were advertised. + * + * @return the immutable list of match destinations. + * @since 5.7 + */ public List getMatchDest() { return matchDest; } + /** + * Returns the opaque server identifier echoed back in the {@code Dictionary-ID} request header, or + * an empty string when the origin supplied none. + * + * @return the dictionary identifier, never {@code null}. + * @since 5.7 + */ public String getId() { return id; } + /** + * Returns the format token declared for this dictionary. Only {@code raw} is honoured. + * + * @return the format type token. + * @since 5.7 + */ public String getType() { return type; } + /** + * Returns the instant the dictionary was stored. + * + * @return the storage instant. + * @since 5.7 + */ public Instant getStoredAt() { return storedAt; } + /** + * Returns the instant at which the dictionary stops being fresh. Freshness ends strictly before + * this instant. + * + * @return the freshness boundary. + * @since 5.7 + */ public Instant getValidUntil() { return validUntil; } + /** + * Tests whether the dictionary is still fresh at the given instant. The check is half-open: the + * dictionary is fresh strictly before {@link #getValidUntil()} and stale from that instant onward. + * + * @param now the instant to test against. + * @return {@code true} if {@code now} precedes the freshness boundary. + * @since 5.7 + */ public boolean isFresh(final Instant now) { return now.isBefore(validUntil); } + /** + * Tests whether the supplied hash equals the stored SHA-256 digest. The comparison uses + * {@link MessageDigest#isEqual(byte[], byte[])} to avoid leaking timing information. + * + * @param hash the hash to compare, may be {@code null}. + * @return {@code true} if {@code hash} is non-{@code null} and matches the stored digest. + * @since 5.7 + */ public boolean matchesHash(final byte[] hash) { - return MessageDigest.isEqual(sha256, hash); + return hash != null && MessageDigest.isEqual(sha256, hash); } private static byte[] sha256(final byte[] content) { @@ -140,4 +256,65 @@ private static byte[] sha256(final byte[] content) { throw new IllegalStateException(ex); } } -} \ No newline at end of file + + private static URI validateSource(final URI source) { + if (!source.isAbsolute() + || source.getHost() == null + || !"https".equalsIgnoreCase(source.getScheme())) { + throw new IllegalArgumentException("Source must be an absolute HTTPS URI"); + } + return source; + } + + private static List copyMatchDest(final List values) { + if (values == null || values.isEmpty()) { + return Collections.emptyList(); + } + final List copy = new ArrayList<>(values.size()); + for (final String value : values) { + if (value == null) { + throw new IllegalArgumentException("Match destination must not be null"); + } + copy.add(validateString(value, "Match destination", Integer.MAX_VALUE)); + } + return Collections.unmodifiableList(copy); + } + + private static String validateString( + final String value, + final String name, + final int maxLength) { + if (value.length() > maxLength) { + throw new IllegalArgumentException(name + " exceeds " + maxLength + " characters"); + } + for (int i = 0; i < value.length(); i++) { + final char ch = value.charAt(i); + if (ch < 0x20 || ch > 0x7e) { + throw new IllegalArgumentException(name + " is not an RFC 9651 String"); + } + } + return value; + } + + private static String validateToken(final String value) { + if (value.isEmpty() || !isTokenStart(value.charAt(0))) { + throw new IllegalArgumentException("Type is not an RFC 9651 Token"); + } + for (int i = 1; i < value.length(); i++) { + if (!isTokenChar(value.charAt(i))) { + throw new IllegalArgumentException("Type is not an RFC 9651 Token"); + } + } + return value; + } + + private static boolean isTokenStart(final char ch) { + return ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z' || ch == '*'; + } + + private static boolean isTokenChar(final char ch) { + return isTokenStart(ch) + || ch >= '0' && ch <= '9' + || "!#$%&'*+-.^_`|~:/".indexOf(ch) >= 0; + } +} diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CompressionDictionaryStore.java b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CompressionDictionaryStore.java index 13e7333191..1057b2240b 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CompressionDictionaryStore.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/entity/compress/CompressionDictionaryStore.java @@ -29,16 +29,21 @@ import java.net.URI; import java.util.List; +import org.apache.hc.client5.http.cookie.CookieStore; + /** * Repository of {@link CompressionDictionary} instances available to the client for * decoding dictionary-compressed Brotli ({@code dcb}) and dictionary-compressed * Zstandard ({@code dcz}) responses under Compression Dictionary Transport. *

- * Dictionaries are indexed both by their SHA-256 hash, which the origin echoes back in - * the {@code Dictionary-ID} header to identify the dictionary a response was encoded - * against, and by request origin, so that the client can select the candidates whose - * match pattern applies to an outbound request and advertise them through the - * {@code Available-Dictionary} header. + * Entries are keyed by privacy partition, request origin and the SHA-256 hash of their content. + * The privacy partition is represented by the {@link CookieStore} used for the exchange, ensuring + * dictionary state is never shared more broadly than cookie state. + * The hash-keyed lookup resolves the exact dictionary a response was encoded against, + * named by the hash carried in a {@code dcb} or {@code dcz} frame or advertised to the + * origin in the {@code Available-Dictionary} header. The origin-scoped lookup yields the + * candidates whose {@code Use-As-Dictionary} match pattern applies to an outbound request, + * from which the client selects what to advertise. *

* Implementations are expected to be thread-safe, since a store is shared across * concurrent exchanges. The store keeps entries as supplied; enforcing freshness against @@ -46,14 +51,56 @@ * * @since 5.7 */ - public interface CompressionDictionaryStore { - void add(CompressionDictionary dictionary); + /** + * Stores a dictionary, replacing any entry already held under the same origin and + * SHA-256 hash. + * + * @param partition the cookie storage partition; must not be {@code null}. + * @param dictionary the dictionary to store; must not be {@code null}. + * @since 5.7 + */ + void add(CookieStore partition, CompressionDictionary dictionary); + + /** + * Resolves the dictionary a response was encoded against, identified by the SHA-256 + * hash carried in a {@code dcb} or {@code dcz} frame or advertised in the + * {@code Available-Dictionary} header, scoped to the response origin. + * + * @param partition the cookie storage partition; must not be {@code null}. + * @param origin the origin the dictionary is associated with; must not be {@code null}. + * @param sha256 the SHA-256 hash of the dictionary content; must not be {@code null}. + * @return the matching dictionary, or {@code null} if none is held for the given + * origin and hash. + * @since 5.7 + */ + CompressionDictionary getByHash(CookieStore partition, URI origin, byte[] sha256); - CompressionDictionary getByHash(URI origin, byte[] sha256); + /** + * Returns the dictionaries associated with the given request origin, that is the + * candidates eligible to be advertised through {@code Available-Dictionary} for a + * request to that origin. + * + * @param partition the cookie storage partition; must not be {@code null}. + * @param origin the request origin to look up; must not be {@code null}. + * @return the matching dictionaries, never {@code null}; an empty list if none apply. + * @since 5.7 + */ + List getByOrigin(CookieStore partition, URI origin); - List getByOrigin(URI origin); + /** + * Removes all dictionaries associated with a cookie storage partition. + * + * @param partition the cookie storage partition; must not be {@code null}. + * @since 5.7 + */ + void clear(CookieStore partition); + /** + * Removes all stored dictionaries. + * + * @since 5.7 + */ void clear(); -} \ No newline at end of file +} diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryCookieStore.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryCookieStore.java new file mode 100644 index 0000000000..1344337a8f --- /dev/null +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryCookieStore.java @@ -0,0 +1,94 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.impl.async; + +import java.time.Instant; +import java.util.Date; +import java.util.List; + +import org.apache.hc.client5.http.cookie.Cookie; +import org.apache.hc.client5.http.cookie.CookieStore; +import org.apache.hc.client5.http.entity.compress.CompressionDictionaryStore; +import org.apache.hc.core5.annotation.Contract; +import org.apache.hc.core5.annotation.ThreadingBehavior; +import org.apache.hc.core5.util.Args; + +/** + * Binds the cookie and compression dictionary privacy partitions without + * adding compression-specific responsibilities to {@link CookieStore}. + * + * @since 5.7 + */ +@Contract(threading = ThreadingBehavior.SAFE_CONDITIONAL) +final class CompressionDictionaryCookieStore implements CookieStore { + + private final CookieStore delegate; + private final CompressionDictionaryStore compressionDictionaryStore; + + CompressionDictionaryCookieStore( + final CookieStore delegate, + final CompressionDictionaryStore compressionDictionaryStore) { + this.delegate = Args.notNull(delegate, "Cookie store"); + this.compressionDictionaryStore = Args.notNull( + compressionDictionaryStore, "Compression dictionary store"); + } + + boolean isBoundTo(final CompressionDictionaryStore store) { + return compressionDictionaryStore == store; + } + + @Override + public void addCookie(final Cookie cookie) { + delegate.addCookie(cookie); + } + + @Override + public List getCookies() { + return delegate.getCookies(); + } + + @Override + @SuppressWarnings("deprecation") + public boolean clearExpired(final Date date) { + return delegate.clearExpired(date); + } + + @Override + public boolean clearExpired(final Instant date) { + return delegate.clearExpired(date); + } + + @Override + public void clear() { + try { + delegate.clear(); + } finally { + compressionDictionaryStore.clear(this); + } + } + +} diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryFreshness.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryFreshness.java index 547362a3a1..636305a478 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryFreshness.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryFreshness.java @@ -1,158 +1,259 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ package org.apache.hc.client5.http.impl.async; import java.time.Duration; import java.time.Instant; -import java.time.ZonedDateTime; +import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.util.Locale; +import org.apache.hc.client5.http.utils.DateUtils; import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpHeaders; import org.apache.hc.core5.http.HttpResponse; +/** + * Derives, from the caching metadata of a dictionary response, the instant up to which the + * dictionary may still be treated as a match on a later request. The calculation follows the + * shared-cache freshness model: a freshness lifetime taken from {@code Cache-Control: max-age} + * or, in its absence, from {@code Expires} measured against {@code Date}, reduced by the response + * age derived from {@code Date}, {@code Age} and the request/response round trip. A + * {@code no-store} directive makes the response non-storable and a {@code no-cache} directive + * makes it immediately non-fresh; either directive, along with any missing or malformed input the + * calculation relies on, yields {@code null}. + *

+ * Used by the Compression Dictionary Transport machinery to decide whether a previously fetched + * dictionary is still eligible to be offered through {@code Available-Dictionary}. + */ final class CompressionDictionaryFreshness { + private static final DateTimeFormatter ASCTIME_FORMATTER = new DateTimeFormatterBuilder() + .parseLenient() + .parseCaseInsensitive() + .appendPattern(DateUtils.PATTERN_ASCTIME) + .toFormatter(Locale.ENGLISH); + + /** Not to be instantiated. */ private CompressionDictionaryFreshness() { } + /** + * Computes the instant up to which the dictionary carried by the given response stays fresh and + * may therefore be advertised as a match. The freshness lifetime is taken from + * {@code Cache-Control: max-age} when present, otherwise from {@code Expires} measured against + * {@code Date} (or against {@code responseTime} when {@code Date} is absent). A corrected initial + * age is derived from {@code Date}, {@code Age} and the transmission delay and subtracted from the + * freshness lifetime; the remaining seconds are added to {@code responseTime} to yield the result. + * + * @param response the dictionary response whose caching headers are inspected. + * @param requestTime the instant the request was sent; combined with {@code responseTime} it + * accounts for the transmission delay when correcting the reported age. + * @param responseTime the instant the response was received; the freshness deadline is expressed + * relative to it. + * @return the instant until which the dictionary remains fresh, or {@code null} when the response + * is non-storable ({@code no-store}), immediately non-fresh ({@code no-cache} or a + * malformed {@code Cache-Control}), has no usable freshness lifetime, carries a malformed + * {@code Age}, is already stale, or when the deadline would overflow the representable + * range. + */ static Instant determineValidUntil( final HttpResponse response, final Instant requestTime, final Instant responseTime) { - final CacheControl cacheControl = - parseCacheControl( - response.getFirstHeader(HttpHeaders.CACHE_CONTROL)); - + final CacheControl cacheControl = parseCacheControl(response.getHeaders(HttpHeaders.CACHE_CONTROL)); if (cacheControl.noStore || cacheControl.noCache) { return null; } + final Instant date = parseHttpDate(response.getFirstHeader(HttpHeaders.DATE)); final long freshnessLifetime; - if (cacheControl.maxAge >= 0) { freshnessLifetime = cacheControl.maxAge; } else { - final Instant expires = - parseHttpDate(response.getFirstHeader(HttpHeaders.EXPIRES)); - final Instant date = - parseHttpDate(response.getFirstHeader(HttpHeaders.DATE)); - - if (expires == null || date == null) { + final Instant expires = parseHttpDate(response.getFirstHeader(HttpHeaders.EXPIRES)); + if (expires == null) { return null; } - - freshnessLifetime = - Math.max(0, Duration.between(date, expires).getSeconds()); + final Instant reference = date != null ? date : responseTime; + freshnessLifetime = Math.max(0, Duration.between(reference, expires).getSeconds()); } - final Instant date = - parseHttpDate(response.getFirstHeader(HttpHeaders.DATE)); - - final long apparentAge = - date != null - ? Math.max( - 0, - Duration.between(date, responseTime) - .getSeconds()) - : 0; - - final long ageValue = - parseAge(response.getFirstHeader(HttpHeaders.AGE)); - - final long responseDelay = - Math.max( - 0, - Duration.between(requestTime, responseTime) - .getSeconds()); - - final long correctedAgeValue = - saturatedAdd(ageValue, responseDelay); - - final long correctedInitialAge = - Math.max(apparentAge, correctedAgeValue); - - final long remaining = - freshnessLifetime - correctedInitialAge; + final long ageValue = parseAge(response.getFirstHeader(HttpHeaders.AGE)); + if (ageValue < 0) { + return null; + } + final long apparentAge = date != null + ? Math.max(0, Duration.between(date, responseTime).getSeconds()) + : 0; + final long responseDelay = Math.max(0, Duration.between(requestTime, responseTime).getSeconds()); + final long correctedAgeValue = saturatedAdd(ageValue, responseDelay); + final long correctedInitialAge = Math.max(apparentAge, correctedAgeValue); + final long remaining = freshnessLifetime - correctedInitialAge; if (remaining <= 0) { return null; } - - return responseTime.plusSeconds(remaining); + try { + return responseTime.plusSeconds(remaining); + } catch (final RuntimeException ex) { + return null; + } } + /** + * Parses the {@code Age} header. A missing header is treated as an age of zero, a well-formed + * non-negative value is returned as seconds, and a negative or unparseable value returns + * {@code -1} to signal that the response must be treated as non-fresh. + * + * @param header the {@code Age} header, or {@code null} if absent. + * @return the age in seconds, {@code 0} when the header is absent, or {@code -1} when the value is + * invalid. + */ private static long parseAge(final Header header) { if (header == null) { return 0; } - try { - return Math.max(0, Long.parseLong(header.getValue().trim())); + final long age = Long.parseLong(header.getValue().trim()); + return age >= 0 ? age : -1; } catch (final NumberFormatException ex) { - return 0; + return -1; } } + /** + * Parses a date-valued header in any HTTP-date format accepted by HTTP semantics. + * + * @param header the date-valued header ({@code Date} or {@code Expires}), or {@code null} if absent. + * @return the parsed instant, or {@code null} when the header is absent or its value cannot be + * parsed. + */ private static Instant parseHttpDate(final Header header) { if (header == null) { return null; } - + final Instant standardDate = DateUtils.parseStandardDate(header.getValue()); + if (standardDate != null) { + return standardDate; + } try { - return ZonedDateTime.parse( - header.getValue(), - DateTimeFormatter.RFC_1123_DATE_TIME) - .toInstant(); + return LocalDateTime.parse(header.getValue(), ASCTIME_FORMATTER) + .toInstant(ZoneOffset.UTC); } catch (final RuntimeException ex) { return null; } } - private static CacheControl parseCacheControl(final Header header) { + /** + * Extracts the {@code no-store}, {@code no-cache} and {@code max-age} directives from the given + * {@code Cache-Control} headers, folding all header lines and comma-separated elements together. + * A quoted {@code max-age} value is unquoted before parsing. A negative, unparseable or + * self-contradicting {@code max-age} (the directive repeated with conflicting values) degrades the + * result to an {@linkplain CacheControl#invalid() invalid} control block flagged {@code no-cache}, + * so a broken directive fails closed as non-fresh rather than being silently ignored. + * + * @param headers the {@code Cache-Control} headers, or {@code null} if none are present. + * @return the parsed directives; never {@code null}. + */ + private static CacheControl parseCacheControl(final Header[] headers) { final CacheControl result = new CacheControl(); - - if (header == null) { + if (headers == null) { return result; } - - for (final String element : header.getValue().split(",")) { - final String directive = element.trim(); - - if ("no-store".equalsIgnoreCase(directive)) { - result.noStore = true; - } else if ("no-cache".equalsIgnoreCase(directive)) { - result.noCache = true; - } else if (directive.regionMatches( - true, 0, "max-age=", 0, 8)) { - try { - result.maxAge = - Math.max( - 0, - Long.parseLong( - directive.substring(8) - .trim())); - } catch (final NumberFormatException ignore) { - result.maxAge = -1; + for (final Header header : headers) { + final String[] elements = header.getValue().split(","); + for (final String element : elements) { + final String directive = element.trim(); + if ("no-store".equalsIgnoreCase(directive)) { + result.noStore = true; + } else if ("no-cache".equalsIgnoreCase(directive)) { + result.noCache = true; + } else if (directive.regionMatches(true, 0, "max-age=", 0, 8)) { + String value = directive.substring(8).trim(); + if (value.length() > 1 && value.charAt(0) == '"' && value.charAt(value.length() - 1) == '"') { + value = value.substring(1, value.length() - 1); + } + try { + final long maxAge = Long.parseLong(value); + if (maxAge < 0) { + return CacheControl.invalid(); + } + if (result.maxAge >= 0 && result.maxAge != maxAge) { + return CacheControl.invalid(); + } + result.maxAge = maxAge; + } catch (final NumberFormatException ex) { + return CacheControl.invalid(); + } } } } - return result; } - private static long saturatedAdd( - final long first, - final long second) { + /** + * Adds two non-negative second counts, clamping to {@link Long#MAX_VALUE} instead of overflowing, + * so that an extreme reported age combined with the transmission delay cannot wrap around into a + * misleadingly small corrected age. + * + * @param first the first summand. + * @param second the second summand. + * @return the sum, or {@link Long#MAX_VALUE} if the true sum would overflow. + */ + private static long saturatedAdd(final long first, final long second) { if (Long.MAX_VALUE - first < second) { return Long.MAX_VALUE; } return first + second; } + /** + * Mutable holder for the subset of {@code Cache-Control} directives that bear on dictionary + * freshness. A {@code maxAge} of {@code -1} means the directive was absent. + */ private static final class CacheControl { - private boolean noStore; private boolean noCache; private long maxAge = -1; + + /** + * Returns a control block that fails closed by flagging {@code no-cache}, used when a + * {@code max-age} directive is malformed or contradictory. + * + * @return a {@code no-cache} control block. + */ + static CacheControl invalid() { + final CacheControl value = new CacheControl(); + value.noCache = true; + return value; + } } -} \ No newline at end of file +} diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryHeaderSupport.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryHeaderSupport.java index 7eb4a20378..c0b7f5c615 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryHeaderSupport.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryHeaderSupport.java @@ -31,46 +31,79 @@ import org.apache.hc.core5.util.Args; /** - * Encodes the request headers used to negotiate a - * {@link org.apache.hc.client5.http.entity.compress.CompressionDictionary} for Compression - * Dictionary Transport. The header values are HTTP Structured Field - * items: {@code Available-Dictionary} carries the dictionary hash as a Byte Sequence and - * {@code Dictionary-ID} carries the server-supplied identifier as a String. + * Names the Compression Dictionary Transport headers and encodes the request headers used to + * negotiate a dictionary. {@code Available-Dictionary} carries the SHA-256 hash of the stored + * dictionary as an HTTP Structured Field Byte Sequence, and {@code Dictionary-ID} carries the + * opaque identifier the origin assigned through {@code Use-As-Dictionary} as a Structured Field + * String. Both header values are produced in the wire form required by the negotiation rather than + * as raw bytes or text. *

- * This class is a stateless collection of static helpers and holds no mutable state; it is therefore - * safe for concurrent use. It cannot be instantiated. + * Stateless and not instantiable. */ final class CompressionDictionaryHeaderSupport { + /** + * Response header through which an origin designates a resource as a dictionary and declares + * how it may be matched against future requests. + */ static final String USE_AS_DICTIONARY = "Use-As-Dictionary"; + + /** + * Request header naming the dictionary the client holds, keyed by the SHA-256 hash of its + * content encoded as a Structured Field Byte Sequence. + */ static final String AVAILABLE_DICTIONARY = "Available-Dictionary"; + + /** + * Request header echoing the opaque identifier the origin bound to the dictionary via + * {@code Use-As-Dictionary}, encoded as a Structured Field String. + */ static final String DICTIONARY_ID = "Dictionary-ID"; private CompressionDictionaryHeaderSupport() { } - static String formatHash(final byte[] hash) { - return ":" + Base64.getEncoder().encodeToString(hash) + ":"; + /** + * Encodes a dictionary hash as the {@code Available-Dictionary} value. The hash is emitted as a + * Structured Field Byte Sequence, that is base64 wrapped in a leading and trailing colon. + * + * @param hash the SHA-256 hash of the dictionary content; must not be {@code null}. + * @return the {@code Available-Dictionary} field value. + */ + static String formatAvailableDictionary(final byte[] hash) { + return ":" + Base64.getEncoder().encodeToString(Args.notNull(hash, "Dictionary hash")) + ":"; } - static String formatString(final String value) { - final StringBuilder buffer = - new StringBuilder(value.length() + 2); - + /** + * Encodes a dictionary identifier as the {@code Dictionary-ID} value. The identifier is emitted + * as a Structured Field String, double quoted with a backslash escaping any embedded quote or + * backslash. The string form only admits printable ASCII, so any character outside the range + * {@code 0x20} to {@code 0x7e} is rejected, as is a value longer than the 1024-character limit + * imposed on the identifier. + * + * @param value the opaque dictionary identifier; must not be {@code null}. + * @return the {@code Dictionary-ID} field value. + * @throws IllegalArgumentException if the value exceeds 1024 characters or contains a character + * that a Structured Field String cannot represent. + */ + static String formatDictionaryId(final String value) { + Args.notNull(value, "Dictionary ID"); + if (value.length() > 1024) { + throw new IllegalArgumentException("Dictionary ID length exceeds 1024 characters"); + } + final StringBuilder buffer = new StringBuilder(value.length() + 2); buffer.append('"'); - for (int i = 0; i < value.length(); i++) { final char ch = value.charAt(i); - + if (ch < 0x20 || ch > 0x7e) { + throw new IllegalArgumentException("Dictionary ID contains a character not permitted in a Structured Field String"); + } if (ch == '"' || ch == '\\') { buffer.append('\\'); } - buffer.append(ch); } - buffer.append('"'); - return buffer.toString(); } -} \ No newline at end of file +} diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryMatcher.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryMatcher.java index 973ffc56d7..f3be1537e0 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryMatcher.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryMatcher.java @@ -47,6 +47,21 @@ interface CompressionDictionaryMatcher { + /** + * Selects the single dictionary to advertise for the given request, or {@code null} when + * none is eligible. Only a candidate that is fresh, shares the request origin and whose + * stored URL pattern matches the request qualifies; when several qualify the most specific + * one is returned. + * + * @param requestUri the absolute request target. Dictionary transport is confined to secure + * origins, so a target whose scheme is not {@code https} matches nothing. + * @param requestDestination the request destination the response is bound for, tested against + * each candidate's {@code match-dest}, or {@code null} when the caller does not support + * request destinations. + * @param dictionaries the stored dictionaries to consider. + * @return the winning dictionary, whose identifier the caller offers to the origin through + * {@code Available-Dictionary}, or {@code null} when no candidate is eligible. + */ CompressionDictionary match( URI requestUri, String requestDestination, diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryUrlPatternMatcher.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryUrlPatternMatcher.java index 33f4013c32..766bd5cc45 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryUrlPatternMatcher.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/CompressionDictionaryUrlPatternMatcher.java @@ -1,15 +1,878 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ package org.apache.hc.client5.http.impl.async; +import java.net.IDN; import java.net.URI; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; +/** + * Validates and evaluates the URL pattern carried by a + * {@code Use-As-Dictionary} response header. + */ interface CompressionDictionaryUrlPatternMatcher { - boolean isValid( - String pattern, - URI dictionaryUri); + /** + * Validates a pattern using the dictionary request URL as its base URL. + * + * @param pattern the URL pattern. + * @param dictionaryUri the dictionary request URL. + * @return whether the pattern is valid and confined to the dictionary origin. + */ + boolean isValid(String pattern, URI dictionaryUri); - boolean matches( - String pattern, - URI dictionaryUri, - URI requestUri); -} \ No newline at end of file + /** + * Evaluates a pattern using the outbound request URL as its base URL. + * + * @param pattern the URL pattern. + * @param dictionaryUri the dictionary request URL. + * @param requestUri the outbound request URL. + * @return whether the outbound URL matches. + */ + boolean matches(String pattern, URI dictionaryUri, URI requestUri); +} + +/** + * URLPattern matcher for RFC 9842. Regular-expression groups are rejected, + * while wildcard, named and brace groups remain available. + */ +final class DefaultCompressionDictionaryUrlPatternMatcher + implements CompressionDictionaryUrlPatternMatcher { + + @Override + public boolean isValid(final String pattern, final URI dictionaryUri) { + if (pattern == null || pattern.isEmpty() || !isAbsoluteHttpUri(dictionaryUri)) { + return false; + } + try { + UrlPattern.compile(pattern, dictionaryUri); + return true; + } catch (final IllegalArgumentException ex) { + return false; + } + } + + @Override + public boolean matches( + final String pattern, + final URI dictionaryUri, + final URI requestUri) { + if (!isValid(pattern, dictionaryUri) + || !isAbsoluteHttpUri(requestUri) + || !sameOrigin(dictionaryUri, requestUri)) { + return false; + } + try { + // RFC 9842 section 2.2.2 deliberately uses the outbound URL as baseURL. + return UrlPattern.compile(pattern, requestUri).matches(requestUri); + } catch (final IllegalArgumentException ex) { + return false; + } + } + + private static boolean isAbsoluteHttpUri(final URI uri) { + return uri != null + && uri.isAbsolute() + && uri.getHost() != null + && ("http".equalsIgnoreCase(uri.getScheme()) + || "https".equalsIgnoreCase(uri.getScheme())); + } + + private static boolean sameOrigin(final URI first, final URI second) { + return first.getScheme().equalsIgnoreCase(second.getScheme()) + && normalizeHost(first.getHost()).equals(normalizeHost(second.getHost())) + && effectivePort(first) == effectivePort(second); + } + + private static int effectivePort(final URI uri) { + if (uri.getPort() >= 0) { + return uri.getPort(); + } + return "https".equalsIgnoreCase(uri.getScheme()) ? 443 : 80; + } + + private static String normalizeHost(final String host) { + final String value = host.length() > 1 && host.charAt(0) == '[' + && host.charAt(host.length() - 1) == ']' + ? host.substring(1, host.length() - 1) + : host; + if (value.indexOf(':') >= 0) { + return value.toLowerCase(Locale.ROOT); + } + return IDN.toASCII(value).toLowerCase(Locale.ROOT); + } + + /** + * Compiled URLPattern string. Constructor-string parsing and component + * matching are intentionally kept out of the dictionary selection policy. + */ + private static final class UrlPattern { + + private final Pattern protocol; + private final Pattern username; + private final Pattern password; + private final Pattern hostname; + private final Pattern port; + private final Pattern pathname; + private final Pattern search; + private final Pattern hash; + + private UrlPattern(final ConstructorParts parts) { + this.protocol = compileComponent(parts.protocol, Component.PROTOCOL); + this.username = compileComponent(parts.username, Component.DEFAULT); + this.password = compileComponent(parts.password, Component.DEFAULT); + this.hostname = compileComponent(parts.hostname, Component.HOSTNAME); + this.port = compileComponent(parts.port, Component.DEFAULT); + this.pathname = compileComponent(parts.pathname, Component.PATHNAME); + this.search = compileComponent(parts.search, Component.DEFAULT); + this.hash = compileComponent(parts.hash, Component.DEFAULT); + } + + static UrlPattern compile(final String input, final URI baseUri) { + return new UrlPattern(ConstructorParts.parse(input, baseUri)); + } + + boolean matches(final URI uri) { + final UserInfo userInfo = UserInfo.from(uri); + return matches(protocol, uri.getScheme().toLowerCase(Locale.ROOT)) + && matches(username, userInfo.username) + && matches(password, userInfo.password) + && matches(hostname, normalizeHost(uri.getHost())) + && matches(port, normalizedPort(uri)) + && matches(pathname, rawPath(uri)) + && matches(search, valueOrEmpty(uri.getRawQuery())) + && matches(hash, valueOrEmpty(uri.getRawFragment())); + } + + private static boolean matches(final Pattern pattern, final String value) { + return pattern == null || pattern.matcher(value).matches(); + } + } + + /** URLPattern constructor-string components; {@code null} means wildcard. */ + private static final class ConstructorParts { + + private String protocol; + private String username; + private String password; + private String hostname; + private String port; + private String pathname; + private String search; + private String hash; + + static ConstructorParts parse(final String input, final URI baseUri) { + if (input == null || input.isEmpty() || baseUri == null || !baseUri.isAbsolute()) { + throw new IllegalArgumentException("Invalid URL pattern"); + } + rejectRegexpGroups(input); + + final ConstructorParts result = new ConstructorParts(); + final int authorityMarker = findAuthorityMarker(input); + if (authorityMarker >= 0) { + result.parseAbsolute(input, authorityMarker); + } else if (input.startsWith("//")) { + result.protocol = baseUri.getScheme().toLowerCase(Locale.ROOT); + result.parseAuthorityAndTail(input, 2); + } else { + result.protocol = baseUri.getScheme().toLowerCase(Locale.ROOT); + result.hostname = normalizeHost(baseUri.getHost()); + result.port = normalizedPort(baseUri); + result.parseRelativeTail(input, baseUri); + } + return result; + } + + private void parseAbsolute(final String input, final int authorityMarker) { + final String scheme = input.substring(0, authorityMarker); + if (scheme.isEmpty() + || !containsComponentPatternSyntax(scheme) && !isScheme(scheme)) { + throw new IllegalArgumentException("Invalid URL pattern protocol"); + } + protocol = scheme.toLowerCase(Locale.ROOT); + parseAuthorityAndTail(input, authorityMarker + 3); + } + + private void parseAuthorityAndTail( + final String input, + final int authorityStart) { + final int bracedSlash = findBracedSlash(input, authorityStart); + if (bracedSlash >= 0) { + final String prefix = input.substring(authorityStart, bracedSlash); + final int openBrace = prefix.lastIndexOf('{'); + parseAuthority(prefix.substring(0, openBrace) + + prefix.substring(openBrace + 1)); + pathname = null; + search = null; + hash = null; + return; + } + final int tailStart = findTailStart(input, authorityStart); + final String authority = input.substring( + authorityStart, tailStart >= 0 ? tailStart : input.length()); + parseAuthority(authority); + + if (tailStart < 0) { + pathname = null; + search = null; + hash = null; + } else { + final String tail = input.substring(tailStart); + if (tail.charAt(0) == '?') { + pathname = "/"; + parseTail(tail, pathname); + } else if (tail.charAt(0) == '#') { + pathname = "/"; + search = ""; + hash = tail.substring(1); + } else { + parseTail(tail, null); + } + } + } + + private static int findBracedSlash( + final String value, + final int start) { + int braces = 0; + boolean escaped = false; + for (int i = start; i < value.length(); i++) { + final char ch = value.charAt(i); + if (escaped) { + escaped = false; + } else if (ch == '\\') { + escaped = true; + } else if (ch == '{') { + braces++; + } else if (ch == '}') { + braces--; + } else if (ch == '/' && braces > 0) { + return i; + } else if (braces == 0 + && (ch == '#' || ch == '?' && !isGroupModifier(value, i))) { + return -1; + } + } + return -1; + } + + private void parseAuthority(final String authority) { + if (authority.isEmpty()) { + throw new IllegalArgumentException("URL pattern has no authority"); + } + + String hostPort = authority; + final int at = authority.lastIndexOf('@'); + if (at >= 0) { + final String userInfo = authority.substring(0, at); + hostPort = authority.substring(at + 1); + final int colon = userInfoDelimiter(userInfo); + if (colon >= 0) { + final String parsedUsername = userInfo.substring(0, colon); + username = parsedUsername.endsWith("\\") + ? parsedUsername.substring(0, parsedUsername.length() - 1) + : parsedUsername; + password = userInfo.substring(colon + 1); + } else { + username = userInfo; + password = ""; + } + } + + final HostPort parsed = HostPort.parse(hostPort, protocol); + hostname = parsed.hostname; + port = parsed.port; + } + + private static int userInfoDelimiter(final String value) { + boolean escaped = false; + for (int i = 0; i < value.length(); i++) { + final char ch = value.charAt(i); + if (escaped) { + escaped = false; + } else if (ch == '\\') { + if (i + 1 < value.length() && value.charAt(i + 1) == ':') { + return i + 1; + } + escaped = true; + } else if (ch == ':') { + if (i == 0 && i + 1 < value.length() + && isNameStart(value.charAt(i + 1))) { + i++; + while (i + 1 < value.length() + && isNameChar(value.charAt(i + 1))) { + i++; + } + } else { + return i; + } + } + } + return -1; + } + + private void parseRelativeTail(final String input, final URI baseUri) { + if (input.charAt(0) == '?') { + pathname = rawPath(baseUri); + parseTail(input, pathname); + } else if (input.charAt(0) == '#') { + pathname = rawPath(baseUri); + search = valueOrEmpty(baseUri.getRawQuery()); + hash = input.substring(1); + } else { + parseTail(input, resolvePath(input, baseUri)); + } + } + + private void parseTail(final String tail, final String resolvedPath) { + final int hashIndex = findDelimiter(tail, '#'); + final String beforeHash = hashIndex >= 0 ? tail.substring(0, hashIndex) : tail; + final int queryIndex = findQueryDelimiter(beforeHash); + final boolean escapedQuery = queryIndex > 0 + && beforeHash.charAt(queryIndex - 1) == '\\'; + + final String path = queryIndex >= 0 + ? beforeHash.substring(0, escapedQuery ? queryIndex - 1 : queryIndex) + : beforeHash; + if (resolvedPath != null) { + pathname = resolvedPath; + } else if (!path.isEmpty()) { + pathname = normalizePath(path); + } + if (queryIndex >= 0) { + search = beforeHash.substring(queryIndex + 1); + } + if (hashIndex >= 0) { + hash = tail.substring(hashIndex + 1); + } + } + + private static String resolvePath(final String input, final URI baseUri) { + final int hashIndex = findDelimiter(input, '#'); + final String beforeHash = hashIndex >= 0 ? input.substring(0, hashIndex) : input; + final int queryIndex = findQueryDelimiter(beforeHash); + final String path = queryIndex >= 0 ? beforeHash.substring(0, queryIndex) : beforeHash; + if (path.startsWith("/")) { + return path; + } + final String basePath = rawPath(baseUri); + final int slash = basePath.lastIndexOf('/'); + return normalizePath( + (slash >= 0 ? basePath.substring(0, slash + 1) : "/") + path); + } + } + + private static final class HostPort { + final String hostname; + final String port; + + HostPort(final String hostname, final String port) { + this.hostname = hostname; + this.port = port; + } + + static HostPort parse(final String value, final String protocol) { + if (value.isEmpty()) { + throw new IllegalArgumentException("URL pattern has no hostname"); + } + final String host; + String port = ""; + if (value.charAt(0) == '[') { + final int end = value.indexOf(']'); + if (end < 0) { + throw new IllegalArgumentException("Invalid IPv6 hostname"); + } + host = value.substring(1, end); + if (end + 1 < value.length()) { + if (value.charAt(end + 1) != ':') { + throw new IllegalArgumentException("Invalid authority"); + } + port = value.substring(end + 2); + } + } else { + final int colon = portDelimiter(value); + if (colon >= 0) { + host = value.substring(0, colon); + port = value.substring(colon + 1); + } else { + host = value; + } + } + if (host.isEmpty()) { + throw new IllegalArgumentException("URL pattern has no hostname"); + } + if (!port.isEmpty() && !containsComponentPatternSyntax(port)) { + final int number; + try { + number = Integer.parseInt(port); + } catch (final NumberFormatException ex) { + throw new IllegalArgumentException("Invalid URL pattern port", ex); + } + if (number < 0 || number > 65535) { + throw new IllegalArgumentException("Invalid URL pattern port"); + } + if (number == defaultPort(protocol)) { + port = ""; + } + } + final String normalizedHost = containsComponentPatternSyntax(host) + ? host.toLowerCase(Locale.ROOT) + : normalizeHost(host); + return new HostPort(normalizedHost, port); + } + + private static int portDelimiter(final String value) { + final int namedPort = value.indexOf("::"); + if (namedPort > 0) { + return namedPort; + } + final int colon = value.lastIndexOf(':'); + if (colon <= 0 || colon + 1 >= value.length()) { + return -1; + } + final char first = value.charAt(colon + 1); + return isDigit(first) || first == '*' || first == '{' + ? colon + : -1; + } + } + + private enum Component { + DEFAULT('\0', false), + PROTOCOL('\0', true), + HOSTNAME('.', true), + PATHNAME('/', false); + + final char delimiter; + final boolean lowerCase; + + Component(final char delimiter, final boolean lowerCase) { + this.delimiter = delimiter; + this.lowerCase = lowerCase; + } + } + + private static Pattern compileComponent(final String input, final Component component) { + if (input == null) { + return null; + } + try { + final Set names = new HashSet<>(); + final String regex = new ComponentCompiler(input, component, names).compile(); + return Pattern.compile("^(?:" + regex + ")$"); + } catch (final PatternSyntaxException ex) { + throw new IllegalArgumentException("Invalid URL pattern", ex); + } + } + + /** Compiles URLPattern groups without accepting custom regular expressions. */ + private static final class ComponentCompiler { + + private final String input; + private final Component component; + private final Set names; + private int pos; + + ComponentCompiler( + final String input, + final Component component, + final Set names) { + this.input = component.lowerCase ? input.toLowerCase(Locale.ROOT) : input; + this.component = component; + this.names = names; + } + + String compile() { + final String result = compileSequence(false); + if (pos != input.length()) { + throw new IllegalArgumentException("Unexpected URL pattern group terminator"); + } + return result; + } + + private String compileSequence(final boolean grouped) { + final StringBuilder result = new StringBuilder(); + final StringBuilder literal = new StringBuilder(); + while (pos < input.length()) { + final char ch = input.charAt(pos++); + if (ch == '\\') { + if (pos >= input.length()) { + throw new IllegalArgumentException("Dangling URL pattern escape"); + } + literal.append(input.charAt(pos++)); + } else if (ch == '}') { + if (!grouped) { + throw new IllegalArgumentException("Unexpected URL pattern group terminator"); + } + appendLiteral(result, literal); + return result.toString(); + } else if (ch == '{') { + appendLiteral(result, literal); + final String body = compileSequence(true); + appendModified(result, body, modifier()); + } else if (ch == ':') { + final int start = pos; + while (pos < input.length() && isNameChar(input.charAt(pos))) { + pos++; + } + if (start == pos || !isNameStart(input.charAt(start))) { + throw new IllegalArgumentException("URL pattern group has no name"); + } + final String name = input.substring(start, pos); + if (!names.add(name)) { + throw new IllegalArgumentException("Duplicate URL pattern group name"); + } + if (pos < input.length() && input.charAt(pos) == '(') { + throw new IllegalArgumentException("Regular-expression groups are not permitted"); + } + final char modifier = modifier(); + final String prefix = detachPrefix(literal, modifier); + appendLiteral(result, literal); + final String body = prefix + defaultGroupRegex(component.delimiter); + appendModified(result, body, modifier); + } else if (ch == '*') { + appendLiteral(result, literal); + appendModified(result, ".*", modifier()); + } else if (ch == '(' || ch == ')') { + throw new IllegalArgumentException("Regular-expression groups are not permitted"); + } else if (ch == '?' || ch == '+') { + throw new IllegalArgumentException("URL pattern modifier has no group"); + } else { + literal.append(ch); + } + } + if (grouped) { + throw new IllegalArgumentException("Unterminated URL pattern group"); + } + appendLiteral(result, literal); + return result.toString(); + } + + private char modifier() { + if (pos < input.length()) { + final char ch = input.charAt(pos); + if (ch == '?' || ch == '*' || ch == '+') { + pos++; + return ch; + } + } + return '\0'; + } + + private String detachPrefix(final StringBuilder literal, final char modifier) { + if (modifier != '\0' + && component.delimiter != '\0' + && literal.length() > 0 + && literal.charAt(literal.length() - 1) == component.delimiter) { + literal.setLength(literal.length() - 1); + return Pattern.quote(String.valueOf(component.delimiter)); + } + return ""; + } + + private static void appendLiteral( + final StringBuilder result, + final StringBuilder literal) { + if (literal.length() > 0) { + result.append(Pattern.quote(literal.toString())); + literal.setLength(0); + } + } + + private static void appendModified( + final StringBuilder result, + final String body, + final char modifier) { + result.append("(?:").append(body).append(')'); + if (modifier != '\0') { + result.append(modifier); + } + } + } + + private static String defaultGroupRegex(final char delimiter) { + return delimiter == '\0' + ? ".+?" + : "[^" + delimiter + "]+?"; + } + + private static void rejectRegexpGroups(final String value) { + boolean escaped = false; + for (int i = 0; i < value.length(); i++) { + final char ch = value.charAt(i); + if (escaped) { + escaped = false; + } else if (ch == '\\') { + escaped = true; + } else if (ch == '(' || ch == ')') { + throw new IllegalArgumentException("Regular-expression groups are not permitted"); + } + } + if (escaped) { + throw new IllegalArgumentException("Dangling URL pattern escape"); + } + } + + private static int findAuthorityMarker(final String value) { + int braces = 0; + boolean escaped = false; + for (int i = 0; i + 2 < value.length(); i++) { + final char ch = value.charAt(i); + if (escaped) { + escaped = false; + } else if (ch == '\\') { + escaped = true; + } else if (ch == '{') { + braces++; + } else if (ch == '}') { + braces--; + } else if (braces == 0 + && ch == ':' + && value.charAt(i + 1) == '/' + && value.charAt(i + 2) == '/') { + return i; + } + } + return -1; + } + + private static int findTailStart(final String value, final int start) { + int braces = 0; + boolean escaped = false; + for (int i = start; i < value.length(); i++) { + final char ch = value.charAt(i); + if (escaped) { + escaped = false; + } else if (ch == '\\') { + escaped = true; + } else if (ch == '{') { + braces++; + } else if (ch == '}') { + braces--; + } else if (braces == 0 + && (ch == '/' || ch == '#' + || ch == '?' && !isGroupModifier(value, i))) { + return i; + } + } + return -1; + } + + private static int findDelimiter(final String value, final char delimiter) { + int braces = 0; + boolean escaped = false; + for (int i = 0; i < value.length(); i++) { + final char ch = value.charAt(i); + if (escaped) { + escaped = false; + } else if (ch == '\\') { + escaped = true; + } else if (ch == '{') { + braces++; + } else if (ch == '}') { + braces--; + } else if (braces == 0 && ch == delimiter) { + return i; + } + } + return -1; + } + + private static int findQueryDelimiter(final String value) { + int braces = 0; + boolean escaped = false; + for (int i = 0; i < value.length(); i++) { + final char ch = value.charAt(i); + if (escaped) { + escaped = false; + } else if (ch == '\\' && i + 1 < value.length() + && value.charAt(i + 1) == '?') { + return i + 1; + } else if (ch == '\\') { + escaped = true; + } else if (ch == '{') { + braces++; + } else if (ch == '}') { + braces--; + } else if (braces == 0 && ch == '?' && !isGroupModifier(value, i)) { + return i; + } + } + return -1; + } + + private static boolean isGroupModifier(final String value, final int questionIndex) { + if (questionIndex == 0) { + return false; + } + final char previous = value.charAt(questionIndex - 1); + if (previous == '}' || previous == '*') { + return true; + } + int i = questionIndex - 1; + while (i >= 0 && isNameChar(value.charAt(i))) { + i--; + } + return i >= 0 + && value.charAt(i) == ':' + && i + 1 < questionIndex + && isNameStart(value.charAt(i + 1)); + } + + private static boolean containsComponentPatternSyntax(final String value) { + boolean escaped = false; + for (int i = 0; i < value.length(); i++) { + final char ch = value.charAt(i); + if (escaped) { + escaped = false; + } else if (ch == '\\') { + escaped = true; + } else if (ch == '*' || ch == ':' || ch == '{' || ch == '}' + || ch == '(' || ch == ')' || ch == '?' || ch == '+') { + return true; + } + } + return escaped; + } + + private static boolean isScheme(final String value) { + if (value.isEmpty() || !isAlpha(value.charAt(0))) { + return false; + } + for (int i = 1; i < value.length(); i++) { + final char ch = value.charAt(i); + if (!isAlpha(ch) && !isDigit(ch) && ch != '+' && ch != '-' && ch != '.') { + return false; + } + } + return true; + } + + private static boolean isNameChar(final char ch) { + return isAlpha(ch) || isDigit(ch) || ch == '_'; + } + + private static boolean isNameStart(final char ch) { + return isAlpha(ch) || ch == '_'; + } + + private static boolean isAlpha(final char ch) { + return ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z'; + } + + private static boolean isDigit(final char ch) { + return ch >= '0' && ch <= '9'; + } + + private static int defaultPort(final String scheme) { + return "https".equalsIgnoreCase(scheme) ? 443 + : "http".equalsIgnoreCase(scheme) ? 80 : -1; + } + + private static String normalizedPort(final URI uri) { + return uri.getPort() < 0 || uri.getPort() == defaultPort(uri.getScheme()) + ? "" + : Integer.toString(uri.getPort()); + } + + private static String rawPath(final URI uri) { + final String value = URI.create(uri.toASCIIString()).normalize().getRawPath(); + return value == null || value.isEmpty() ? "/" : value; + } + + private static String normalizePath(final String value) { + final boolean absolute = value.startsWith("/"); + final String[] segments = value.split("/", -1); + final List normalized = new ArrayList<>(); + for (int i = absolute ? 1 : 0; i < segments.length; i++) { + final String segment = segments[i]; + if (isSingleDot(segment)) { + continue; + } + if (isDoubleDot(segment)) { + if (!normalized.isEmpty()) { + normalized.remove(normalized.size() - 1); + } + } else { + normalized.add(segment); + } + } + final StringBuilder result = new StringBuilder(value.length()); + if (absolute) { + result.append('/'); + } + for (int i = 0; i < normalized.size(); i++) { + if (i > 0) { + result.append('/'); + } + result.append(normalized.get(i)); + } + return result.length() > 0 ? result.toString() : absolute ? "/" : ""; + } + + private static boolean isSingleDot(final String value) { + return ".".equals(value) || "%2e".equalsIgnoreCase(value); + } + + private static boolean isDoubleDot(final String value) { + return "..".equals(value) + || ".%2e".equalsIgnoreCase(value) + || "%2e.".equalsIgnoreCase(value) + || "%2e%2e".equalsIgnoreCase(value); + } + + private static String valueOrEmpty(final String value) { + return value != null ? value : ""; + } + + private static final class UserInfo { + final String username; + final String password; + + UserInfo(final String username, final String password) { + this.username = username; + this.password = password; + } + + static UserInfo from(final URI uri) { + final String value = uri.getRawUserInfo(); + if (value == null) { + return new UserInfo("", ""); + } + final int colon = value.indexOf(':'); + return colon >= 0 + ? new UserInfo(value.substring(0, colon), value.substring(colon + 1)) + : new UserInfo(value, ""); + } + } +} diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/ContentCompressionAsyncExec.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/ContentCompressionAsyncExec.java index db3c3e7ca4..dc0bd11656 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/ContentCompressionAsyncExec.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/ContentCompressionAsyncExec.java @@ -29,13 +29,12 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; -import java.time.DateTimeException; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; +import java.util.Map; import java.util.Set; import java.util.function.UnaryOperator; @@ -48,6 +47,7 @@ import org.apache.hc.client5.http.async.methods.InflatingDictionaryZstdDataConsumer; import org.apache.hc.client5.http.async.methods.InflatingGzipDataConsumer; import org.apache.hc.client5.http.async.methods.InflatingZstdDataConsumer; +import org.apache.hc.client5.http.cookie.CookieStore; import org.apache.hc.client5.http.entity.compress.CompressionDictionary; import org.apache.hc.client5.http.entity.compress.CompressionDictionaryStore; import org.apache.hc.client5.http.entity.compress.ContentCoding; @@ -104,19 +104,15 @@ public ContentCompressionAsyncExec( final List dictionaryTokens = new ArrayList<>(); if (compressionDictionaryStore != null) { - if (decoderMap.containsKey(ContentCoding.DCB.token())) { + if (containsToken(decoderMap, ContentCoding.DCB.token())) { dictionaryTokens.add(ContentCoding.DCB.token()); } else if (Brotli4jRuntime.available()) { - rb.register(ContentCoding.DCB.token(), - d -> new InflatingDictionaryBrotliDataConsumer(d, compressionDictionaryStore)); dictionaryTokens.add(ContentCoding.DCB.token()); } - if (decoderMap.containsKey(ContentCoding.DCZ.token())) { + if (containsToken(decoderMap, ContentCoding.DCZ.token())) { dictionaryTokens.add(ContentCoding.DCZ.token()); } else if (ZstdRuntime.available()) { - rb.register(ContentCoding.DCZ.token(), - d -> new InflatingDictionaryZstdDataConsumer(d, compressionDictionaryStore)); dictionaryTokens.add(ContentCoding.DCZ.token()); } } @@ -180,14 +176,10 @@ public ContentCompressionAsyncExec( final List dictionaryTokens = new ArrayList<>(); if (compressionDictionaryStore != null) { if (Brotli4jRuntime.available()) { - rb.register(ContentCoding.DCB.token(), - d -> new InflatingDictionaryBrotliDataConsumer(d, compressionDictionaryStore)); dictionaryTokens.add(ContentCoding.DCB.token()); } if (ZstdRuntime.available()) { - rb.register(ContentCoding.DCZ.token(), - d -> new InflatingDictionaryZstdDataConsumer(d, compressionDictionaryStore)); dictionaryTokens.add(ContentCoding.DCZ.token()); } } @@ -226,9 +218,11 @@ public void execute( final HttpClientContext ctx = scope != null ? scope.clientContext : HttpClientContext.create(); final boolean enabled = ctx.getRequestConfigOrDefault().isContentCompressionEnabled(); final URI requestUri = resolveRequestUri(request, scope); + final Instant requestTime = Instant.now(); + final CookieStore privacyPartition = getPrivacyPartition(ctx); final CompressionDictionary dictionary = enabled - ? findDictionary(request, requestUri) + ? findDictionary(request, requestUri, privacyPartition) : null; if (dictionary != null) { @@ -267,14 +261,12 @@ public AsyncDataConsumer handleResponse(final HttpResponse rsp, } final UseAsDictionary useAsDictionary = - parseUseAsDictionary(rsp, requestUri); - - final Instant storedAt = useAsDictionary != null - ? Instant.now() - : null; + parseUseAsDictionary(rsp, requestUri, privacyPartition); + final Instant responseTime = Instant.now(); + final Instant storedAt = useAsDictionary != null ? responseTime : null; final Instant validUntil = storedAt != null - ? getValidUntil(rsp, storedAt) + ? CompressionDictionaryFreshness.determineValidUntil(rsp, requestTime, responseTime) : null; final List codecs = ContentCodingSupport.parseContentCodecs(details); @@ -289,6 +281,7 @@ public AsyncDataConsumer handleResponse(final HttpResponse rsp, downstream = new DictionaryCapturingAsyncDataConsumer( downstream, compressionDictionaryStore, + privacyPartition, requestUri, useAsDictionary, storedAt, @@ -306,11 +299,29 @@ public AsyncDataConsumer handleResponse(final HttpResponse rsp, "Dictionary Content-Encoding without negotiated dictionary: " + codec); } - final UnaryOperator op = decoders.lookup(codec); - if (op != null) { - downstream = op.apply(downstream); + if (ContentCoding.DCB.token().equalsIgnoreCase(codec)) { + if (!dictionaryAcceptTokens.contains(ContentCoding.DCB.token())) { + throw new HttpException("Unsupported Content-Encoding: " + codec); + } + final UnaryOperator op = decoders.lookup(codec); + downstream = op != null + ? op.apply(downstream) + : new InflatingDictionaryBrotliDataConsumer(downstream, dictionary); + } else if (ContentCoding.DCZ.token().equalsIgnoreCase(codec)) { + if (!dictionaryAcceptTokens.contains(ContentCoding.DCZ.token())) { + throw new HttpException("Unsupported Content-Encoding: " + codec); + } + final UnaryOperator op = decoders.lookup(codec); + downstream = op != null + ? op.apply(downstream) + : new InflatingDictionaryZstdDataConsumer(downstream, dictionary); } else { - throw new HttpException("Unsupported Content-Encoding: " + codec); + final UnaryOperator op = decoders.lookup(codec); + if (op != null) { + downstream = op.apply(downstream); + } else { + throw new HttpException("Unsupported Content-Encoding: " + codec); + } } } return downstream; @@ -321,6 +332,7 @@ public AsyncDataConsumer handleResponse(final HttpResponse rsp, downstream = new DictionaryCapturingAsyncDataConsumer( downstream, compressionDictionaryStore, + privacyPartition, requestUri, useAsDictionary, storedAt, @@ -350,9 +362,11 @@ public void failed(final Exception ex) { private CompressionDictionary findDictionary( final HttpRequest request, - final URI requestUri) { + final URI requestUri, + final CookieStore privacyPartition) { if (compressionDictionaryStore == null || compressionDictionaryMatcher == null + || privacyPartition == null || requestUri == null || !"https".equalsIgnoreCase(requestUri.getScheme()) || request.containsHeader(CompressionDictionaryHeaderSupport.AVAILABLE_DICTIONARY) @@ -362,92 +376,66 @@ private CompressionDictionary findDictionary( return compressionDictionaryMatcher.match( requestUri, - compressionDictionaryStore.getByOrigin(requestUri)); + null, + compressionDictionaryStore.getByOrigin(privacyPartition, requestUri)); } private UseAsDictionary parseUseAsDictionary( final HttpResponse response, - final URI requestUri) { + final URI requestUri, + final CookieStore privacyPartition) { if (compressionDictionaryStore == null + || privacyPartition == null || requestUri == null || !"https".equalsIgnoreCase(requestUri.getScheme())) { return null; } - final Header header = - response.getFirstHeader(CompressionDictionaryHeaderSupport.USE_AS_DICTIONARY); - if (header == null) { + final Header[] headers = response.getHeaders(CompressionDictionaryHeaderSupport.USE_AS_DICTIONARY); + if (headers == null || headers.length == 0) { return null; } - try { - final UseAsDictionary useAsDictionary = UseAsDictionary.parse(header.getValue()); - return useAsDictionary.isSupported() ? useAsDictionary : null; - } catch (final ParseException ex) { - return null; - } - } - - private static Instant getValidUntil( - final HttpResponse response, - final Instant storedAt) { - long maxAge = -1; - - for (final Header header : response.getHeaders(HttpHeaders.CACHE_CONTROL)) { - final String[] directives = header.getValue().split(","); - for (final String value : directives) { - final String directive = value.trim(); - final String lowerCase = directive.toLowerCase(Locale.ROOT); - - if ("no-store".equals(lowerCase) || "no-cache".equals(lowerCase)) { - return null; - } - - if (lowerCase.startsWith("max-age=")) { - final String maxAgeValue = unquote( - directive.substring(directive.indexOf('=') + 1).trim()); - try { - maxAge = Long.parseLong(maxAgeValue); - } catch (final NumberFormatException ex) { - return null; - } - } + final StringBuilder value = new StringBuilder(); + for (final Header header : headers) { + if (value.length() > 0) { + value.append(','); } + value.append(header.getValue()); } - if (maxAge <= 0) { - return null; - } - - long age = 0; - final Header ageHeader = response.getFirstHeader("Age"); - if (ageHeader != null) { - try { - age = Long.parseLong(ageHeader.getValue()); - } catch (final NumberFormatException ex) { + try { + final UseAsDictionary useAsDictionary = UseAsDictionary.parse(value.toString()); + if (!useAsDictionary.isSupported() + || !new DefaultCompressionDictionaryUrlPatternMatcher().isValid( + useAsDictionary.getMatch(), requestUri)) { return null; } - } - - final long remaining = maxAge - Math.max(0, age); - if (remaining <= 0) { + return useAsDictionary; + } catch (final ParseException | IllegalArgumentException ex) { return null; } + } - try { - return storedAt.plusSeconds(remaining); - } catch (final DateTimeException | ArithmeticException ex) { - return null; + private CookieStore getPrivacyPartition(final HttpClientContext context) { + final CookieStore cookieStore = context.getCookieStore(); + if (cookieStore instanceof CompressionDictionaryCookieStore + && ((CompressionDictionaryCookieStore) cookieStore) + .isBoundTo(compressionDictionaryStore)) { + return cookieStore; } + return null; } - private static String unquote(final String value) { - if (value.length() > 1 - && value.charAt(0) == '"' - && value.charAt(value.length() - 1) == '"') { - return value.substring(1, value.length() - 1); + private static boolean containsToken( + final Map map, + final String expected) { + for (final String token : map.keySet()) { + if (expected.equalsIgnoreCase(token)) { + return true; + } } - return value; + return false; } private static URI resolveRequestUri( @@ -505,4 +493,4 @@ public Set getTrailerNames() { } }; } -} \ No newline at end of file +} diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/DefaultCompressionDictionaryMatcher.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/DefaultCompressionDictionaryMatcher.java index 0e3ba8e2e5..1fd92495dc 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/DefaultCompressionDictionaryMatcher.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/DefaultCompressionDictionaryMatcher.java @@ -60,11 +60,42 @@ final class DefaultCompressionDictionaryMatcher private final Clock clock; private final CompressionDictionaryUrlPatternMatcher urlPatternMatcher; + /** + * Creates a matcher backed by the system UTC clock and the default URL pattern matcher. + */ + DefaultCompressionDictionaryMatcher() { + this(Clock.systemUTC(), new DefaultCompressionDictionaryUrlPatternMatcher()); + } + + /** + * Creates a matcher with an explicit clock, used to make freshness evaluation deterministic + * in tests. + * + * @param clock the clock supplying the instant against which candidate freshness is judged. + */ + DefaultCompressionDictionaryMatcher(final Clock clock) { + this(clock, new DefaultCompressionDictionaryUrlPatternMatcher()); + } + + /** + * Creates a matcher with an explicit URL pattern matcher over the system UTC clock. + * + * @param urlPatternMatcher the strategy that validates and evaluates a candidate's stored + * {@code match} pattern against the request. + */ DefaultCompressionDictionaryMatcher( final CompressionDictionaryUrlPatternMatcher urlPatternMatcher) { this(Clock.systemUTC(), urlPatternMatcher); } + /** + * Creates a matcher with both collaborators supplied explicitly. + * + * @param clock the clock supplying the instant against which candidate freshness is judged. + * @param urlPatternMatcher the strategy that validates and evaluates a candidate's stored + * {@code match} pattern against the request. + * @throws NullPointerException if either argument is {@code null}. + */ DefaultCompressionDictionaryMatcher( final Clock clock, final CompressionDictionaryUrlPatternMatcher urlPatternMatcher) { @@ -73,15 +104,42 @@ final class DefaultCompressionDictionaryMatcher Args.notNull(urlPatternMatcher, "URL pattern matcher"); } + /** + * Convenience overload for callers that do not carry a request destination; equivalent to + * {@link #match(URI, String, Collection)} with a {@code null} destination. + * + * @param requestUri the absolute request target. + * @param dictionaries the stored dictionaries to consider. + * @return the winning dictionary, or {@code null} when no candidate is eligible. + */ + CompressionDictionary match( + final URI requestUri, + final Collection dictionaries) { + return match(requestUri, null, dictionaries); + } + + /** + * {@inheritDoc} + *

+ * A candidate survives only when it is fresh at the clock instant, is a {@code raw} + * dictionary, shares the request origin, applies to the request destination and carries a + * {@code match} pattern that is valid for its source and matches the request. Among the + * survivors the winner is chosen by three keys in order: a {@code match-dest} that names the + * request destination outranks a destination-agnostic one, then the longest {@code match} + * pattern wins, and finally the most recently stored dictionary breaks any remaining tie. + * + * @throws NullPointerException if {@code requestUri} or {@code dictionaries} is {@code null}. + */ @Override public CompressionDictionary match( final URI requestUri, final String requestDestination, final Collection dictionaries) { - if (requestUri == null - || !"https".equalsIgnoreCase(requestUri.getScheme()) - || dictionaries == null + Args.notNull(requestUri, "Request URI"); + Args.notNull(dictionaries, "Dictionaries"); + + if (!"https".equalsIgnoreCase(requestUri.getScheme()) || dictionaries.isEmpty()) { return null; } @@ -90,12 +148,13 @@ public CompressionDictionary match( return dictionaries.stream() .filter(dictionary -> dictionary.isFresh(now)) - .filter(dictionary -> "raw".equalsIgnoreCase( - dictionary.getType())) + .filter(dictionary -> "raw".equals(dictionary.getType())) .filter(dictionary -> sameOrigin( dictionary.getSource(), requestUri)) .filter(dictionary -> destinationMatches( dictionary, requestDestination)) + .filter(dictionary -> urlPatternMatcher.isValid( + dictionary.getMatch(), dictionary.getSource())) .filter(dictionary -> urlPatternMatcher.matches( dictionary.getMatch(), dictionary.getSource(), @@ -115,6 +174,12 @@ public CompressionDictionary match( .orElse(null); } + /** + * Ranks a candidate for tie-breaking: a dictionary whose {@code match-dest} explicitly names + * the request destination is preferred over one that matches irrespective of destination. + * Returns {@code 1} for such an explicit destination match and {@code 0} otherwise, including + * when the caller carries no destination or the candidate constrains none. + */ private static int destinationPrecedence( final CompressionDictionary dictionary, final String requestDestination) { @@ -129,14 +194,18 @@ private static int destinationPrecedence( : 0; } + /** + * Tests whether a candidate's {@code match-dest} admits the request destination. An empty + * {@code match-dest} places no constraint and admits every destination. + */ private static boolean destinationMatches( final CompressionDictionary dictionary, final String requestDestination) { if (requestDestination == null) { /* - * RFC 9842: clients that do not support request destinations - * MUST treat match-dest as an empty list. + * Compression Dictionary Transport: a client that does not support request + * destinations treats match-dest as an empty list, so the constraint is waived. */ return true; } @@ -145,6 +214,11 @@ private static boolean destinationMatches( || dictionary.getMatchDest().contains(requestDestination); } + /** + * Tests whether two URIs denote the same origin, that is the same scheme, host and effective + * port. Dictionary transport is scoped to a single origin, so a candidate stored for one + * origin is never advertised on a request to another. + */ private static boolean sameOrigin( final URI first, final URI second) { @@ -162,6 +236,11 @@ private static boolean equalsIgnoreCase( && first.equalsIgnoreCase(second); } + /** + * Resolves the port that participates in origin comparison, substituting the scheme default + * when the URI carries no explicit port: {@code 443} for {@code https} and {@code 80} for + * {@code http}. Any other scheme with no port yields {@code -1}. + */ private static int effectivePort(final URI uri) { if (uri.getPort() >= 0) { return uri.getPort(); @@ -172,4 +251,4 @@ private static int effectivePort(final URI uri) { ? 80 : -1; } -} \ No newline at end of file +} diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/DictionaryCapturingAsyncDataConsumer.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/DictionaryCapturingAsyncDataConsumer.java index 5bace0cdd1..ad32e8d8ad 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/DictionaryCapturingAsyncDataConsumer.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/DictionaryCapturingAsyncDataConsumer.java @@ -33,6 +33,7 @@ import java.time.Instant; import java.util.List; +import org.apache.hc.client5.http.cookie.CookieStore; import org.apache.hc.client5.http.entity.compress.CompressionDictionary; import org.apache.hc.client5.http.entity.compress.CompressionDictionaryStore; import org.apache.hc.core5.http.Header; @@ -68,6 +69,7 @@ final class DictionaryCapturingAsyncDataConsumer implements AsyncDataConsumer { private final AsyncDataConsumer downstream; private final CompressionDictionaryStore store; + private final CookieStore partition; private final URI source; private final UseAsDictionary directive; private final Instant storedAt; @@ -77,9 +79,27 @@ final class DictionaryCapturingAsyncDataConsumer implements AsyncDataConsumer { private boolean discarded; + /** + * Creates a capturing decorator for a single response whose {@code Use-As-Dictionary} + * offer has already been parsed. The arguments are validated eagerly: the references must + * be non-null and {@code maxSize} must be positive. + * + * @param downstream the body consumer to which every chunk is forwarded unchanged. + * @param store the store that receives the captured dictionary on a complete, supported delivery. + * @param partition the cookie storage partition associated with the exchange. + * @param source the request {@link URI} the dictionary is keyed by. + * @param directive the parsed {@code Use-As-Dictionary} offer supplying the match pattern, + * destination, identifier and type; a dictionary is stored only when the offer is + * {@link UseAsDictionary#isSupported() supported}. + * @param storedAt the instant recorded as the dictionary's creation time. + * @param validUntil the instant past which the stored dictionary is no longer valid. + * @param maxSize the capture ceiling in bytes; once the buffer would exceed it capture is + * abandoned for the rest of the exchange and no dictionary is stored. + */ DictionaryCapturingAsyncDataConsumer( final AsyncDataConsumer downstream, final CompressionDictionaryStore store, + final CookieStore partition, final URI source, final UseAsDictionary directive, final Instant storedAt, @@ -87,6 +107,7 @@ final class DictionaryCapturingAsyncDataConsumer implements AsyncDataConsumer { final int maxSize) { this.downstream = Args.notNull(downstream, "Downstream data consumer"); this.store = Args.notNull(store, "Dictionary store"); + this.partition = Args.notNull(partition, "Cookie partition"); this.source = Args.notNull(source, "Source"); this.directive = Args.notNull(directive, "Directive"); this.storedAt = Args.notNull(storedAt, "Stored at"); @@ -95,6 +116,14 @@ final class DictionaryCapturingAsyncDataConsumer implements AsyncDataConsumer { this.buffer = new ByteArrayOutputStream(Math.min(maxSize, 8192)); } + /** + * Propagates the capacity update unchanged; this decorator imposes no flow control of its + * own and leaves back-pressure entirely to the downstream consumer. + * + * @param capacityChannel the channel through which the downstream consumer signals the + * capacity it is prepared to accept. + * @throws IOException if the downstream consumer fails while handling the capacity update. + */ @Override public void updateCapacity(final CapacityChannel capacityChannel) throws IOException { downstream.updateCapacity(capacityChannel); @@ -106,6 +135,10 @@ public void updateCapacity(final CapacityChannel capacityChannel) throws IOExcep * range from the entry position to the position left by {@code downstream} is captured, * so partially consumed buffers are handled correctly. Capture is abandoned for the rest * of the exchange once the buffered size would exceed {@code maxSize}. + * + * @param src the received bytes; its position advanced by {@code downstream} marks the + * range that was accepted and is therefore captured. + * @throws IOException if the downstream consumer fails to accept the chunk. */ @Override public void consume(final ByteBuffer src) throws IOException { @@ -134,17 +167,23 @@ public void consume(final ByteBuffer src) throws IOException { } /** - * Registers the captured body as a compression dictionary when capture completed intact - * and the directive is supported, then signals stream end to the downstream consumer. - * The downstream consumer is always notified, whether or not a dictionary was stored. + * Signals stream end to the downstream consumer, then registers the captured body as a + * compression dictionary when capture completed intact and the directive is supported. + * A downstream failure prevents the response from being stored as a dictionary. + * + * @param trailers the trailing headers, forwarded unchanged to the downstream consumer. + * @throws HttpException if the downstream consumer rejects the end of stream. + * @throws IOException in case of an I/O error while signalling stream end downstream. */ @Override public void streamEnd( final List trailers) throws HttpException, IOException { + downstream.streamEnd(trailers); + if (!discarded && directive.isSupported()) { - store.add(new CompressionDictionary( + store.add(partition, new CompressionDictionary( buffer.toByteArray(), source, directive.getMatch(), @@ -154,8 +193,6 @@ public void streamEnd( storedAt, validUntil)); } - - downstream.streamEnd(trailers); } /** @@ -168,4 +205,4 @@ public void releaseResources() { buffer.reset(); downstream.releaseResources(); } -} \ No newline at end of file +} diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/HttpAsyncClientBuilder.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/HttpAsyncClientBuilder.java index 6054e2d449..8c607f5804 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/HttpAsyncClientBuilder.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/HttpAsyncClientBuilder.java @@ -891,7 +891,11 @@ public HttpAsyncClientBuilder setContentDecoderMap( * Transport as defined by RFC 9842. *

* When configured, matching dictionaries can be advertised and used for - * {@code dcb} and {@code dcz} response decompression. + * {@code dcb} and {@code dcz} response decompression. The default cookie + * store is bound to the dictionary store so clearing cookies also clears + * the corresponding dictionary partition. Dictionary transport is disabled + * when a custom cookie store is configured on the builder or supplied by an + * execution context, because its clearing lifecycle cannot be observed. *

* * @param compressionDictionaryStore the compression dictionary store, @@ -1271,6 +1275,10 @@ public CloseableHttpAsyncClient build() { CookieStore cookieStoreCopy = this.cookieStore; if (cookieStoreCopy == null) { cookieStoreCopy = new BasicCookieStore(); + if (compressionDictionaryStore != null) { + cookieStoreCopy = new CompressionDictionaryCookieStore( + cookieStoreCopy, compressionDictionaryStore); + } } CredentialsProvider credentialsProviderCopy = this.credentialsProvider; diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/UseAsDictionary.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/UseAsDictionary.java index cc810380ef..df850864db 100644 --- a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/UseAsDictionary.java +++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/UseAsDictionary.java @@ -26,29 +26,37 @@ */ package org.apache.hc.client5.http.impl.async; +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Base64; import java.util.Collections; import java.util.List; -import java.util.Locale; import org.apache.hc.core5.http.ParseException; import org.apache.hc.core5.util.Args; /** - * Parsed representation of the {@code Use-As-Dictionary} response header defined by Compression - * Dictionary Transport. The origin returns this header alongside a resource it - * offers as a future compression dictionary; it is a Structured Field Dictionary whose members - * describe how the resource may later be matched and referenced. This type retains the members - * the client acts on: {@code match} (a URL Pattern, mandatory) and {@code id} (an optional - * opaque identifier echoed back in the {@code Dictionary-ID} request header), together with the - * {@code type} that names the dictionary format. + * Parsed form of the {@code Use-As-Dictionary} response header used by Compression Dictionary + * Transport. The header advertises that a response may serve as a compression dictionary for + * subsequent requests and is an HTTP Structured Field Dictionary carrying a required {@code match} + * URL pattern, an optional {@code match-dest} list of request destinations, an optional opaque + * {@code id} echoed back in {@code Dictionary-ID}, and a {@code type} token naming the dictionary + * format. *

- * Only the {@code raw} dictionary type is recognised; other values leave {@link #isSupported()} - * {@code false} so the caller can ignore an offer it cannot honour rather than fail. Instances - * are immutable and therefore safe to share between threads. + * Only the {@link #RAW raw} type is understood. An offer of any other type parses successfully but + * reports {@link #isSupported()} as {@code false}, so an unrecognised format renders the dictionary + * unusable rather than aborting the exchange. */ final class UseAsDictionary { + /** + * The single dictionary {@code type} this implementation honours, and the default when the + * {@code type} member is absent. + */ static final String RAW = "raw"; private final String match; @@ -56,6 +64,16 @@ final class UseAsDictionary { private final String id; private final String type; + /** + * Creates a parsed dictionary offer. {@code matchDest} is defensively copied and exposed as an + * unmodifiable list; a {@code null} list becomes empty, a {@code null} {@code id} becomes the + * empty string, and a {@code null} {@code type} defaults to {@link #RAW}. + * + * @param match the URL pattern the dictionary applies to. + * @param matchDest the request destinations the offer is restricted to, or {@code null} for none. + * @param id the opaque identifier echoed in {@code Dictionary-ID}, or {@code null} for none. + * @param type the dictionary format token, or {@code null} for {@link #RAW}. + */ UseAsDictionary( final String match, final List matchDest, @@ -64,28 +82,598 @@ final class UseAsDictionary { this.match = match; this.matchDest = matchDest != null ? Collections.unmodifiableList(new ArrayList<>(matchDest)) - : Collections.emptyList(); + : Collections.emptyList(); this.id = id != null ? id : ""; this.type = type != null ? type : RAW; } + /** + * Parses a {@code Use-As-Dictionary} header value as an HTTP Structured Field Dictionary. + * Members other than {@code match}, {@code match-dest}, {@code id} and {@code type} are accepted + * and ignored, as are member parameters, so unknown extensions do not break parsing. A member + * whose value has the wrong Structured Field type for its key is rejected. The {@code match} + * member is mandatory; the others fall back to their defaults when absent. + * + * @param value the raw header value; must not be blank. + * @return the parsed offer. + * @throws ParseException if the value is malformed against the Structured Field grammar, if + * {@code match} is missing or not a non-empty String, if {@code match-dest} is not an inner + * list of Strings, if {@code id} is not a String of at most 1024 characters, or if + * {@code type} is not a Token. + */ + static UseAsDictionary parse(final String value) throws ParseException { + Args.notBlank(value, "Use-As-Dictionary"); + for (int i = 0; i < value.length(); i++) { + if (value.charAt(i) > 0x7f) { + throw new ParseException("Structured Field value is not ASCII"); + } + } + final Parser parser = new Parser(value); + String match = null; + List matchDest = Collections.emptyList(); + String id = ""; + String type = RAW; + + parser.skipSp(); + while (!parser.atEnd()) { + final String key = parser.parseKey(); + final Value member; + if (parser.consume('=')) { + member = parser.parseMemberValue(); + } else { + member = Value.bool(); + parser.parseParameters(); + } + + if ("match".equals(key)) { + if (member.kind != Value.STRING || member.stringValue.length() == 0) { + throw new ParseException("Invalid Use-As-Dictionary match member"); + } + match = member.stringValue; + } else if ("match-dest".equals(key)) { + if (member.kind != Value.INNER_LIST || !member.stringList) { + throw new ParseException("Invalid Use-As-Dictionary match-dest member"); + } + matchDest = member.listValue; + } else if ("id".equals(key)) { + if (member.kind != Value.STRING || member.stringValue.length() > 1024) { + throw new ParseException("Invalid Use-As-Dictionary id member"); + } + id = member.stringValue; + } else if ("type".equals(key)) { + if (member.kind != Value.TOKEN) { + throw new ParseException("Invalid Use-As-Dictionary type member"); + } + type = member.stringValue; + } + + parser.skipOws(); + if (parser.atEnd()) { + break; + } + if (!parser.consume(',')) { + throw new ParseException("Invalid Use-As-Dictionary dictionary separator"); + } + parser.skipOws(); + if (parser.atEnd()) { + throw new ParseException("Trailing comma in Use-As-Dictionary"); + } + } + + if (match == null) { + throw new ParseException("Use-As-Dictionary requires a match member"); + } + return new UseAsDictionary(match, matchDest, id, type); + } + + /** + * @return the URL pattern the dictionary applies to. + */ String getMatch() { return match; } + /** + * @return the request destinations the offer is restricted to, unmodifiable and empty when + * unrestricted. + */ List getMatchDest() { return matchDest; } + /** + * @return the opaque identifier to echo in {@code Dictionary-ID}, or the empty string when none + * was offered. + */ String getId() { return id; } + /** + * @return the dictionary format token, {@link #RAW} when the {@code type} member was absent. + */ String getType() { return type; } + /** + * Whether the offered {@code type} is one this client can act on. Only {@link #RAW} is honoured; + * any other token leaves the offer parseable but unusable. + * + * @return {@code true} if the dictionary can be used. + */ boolean isSupported() { - return RAW.equalsIgnoreCase(type); + return RAW.equals(type); + } + + /** + * A parsed Structured Field member value, reduced to only what the dictionary members care + * about. Item types with no bearing on {@code Use-As-Dictionary} (numbers, byte sequences, + * booleans, dates, display strings) are recognised for validation but collapsed to + * {@link #OTHER}; only Strings, Tokens and inner lists carry their value forward. For an inner + * list {@link #stringList} records whether every element was a String, which distinguishes a + * valid {@code match-dest} from one contaminated by non-String items. + */ + private static final class Value { + static final int OTHER = 0; + static final int STRING = 1; + static final int TOKEN = 2; + static final int INNER_LIST = 3; + + final int kind; + final String stringValue; + final List listValue; + final boolean stringList; + + Value(final int kind, final String stringValue, final List listValue, final boolean stringList) { + this.kind = kind; + this.stringValue = stringValue; + this.listValue = listValue; + this.stringList = stringList; + } + + static Value string(final String value) { + return new Value(STRING, value, null, false); + } + + static Value token(final String value) { + return new Value(TOKEN, value, null, false); + } + + static Value other() { + return new Value(OTHER, null, null, false); + } + + static Value bool() { + return other(); + } + + static Value innerList(final List values, final boolean stringsOnly) { + return new Value(INNER_LIST, null, + Collections.unmodifiableList(new ArrayList<>(values)), stringsOnly); + } + } + + /** + * A minimal recursive-descent parser for the subset of the HTTP Structured Fields grammar the + * {@code Use-As-Dictionary} header exercises: a Dictionary of keys mapping to bare items or + * inner lists, with parameters. It advances a cursor over the input and validates each construct + * strictly, but only retains the String, Token and inner-list values the caller inspects; every + * other item type is parsed for well-formedness and discarded. The parser is single-use and not + * thread-safe. + */ + private static final class Parser { + private final String value; + private int pos; + + Parser(final String value) { + this.value = value; + } + + boolean atEnd() { + return pos >= value.length(); + } + + /** + * Skips optional whitespace, the space and horizontal-tab run allowed around Dictionary + * members. + */ + void skipOws() { + while (!atEnd()) { + final char ch = value.charAt(pos); + if (ch == ' ' || ch == '\t') { + pos++; + } else { + break; + } + } + } + + /** + * Skips SP characters. RFC 9651 permits OWS around top-level Dictionary + * separators, but only SP before the field value and after a parameter + * semicolon. + */ + void skipSp() { + while (!atEnd() && value.charAt(pos) == ' ') { + pos++; + } + } + + /** + * Consumes the next character if it matches, reporting whether it did without advancing on a + * mismatch. + * + * @param expected the character to match. + * @return {@code true} if the character was present and consumed. + */ + boolean consume(final char expected) { + if (!atEnd() && value.charAt(pos) == expected) { + pos++; + return true; + } + return false; + } + + /** + * Parses a Structured Field key, the lower-case identifier that names a Dictionary member or + * a parameter. + * + * @return the key. + * @throws ParseException if no valid key starts at the cursor. + */ + String parseKey() throws ParseException { + if (atEnd() || !isKeyStart(value.charAt(pos))) { + throw new ParseException("Invalid Structured Field dictionary key"); + } + final int start = pos++; + while (!atEnd() && isKeyChar(value.charAt(pos))) { + pos++; + } + return value.substring(start, pos); + } + + /** + * Parses the value bound to a Dictionary key, which is either an inner list or a bare item + * followed by its parameters. + * + * @return the parsed value. + * @throws ParseException if the value is malformed. + */ + Value parseMemberValue() throws ParseException { + final Value result; + if (!atEnd() && value.charAt(pos) == '(') { + result = parseInnerList(); + } else { + result = parseBareItem(); + parseParameters(); + } + return result; + } + + /** + * Parses a parenthesised inner list. Only String elements are collected; the presence of any + * non-String element clears the strings-only flag so a {@code match-dest} carrying anything + * other than Strings can be rejected upstream. Per-item and whole-list parameters are parsed + * and discarded. + * + * @return an {@link Value#INNER_LIST} value. + * @throws ParseException if the list is unterminated or an element or separator is invalid. + */ + Value parseInnerList() throws ParseException { + pos++; + final List strings = new ArrayList<>(); + boolean stringsOnly = true; + while (true) { + while (!atEnd() && value.charAt(pos) == ' ') { + pos++; + } + if (atEnd()) { + throw new ParseException("Unterminated Structured Field inner list"); + } + if (value.charAt(pos) == ')') { + pos++; + break; + } + final Value item = parseBareItem(); + if (item.kind == Value.STRING) { + strings.add(item.stringValue); + } else { + stringsOnly = false; + } + parseParameters(); + if (!atEnd() && value.charAt(pos) != ')' && value.charAt(pos) != ' ') { + throw new ParseException("Invalid Structured Field inner list separator"); + } + } + parseParameters(); + return Value.innerList(strings, stringsOnly); + } + + /** + * Parses a single bare item, dispatching on the leading character across the whole item + * grammar: String, Token, Integer or Decimal, Byte Sequence, Boolean, Date and Display + * String. Only Strings and Tokens retain their value; the remaining types are validated and + * reduced to {@link Value#OTHER}. + * + * @return the parsed item. + * @throws ParseException if no valid item starts at the cursor. + */ + Value parseBareItem() throws ParseException { + if (atEnd()) { + throw new ParseException("Missing Structured Field item"); + } + final char ch = value.charAt(pos); + if (ch == '"') { + return Value.string(parseString()); + } + if (isTokenStart(ch)) { + return Value.token(parseToken()); + } + if (ch == '-' || isDigit(ch)) { + parseNumber(); + return Value.other(); + } + if (ch == ':') { + parseByteSequence(); + return Value.other(); + } + if (ch == '?') { + parseBoolean(); + return Value.other(); + } + if (ch == '@') { + pos++; + parseInteger(); + return Value.other(); + } + if (ch == '%') { + parseDisplayString(); + return Value.other(); + } + throw new ParseException("Invalid Structured Field bare item"); + } + + /** + * Parses a quoted String, unescaping the only two permitted escapes, backslash and double + * quote, and rejecting any character outside printable ASCII. + * + * @return the unescaped String content. + * @throws ParseException if an escape or character is invalid or the String is unterminated. + */ + String parseString() throws ParseException { + pos++; + final StringBuilder result = new StringBuilder(); + while (!atEnd()) { + final char ch = value.charAt(pos++); + if (ch == '"') { + return result.toString(); + } + if (ch == '\\') { + if (atEnd()) { + throw new ParseException("Invalid Structured Field String escape"); + } + final char escaped = value.charAt(pos++); + if (escaped != '\\' && escaped != '"') { + throw new ParseException("Invalid Structured Field String escape"); + } + result.append(escaped); + } else { + if (ch < 0x20 || ch > 0x7e) { + throw new ParseException("Invalid Structured Field String character"); + } + result.append(ch); + } + } + throw new ParseException("Unterminated Structured Field String"); + } + + /** + * Parses a Token, the unquoted identifier used by the {@code type} member. + * + * @return the token text. + * @throws ParseException if no token character follows the start. + */ + String parseToken() throws ParseException { + final int start = pos++; + while (!atEnd() && isTokenChar(value.charAt(pos))) { + pos++; + } + if (pos == start) { + throw new ParseException("Invalid Structured Field Token"); + } + return value.substring(start, pos); + } + + /** + * Parses an Integer or Decimal, consuming an optional sign, the integer digits and, for a + * Decimal, a fractional part of at most three digits. The value is validated but not + * retained. + * + * @throws ParseException if no digits are present or the decimal form is malformed. + */ + void parseNumber() throws ParseException { + if (consume('-') && atEnd()) { + throw new ParseException("Invalid Structured Field number"); + } + final int start = pos; + while (!atEnd() && isDigit(value.charAt(pos))) { + pos++; + } + if (pos == start) { + throw new ParseException("Invalid Structured Field number"); + } + final int integerDigits = pos - start; + if (consume('.')) { + if (integerDigits > 12) { + throw new ParseException("Structured Field decimal is out of range"); + } + final int fraction = pos; + while (!atEnd() && isDigit(value.charAt(pos)) && pos - fraction < 3) { + pos++; + } + if (pos == fraction || !atEnd() && isDigit(value.charAt(pos))) { + throw new ParseException("Invalid Structured Field decimal"); + } + } else if (integerDigits > 15) { + throw new ParseException("Structured Field integer is out of range"); + } + } + + /** + * Parses the integer that follows the {@code @} marker of a Date item. The value is + * validated but not retained. + * + * @throws ParseException if no digits are present. + */ + void parseInteger() throws ParseException { + if (consume('-') && atEnd()) { + throw new ParseException("Invalid Structured Field integer"); + } + final int start = pos; + while (!atEnd() && isDigit(value.charAt(pos))) { + pos++; + } + if (pos == start || pos - start > 15) { + throw new ParseException("Invalid Structured Field integer"); + } + } + + /** + * Parses a colon-delimited Byte Sequence, verifying that the enclosed text is well-formed + * Base64. The decoded bytes are discarded. + * + * @throws ParseException if the encoding is invalid or the sequence is unterminated. + */ + void parseByteSequence() throws ParseException { + pos++; + final int start = pos; + while (!atEnd() && value.charAt(pos) != ':') { + final char ch = value.charAt(pos); + if (!(isAlpha(ch) || isDigit(ch) || ch == '+' || ch == '/' || ch == '=')) { + throw new ParseException("Invalid Structured Field Byte Sequence"); + } + pos++; + } + if (atEnd()) { + throw new ParseException("Unterminated Structured Field Byte Sequence"); + } + final String encoded = value.substring(start, pos++); + try { + Base64.getDecoder().decode(encoded); + } catch (final IllegalArgumentException ex) { + throw new ParseException("Invalid Structured Field Byte Sequence"); + } + } + + /** + * Parses a Boolean, the {@code ?0} or {@code ?1} form. The value is validated but not + * retained. + * + * @throws ParseException if the character after {@code ?} is neither {@code 0} nor {@code 1}. + */ + void parseBoolean() throws ParseException { + pos++; + if (atEnd() || value.charAt(pos) != '0' && value.charAt(pos) != '1') { + throw new ParseException("Invalid Structured Field Boolean"); + } + pos++; + } + + /** + * Parses a Display String, the {@code %"..."} form whose content is UTF-8 percent-encoded. + * Percent escapes must be two lower-case hex digits; the decoded text is discarded. + * + * @throws ParseException if the opening quote, an escape or a character is invalid, or the + * string is unterminated. + */ + void parseDisplayString() throws ParseException { + pos++; + if (!consume('"')) { + throw new ParseException("Invalid Structured Field Display String"); + } + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + while (!atEnd()) { + final char ch = value.charAt(pos++); + if (ch == '"') { + try { + StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes.toByteArray())); + return; + } catch (final CharacterCodingException ex) { + throw new ParseException("Invalid UTF-8 in Structured Field Display String"); + } + } + if (ch == '%') { + if (pos + 1 >= value.length() + || !isLowerHex(value.charAt(pos)) + || !isLowerHex(value.charAt(pos + 1))) { + throw new ParseException("Invalid Structured Field Display String escape"); + } + bytes.write(hexValue(value.charAt(pos)) * 16 + hexValue(value.charAt(pos + 1))); + pos += 2; + } else if (ch < 0x20 || ch > 0x7e) { + throw new ParseException("Invalid Structured Field Display String character"); + } else { + bytes.write((byte) ch); + } + } + throw new ParseException("Unterminated Structured Field Display String"); + } + + /** + * Consumes any trailing {@code ;key=value} parameters attached to an item or list. This + * implementation assigns no meaning to parameters, so they are validated and discarded. + * + * @throws ParseException if a parameter key or value is malformed. + */ + void parseParameters() throws ParseException { + while (!atEnd() && value.charAt(pos) == ';') { + pos++; + skipSp(); + final String ignored = parseKey(); + if (ignored.length() == 0) { + throw new ParseException("Invalid Structured Field parameter"); + } + if (consume('=')) { + parseBareItem(); + } + } + } + + private static boolean isKeyStart(final char ch) { + return ch >= 'a' && ch <= 'z' || ch == '*'; + } + + private static boolean isKeyChar(final char ch) { + return isKeyStart(ch) || isDigit(ch) || ch == '_' || ch == '-' || ch == '.'; + } + + private static boolean isTokenStart(final char ch) { + return isAlpha(ch) || ch == '*'; + } + + private static boolean isTokenChar(final char ch) { + return isAlpha(ch) || isDigit(ch) + || "!#$%&'*+-.^_`|~:/".indexOf(ch) >= 0; + } + + private static boolean isAlpha(final char ch) { + return ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z'; + } + + private static boolean isDigit(final char ch) { + return ch >= '0' && ch <= '9'; + } + + private static boolean isLowerHex(final char ch) { + return isDigit(ch) || ch >= 'a' && ch <= 'f'; + } + + private static int hexValue(final char ch) { + return isDigit(ch) ? ch - '0' : ch - 'a' + 10; + } } -} \ No newline at end of file +} diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/TestInflatingDictionaryBrotliDataConsumer.java b/httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/TestInflatingDictionaryBrotliDataConsumer.java index e59b37118e..673420b974 100644 --- a/httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/TestInflatingDictionaryBrotliDataConsumer.java +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/TestInflatingDictionaryBrotliDataConsumer.java @@ -26,30 +26,36 @@ */ package org.apache.hc.client5.http.async.methods; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verifyNoInteractions; -import static org.mockito.Mockito.when; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URI; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.Collections; +import java.util.List; + +import com.aayushatharva.brotli4j.Brotli4jLoader; +import com.aayushatharva.brotli4j.encoder.BrotliOutputStream; +import com.aayushatharva.brotli4j.encoder.Encoder; +import com.aayushatharva.brotli4j.encoder.PreparedDictionary; -import org.apache.hc.client5.http.entity.compress.BasicCompressionDictionaryStore; import org.apache.hc.client5.http.entity.compress.CompressionDictionary; -import org.apache.hc.client5.http.entity.compress.CompressionDictionaryStore; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpException; import org.apache.hc.core5.http.nio.AsyncDataConsumer; +import org.apache.hc.core5.http.nio.CapacityChannel; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.Test; /** - * Native-free unit tests for {@link InflatingDictionaryBrotliDataConsumer}. - *

- * These tests exercise only the header-parsing and dictionary-lookup logic, - * all of which runs before the lazy Brotli {@code DecoderJNI.Wrapper} is - * created, so no native Brotli library is required. + * Unit tests for {@link InflatingDictionaryBrotliDataConsumer}. */ class TestInflatingDictionaryBrotliDataConsumer { @@ -82,9 +88,9 @@ private static byte[] header(final byte[] magic, final byte[] hash) { @Test void invalidMagicThrowsFromConsumeSingleBuffer() { final AsyncDataConsumer downstream = mock(AsyncDataConsumer.class); - final CompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final CompressionDictionary dictionary = dictionary(new byte[]{1}); final InflatingDictionaryBrotliDataConsumer consumer = - new InflatingDictionaryBrotliDataConsumer(downstream, store); + new InflatingDictionaryBrotliDataConsumer(downstream, dictionary); final byte[] bogusMagic = {0x00, 0x44, 0x43, 0x42}; final byte[] hash = new byte[HASH_LENGTH]; @@ -98,9 +104,9 @@ void invalidMagicThrowsFromConsumeSingleBuffer() { @Test void invalidMagicThrowsWhenHeaderSplitAcrossTwoBuffers() { final AsyncDataConsumer downstream = mock(AsyncDataConsumer.class); - final CompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final CompressionDictionary dictionary = dictionary(new byte[]{1}); final InflatingDictionaryBrotliDataConsumer consumer = - new InflatingDictionaryBrotliDataConsumer(downstream, store); + new InflatingDictionaryBrotliDataConsumer(downstream, dictionary); final byte[] bogusMagic = {0x00, 0x44, 0x43, 0x42}; final byte[] hash = new byte[HASH_LENGTH]; @@ -118,13 +124,12 @@ void invalidMagicThrowsWhenHeaderSplitAcrossTwoBuffers() { } @Test - void unknownDictionaryThrowsFromConsume() { + void differentDictionaryHashThrowsFromConsume() { final AsyncDataConsumer downstream = mock(AsyncDataConsumer.class); - final CompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final CompressionDictionary dictionary = dictionary(new byte[]{1}); final InflatingDictionaryBrotliDataConsumer consumer = - new InflatingDictionaryBrotliDataConsumer(downstream, store); + new InflatingDictionaryBrotliDataConsumer(downstream, dictionary); - // Valid MAGIC, but the store is empty so getByHash returns null. final byte[] hash = new byte[HASH_LENGTH]; for (int i = 0; i < HASH_LENGTH; i++) { hash[i] = (byte) i; @@ -132,16 +137,16 @@ void unknownDictionaryThrowsFromConsume() { final ByteBuffer src = ByteBuffer.wrap(header(MAGIC, hash)); final IOException ex = assertThrows(IOException.class, () -> consumer.consume(src)); - assertTrue(ex.getMessage().contains("Compression dictionary not available"), ex.getMessage()); + assertTrue(ex.getMessage().contains("does not use the negotiated dictionary"), ex.getMessage()); verifyNoInteractions(downstream); } @Test - void unknownDictionaryThrowsWhenHeaderSplitAcrossTwoBuffers() { + void differentDictionaryHashThrowsWhenHeaderSplitAcrossTwoBuffers() { final AsyncDataConsumer downstream = mock(AsyncDataConsumer.class); - final CompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final CompressionDictionary dictionary = dictionary(new byte[]{1}); final InflatingDictionaryBrotliDataConsumer consumer = - new InflatingDictionaryBrotliDataConsumer(downstream, store); + new InflatingDictionaryBrotliDataConsumer(downstream, dictionary); final byte[] hash = new byte[HASH_LENGTH]; final byte[] full = header(MAGIC, hash); @@ -151,23 +156,17 @@ void unknownDictionaryThrowsWhenHeaderSplitAcrossTwoBuffers() { final ByteBuffer second = ByteBuffer.wrap(full, 20, full.length - 20).slice(); final IOException ex = assertThrows(IOException.class, () -> consumer.consume(second)); - assertTrue(ex.getMessage().contains("Compression dictionary not available"), ex.getMessage()); + assertTrue(ex.getMessage().contains("does not use the negotiated dictionary"), ex.getMessage()); verifyNoInteractions(downstream); } @Test - void knownHashPresentAndMatchingPassesLookupAndReachesDecoderInit() { - // A real store keyed by the dictionary's own SHA-256; the header hash - // matches, so the lookup succeeds and control proceeds to lazy decoder - // creation. Without the native library that step throws, but the lookup - // guard itself must be cleared (message must NOT be about availability). + void knownHashPresentAndMatchingReachesDecoderInit() { final AsyncDataConsumer downstream = mock(AsyncDataConsumer.class); - final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); final CompressionDictionary dict = dictionary(new byte[]{'h', 'e', 'l', 'l', 'o'}); - store.add(dict); final InflatingDictionaryBrotliDataConsumer consumer = - new InflatingDictionaryBrotliDataConsumer(downstream, store); + new InflatingDictionaryBrotliDataConsumer(downstream, dict); final ByteBuffer src = ByteBuffer.wrap(header(MAGIC, dict.getSha256())); @@ -175,8 +174,7 @@ void knownHashPresentAndMatchingPassesLookupAndReachesDecoderInit() { consumer.consume(src); // If the native library is present, header consumed without error. } catch (final IOException ex) { - // If native init fails, it must not be the availability guard. - assertTrue(!ex.getMessage().contains("Compression dictionary not available"), ex.getMessage()); + assertTrue(!ex.getMessage().contains("does not use the negotiated dictionary"), ex.getMessage()); } catch (final Throwable linkError) { // UnsatisfiedLinkError / NoClassDefFoundError when native lib absent. assertTrue(true); @@ -185,31 +183,27 @@ void knownHashPresentAndMatchingPassesLookupAndReachesDecoderInit() { @Test void hashPresentButNotMatchingThrows() { - // Store returns a dictionary for any hash, but its SHA-256 differs from - // the header hash, so matchesHash() fails. final AsyncDataConsumer downstream = mock(AsyncDataConsumer.class); final CompressionDictionary dict = dictionary(new byte[]{1, 2, 3, 4}); - final CompressionDictionaryStore store = mock(CompressionDictionaryStore.class); - when(store.getByHash(org.mockito.ArgumentMatchers.any())).thenReturn(dict); final InflatingDictionaryBrotliDataConsumer consumer = - new InflatingDictionaryBrotliDataConsumer(downstream, store); + new InflatingDictionaryBrotliDataConsumer(downstream, dict); // Header hash of all-zeros will not equal the dictionary's real SHA-256. final byte[] hash = new byte[HASH_LENGTH]; final ByteBuffer src = ByteBuffer.wrap(header(MAGIC, hash)); final IOException ex = assertThrows(IOException.class, () -> consumer.consume(src)); - assertTrue(ex.getMessage().contains("Compression dictionary not available"), ex.getMessage()); + assertTrue(ex.getMessage().contains("does not use the negotiated dictionary"), ex.getMessage()); verifyNoInteractions(downstream); } @Test void truncatedHeaderThenStreamEndThrows() { final AsyncDataConsumer downstream = mock(AsyncDataConsumer.class); - final CompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final CompressionDictionary dictionary = dictionary(new byte[]{1}); final InflatingDictionaryBrotliDataConsumer consumer = - new InflatingDictionaryBrotliDataConsumer(downstream, store); + new InflatingDictionaryBrotliDataConsumer(downstream, dictionary); // Fewer than 36 bytes: header stays incomplete, decoder stays null. final byte[] partial = new byte[10]; @@ -226,9 +220,9 @@ void truncatedHeaderThenStreamEndThrows() { @Test void streamEndWithNoDataThrowsTruncatedHeader() { final AsyncDataConsumer downstream = mock(AsyncDataConsumer.class); - final CompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final CompressionDictionary dictionary = dictionary(new byte[]{1}); final InflatingDictionaryBrotliDataConsumer consumer = - new InflatingDictionaryBrotliDataConsumer(downstream, store); + new InflatingDictionaryBrotliDataConsumer(downstream, dictionary); final IOException ex = assertThrows(IOException.class, () -> consumer.streamEnd(Collections.emptyList())); @@ -236,6 +230,106 @@ void streamEndWithNoDataThrowsTruncatedHeader() { verifyNoInteractions(downstream); } + @Test + void roundTripWithSharedDictionary() throws Exception { + Assumptions.assumeTrue(brotliAvailable(), "Brotli native runtime is unavailable"); + + final byte[] dictionaryContent = + "the quick brown fox jumps over the lazy dog".getBytes(StandardCharsets.UTF_8); + final byte[] payload = ("the quick brown fox jumps over the lazy dog " + + "and then the quick brown fox runs away").getBytes(StandardCharsets.UTF_8); + final CompressionDictionary dictionary = dictionary(dictionaryContent); + final byte[] stream = concat(MAGIC, dictionary.getSha256(), compress(dictionaryContent, payload)); + final AccumulatingConsumer downstream = new AccumulatingConsumer(); + final InflatingDictionaryBrotliDataConsumer consumer = + new InflatingDictionaryBrotliDataConsumer(downstream, dictionary); + + for (int i = 0; i < stream.length; i++) { + consumer.consume(ByteBuffer.wrap(stream, i, 1)); + } + consumer.streamEnd(Collections.

emptyList()); + + assertArrayEquals(payload, downstream.toByteArray()); + assertTrue(downstream.ended); + consumer.releaseResources(); + } + + private static byte[] compress(final byte[] dictionary, final byte[] payload) throws IOException { + final ByteBuffer buffer = ByteBuffer.allocateDirect(dictionary.length); + buffer.put(dictionary).flip(); + final PreparedDictionary prepared = Encoder.prepareDictionary(buffer, 0); + final ByteArrayOutputStream compressed = new ByteArrayOutputStream(); + try { + final Encoder.Parameters parameters = Encoder.Parameters.create(6, 24, Encoder.Mode.TEXT); + try (BrotliOutputStream out = new BrotliOutputStream(compressed, parameters)) { + out.attachDictionary(prepared); + out.write(payload); + } + return compressed.toByteArray(); + } finally { + if (prepared instanceof AutoCloseable) { + try { + ((AutoCloseable) prepared).close(); + } catch (final Exception ex) { + throw new IOException("Unable to release Brotli dictionary", ex); + } + } + } + } + + private static byte[] concat(final byte[]... parts) { + int length = 0; + for (final byte[] part : parts) { + length += part.length; + } + final byte[] result = new byte[length]; + int offset = 0; + for (final byte[] part : parts) { + System.arraycopy(part, 0, result, offset, part.length); + offset += part.length; + } + return result; + } + + private static boolean brotliAvailable() { + try { + Brotli4jLoader.ensureAvailability(); + return true; + } catch (final Throwable ex) { + return false; + } + } + + private static final class AccumulatingConsumer implements AsyncDataConsumer { + private final ByteArrayOutputStream out = new ByteArrayOutputStream(); + private boolean ended; + + @Override + public void updateCapacity(final CapacityChannel capacityChannel) throws IOException { + capacityChannel.update(Integer.MAX_VALUE); + } + + @Override + public void consume(final ByteBuffer src) { + while (src.hasRemaining()) { + out.write(src.get()); + } + } + + @Override + public void streamEnd(final List trailers) throws HttpException, IOException { + ended = true; + } + + @Override + public void releaseResources() { + } + + byte[] toByteArray() { + return out.toByteArray(); + } + } + private static void assertDoesNotThrowConsume( final InflatingDictionaryBrotliDataConsumer consumer, final ByteBuffer src) { try { diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/TestInflatingDictionaryZstdDataConsumer.java b/httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/TestInflatingDictionaryZstdDataConsumer.java index 10f6a2ea35..defdc9e738 100644 --- a/httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/TestInflatingDictionaryZstdDataConsumer.java +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/async/methods/TestInflatingDictionaryZstdDataConsumer.java @@ -37,7 +37,6 @@ import com.github.luben.zstd.ZstdCompressCtx; -import org.apache.hc.client5.http.entity.compress.BasicCompressionDictionaryStore; import org.apache.hc.client5.http.entity.compress.CompressionDictionary; import org.apache.hc.client5.http.impl.ZstdRuntime; import org.apache.hc.core5.http.Header; @@ -130,9 +129,9 @@ void invalidMagicRaisesIOException() { Assumptions.assumeTrue(ZstdRuntime.available()); final AccumulatingConsumer downstream = new AccumulatingConsumer(); - final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final CompressionDictionary dictionary = dictionaryOf(new byte[]{1}); final InflatingDictionaryZstdDataConsumer consumer = - new InflatingDictionaryZstdDataConsumer(downstream, store); + new InflatingDictionaryZstdDataConsumer(downstream, dictionary); final byte[] header = new byte[MAGIC.length + HASH_LENGTH]; // deliberately wrong first byte @@ -146,13 +145,13 @@ void invalidMagicRaisesIOException() { } @Test - void unknownDictionaryHashRaisesIOException() { + void differentDictionaryHashRaisesIOException() { Assumptions.assumeTrue(ZstdRuntime.available()); final AccumulatingConsumer downstream = new AccumulatingConsumer(); - final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final CompressionDictionary dictionary = dictionaryOf(new byte[]{1}); final InflatingDictionaryZstdDataConsumer consumer = - new InflatingDictionaryZstdDataConsumer(downstream, store); + new InflatingDictionaryZstdDataConsumer(downstream, dictionary); final byte[] hash = new byte[HASH_LENGTH]; for (int i = 0; i < hash.length; i++) { @@ -162,7 +161,7 @@ void unknownDictionaryHashRaisesIOException() { final ByteBuffer src = ByteBuffer.wrap(header); final IOException ex = Assertions.assertThrows(IOException.class, () -> consumer.consume(src)); - Assertions.assertEquals("Compression dictionary not available", ex.getMessage()); + Assertions.assertEquals("DCZ stream does not use the negotiated dictionary", ex.getMessage()); consumer.releaseResources(); } @@ -172,9 +171,9 @@ void truncatedHeaderThenStreamEndRaisesIOException() { Assumptions.assumeTrue(ZstdRuntime.available()); final AccumulatingConsumer downstream = new AccumulatingConsumer(); - final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final CompressionDictionary dictionary = dictionaryOf(new byte[]{1}); final InflatingDictionaryZstdDataConsumer consumer = - new InflatingDictionaryZstdDataConsumer(downstream, store); + new InflatingDictionaryZstdDataConsumer(downstream, dictionary); // feed only part of the magic; header stays incomplete, so not initialized final ByteBuffer src = ByteBuffer.wrap(new byte[] {MAGIC[0], MAGIC[1], MAGIC[2]}); @@ -188,25 +187,25 @@ void truncatedHeaderThenStreamEndRaisesIOException() { } @Test - void headerSplitAcrossBuffersRaisesUnknownDictionary() { + void headerSplitAcrossBuffersRejectsDifferentDictionary() { Assumptions.assumeTrue(ZstdRuntime.available()); final AccumulatingConsumer downstream = new AccumulatingConsumer(); - final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final CompressionDictionary dictionary = dictionaryOf(new byte[]{1}); final InflatingDictionaryZstdDataConsumer consumer = - new InflatingDictionaryZstdDataConsumer(downstream, store); + new InflatingDictionaryZstdDataConsumer(downstream, dictionary); final byte[] hash = new byte[HASH_LENGTH]; final byte[] header = concat(MAGIC, hash); // split header across two ByteBuffers; the valid MAGIC still validates, - // then the (all-zero, absent) dictionary hash is rejected. + // then the all-zero hash is rejected because it is not the negotiated dictionary. final ByteBuffer first = ByteBuffer.wrap(header, 0, 5); final ByteBuffer second = ByteBuffer.wrap(header, 5, header.length - 5); Assertions.assertDoesNotThrow(() -> consumer.consume(first)); final IOException ex = Assertions.assertThrows(IOException.class, () -> consumer.consume(second)); - Assertions.assertEquals("Compression dictionary not available", ex.getMessage()); + Assertions.assertEquals("DCZ stream does not use the negotiated dictionary", ex.getMessage()); consumer.releaseResources(); } @@ -220,15 +219,13 @@ void roundTripSingleBuffer() throws Exception { + "and then the quick brown fox runs away").getBytes(StandardCharsets.UTF_8); final CompressionDictionary dictionary = dictionaryOf(dictionaryContent); - final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); - store.add(dictionary); final byte[] frame = compress(dictionaryContent, payload); final byte[] stream = concat(MAGIC, dictionary.getSha256(), frame); final AccumulatingConsumer downstream = new AccumulatingConsumer(); final InflatingDictionaryZstdDataConsumer consumer = - new InflatingDictionaryZstdDataConsumer(downstream, store); + new InflatingDictionaryZstdDataConsumer(downstream, dictionary); consumer.consume(ByteBuffer.wrap(stream)); consumer.streamEnd(Collections.
emptyList()); @@ -251,15 +248,13 @@ void roundTripSplitAcrossManyBuffers() throws Exception { final byte[] payload = sb.toString().getBytes(StandardCharsets.UTF_8); final CompressionDictionary dictionary = dictionaryOf(dictionaryContent); - final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); - store.add(dictionary); final byte[] frame = compress(dictionaryContent, payload); final byte[] stream = concat(MAGIC, dictionary.getSha256(), frame); final AccumulatingConsumer downstream = new AccumulatingConsumer(); final InflatingDictionaryZstdDataConsumer consumer = - new InflatingDictionaryZstdDataConsumer(downstream, store); + new InflatingDictionaryZstdDataConsumer(downstream, dictionary); // feed one byte at a time to exercise header/frame splitting for (int i = 0; i < stream.length; i++) { diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/entity/compress/TestBasicCompressionDictionaryStore.java b/httpclient5/src/test/java/org/apache/hc/client5/http/entity/compress/TestBasicCompressionDictionaryStore.java index d888322e84..228e446261 100644 --- a/httpclient5/src/test/java/org/apache/hc/client5/http/entity/compress/TestBasicCompressionDictionaryStore.java +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/entity/compress/TestBasicCompressionDictionaryStore.java @@ -42,12 +42,15 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import org.apache.hc.client5.http.cookie.BasicCookieStore; +import org.apache.hc.client5.http.cookie.CookieStore; import org.junit.jupiter.api.Test; class TestBasicCompressionDictionaryStore { private static final Instant STORED_AT = Instant.parse("2020-01-01T00:00:00Z"); private static final Instant VALID_UNTIL = Instant.parse("2030-01-01T00:00:00Z"); + private final CookieStore partition = new BasicCookieStore(); private static CompressionDictionary dictionary(final byte[] content, final URI source) { return new CompressionDictionary(content, source, "/path", "id", STORED_AT, VALID_UNTIL); @@ -60,19 +63,27 @@ private static CompressionDictionary dictionary(final int n) { @Test void addNullThrows() { final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); - assertThrows(NullPointerException.class, () -> store.add(null)); + assertThrows(NullPointerException.class, () -> store.add(null, dictionary(1))); + assertThrows(NullPointerException.class, () -> store.add(partition, null)); } @Test void getByHashNullThrows() { final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); - assertThrows(NullPointerException.class, () -> store.getByHash(null)); + assertThrows(NullPointerException.class, + () -> store.getByHash(null, URI.create("https://example.com/"), new byte[32])); + assertThrows(NullPointerException.class, + () -> store.getByHash(partition, null, new byte[32])); + assertThrows(NullPointerException.class, + () -> store.getByHash(partition, URI.create("https://example.com/"), null)); } @Test void getByOriginNullThrows() { final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); - assertThrows(NullPointerException.class, () -> store.getByOrigin(null)); + assertThrows(NullPointerException.class, + () -> store.getByOrigin(null, URI.create("https://example.com/"))); + assertThrows(NullPointerException.class, () -> store.getByOrigin(partition, null)); } @Test @@ -85,24 +96,24 @@ void nonPositiveMaxEntriesThrows() { void defaultConstructorWorks() { final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); final CompressionDictionary dict = dictionary(1); - store.add(dict); - assertSame(dict, store.getByHash(dict.getSha256())); + store.add(partition, dict); + assertSame(dict, store.getByHash(partition, URI.create("https://example.com/"), dict.getSha256())); } @Test void addThenGetByHashReturnsDictionary() { final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); final CompressionDictionary dict = dictionary(42); - store.add(dict); - assertSame(dict, store.getByHash(dict.getSha256())); + store.add(partition, dict); + assertSame(dict, store.getByHash(partition, URI.create("https://example.com/"), dict.getSha256())); } @Test void getByHashUnknownReturnsNull() { final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); - store.add(dictionary(1)); + store.add(partition, dictionary(1)); final CompressionDictionary other = dictionary(99); - assertNull(store.getByHash(other.getSha256())); + assertNull(store.getByHash(partition, URI.create("https://example.com/"), other.getSha256())); } @Test @@ -111,10 +122,10 @@ void sameHashTwiceKeepsSingleEntry() { final URI source = URI.create("https://example.com/"); final CompressionDictionary first = dictionary(new byte[]{7}, source); final CompressionDictionary second = dictionary(new byte[]{7}, source); - store.add(first); - store.add(second); - assertSame(second, store.getByHash(first.getSha256())); - assertEquals(1, store.getByOrigin(source).size()); + store.add(partition, first); + store.add(partition, second); + assertSame(second, store.getByHash(partition, URI.create("https://example.com/"), first.getSha256())); + assertEquals(1, store.getByOrigin(partition, source).size()); } @Test @@ -123,24 +134,12 @@ void evictionRemovesOldest() { final CompressionDictionary first = dictionary(1); final CompressionDictionary second = dictionary(2); final CompressionDictionary third = dictionary(3); - store.add(first); - store.add(second); - store.add(third); - assertNull(store.getByHash(first.getSha256())); - assertSame(second, store.getByHash(second.getSha256())); - assertSame(third, store.getByHash(third.getSha256())); - } - - @Test - void getByOriginFiltersByScheme() { - final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); - final CompressionDictionary https = dictionary(new byte[]{1}, URI.create("https://example.com/")); - final CompressionDictionary http = dictionary(new byte[]{2}, URI.create("http://example.com/")); - store.add(https); - store.add(http); - final List result = store.getByOrigin(URI.create("https://example.com/other")); - assertEquals(1, result.size()); - assertSame(https, result.get(0)); + store.add(partition, first); + store.add(partition, second); + store.add(partition, third); + assertNull(store.getByHash(partition, URI.create("https://example.com/"), first.getSha256())); + assertSame(second, store.getByHash(partition, URI.create("https://example.com/"), second.getSha256())); + assertSame(third, store.getByHash(partition, URI.create("https://example.com/"), third.getSha256())); } @Test @@ -148,9 +147,9 @@ void getByOriginFiltersByHost() { final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); final CompressionDictionary a = dictionary(new byte[]{1}, URI.create("https://a.example.com/")); final CompressionDictionary b = dictionary(new byte[]{2}, URI.create("https://b.example.com/")); - store.add(a); - store.add(b); - final List result = store.getByOrigin(URI.create("https://b.example.com/")); + store.add(partition, a); + store.add(partition, b); + final List result = store.getByOrigin(partition, URI.create("https://b.example.com/")); assertEquals(1, result.size()); assertSame(b, result.get(0)); } @@ -159,39 +158,30 @@ void getByOriginFiltersByHost() { void getByOriginTreatsHttpsDefaultPortAsEqual() { final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); final CompressionDictionary implicit = dictionary(new byte[]{1}, URI.create("https://example.com/")); - store.add(implicit); - final List result = store.getByOrigin(URI.create("https://example.com:443/")); + store.add(partition, implicit); + final List result = store.getByOrigin(partition, URI.create("https://example.com:443/")); assertEquals(1, result.size()); assertSame(implicit, result.get(0)); } - @Test - void getByOriginTreatsHttpDefaultPortAsEqual() { - final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); - final CompressionDictionary explicit = dictionary(new byte[]{1}, URI.create("http://example.com:80/")); - store.add(explicit); - final List result = store.getByOrigin(URI.create("http://example.com/")); - assertEquals(1, result.size()); - assertSame(explicit, result.get(0)); - } - @Test void getByOriginDistinguishesNonDefaultPort() { final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); final CompressionDictionary def = dictionary(new byte[]{1}, URI.create("https://example.com/")); final CompressionDictionary alt = dictionary(new byte[]{2}, URI.create("https://example.com:8443/")); - store.add(def); - store.add(alt); - assertEquals(1, store.getByOrigin(URI.create("https://example.com:8443/")).size()); - assertSame(alt, store.getByOrigin(URI.create("https://example.com:8443/")).get(0)); - assertSame(def, store.getByOrigin(URI.create("https://example.com/")).get(0)); + store.add(partition, def); + store.add(partition, alt); + assertEquals(1, store.getByOrigin(partition, URI.create("https://example.com:8443/")).size()); + assertSame(alt, store.getByOrigin(partition, URI.create("https://example.com:8443/")).get(0)); + assertSame(def, store.getByOrigin(partition, URI.create("https://example.com/")).get(0)); } @Test void getByOriginNoMatchReturnsEmpty() { final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); - store.add(dictionary(new byte[]{1}, URI.create("https://example.com/"))); - final List result = store.getByOrigin(URI.create("https://other.example.com/")); + store.add(partition, dictionary(new byte[]{1}, URI.create("https://example.com/"))); + final List result = + store.getByOrigin(partition, URI.create("https://other.example.com/")); assertTrue(result.isEmpty()); } @@ -199,10 +189,28 @@ void getByOriginNoMatchReturnsEmpty() { void clearEmptiesStore() { final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); final CompressionDictionary dict = dictionary(1); - store.add(dict); + store.add(partition, dict); store.clear(); - assertNull(store.getByHash(dict.getSha256())); - assertTrue(store.getByOrigin(URI.create("https://example.com/")).isEmpty()); + assertNull(store.getByHash(partition, URI.create("https://example.com/"), dict.getSha256())); + assertTrue(store.getByOrigin(partition, URI.create("https://example.com/")).isEmpty()); + } + + @Test + void partitionsAreIsolatedAndCanBeClearedIndependently() { + final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); + final CookieStore otherPartition = new BasicCookieStore(); + final CompressionDictionary first = dictionary(1); + final CompressionDictionary second = dictionary(2); + store.add(partition, first); + store.add(otherPartition, second); + + assertSame(first, store.getByHash(partition, first.getSource(), first.getSha256())); + assertNull(store.getByHash(otherPartition, first.getSource(), first.getSha256())); + assertSame(second, store.getByHash(otherPartition, second.getSource(), second.getSha256())); + + store.clear(partition); + assertTrue(store.getByOrigin(partition, first.getSource()).isEmpty()); + assertSame(second, store.getByHash(otherPartition, second.getSource(), second.getSha256())); } @Test @@ -223,7 +231,7 @@ void concurrentAddsStayWithinMaxEntries() throws InterruptedException { start.await(); for (int i = 0; i < perThread; i++) { final byte[] content = new byte[]{(byte) base, (byte) i}; - store.add(dictionary(content, URI.create("https://example.com/"))); + store.add(partition, dictionary(content, URI.create("https://example.com/"))); } } catch (final Throwable ex) { failure.compareAndSet(null, ex); @@ -238,14 +246,14 @@ void concurrentAddsStayWithinMaxEntries() throws InterruptedException { executor.shutdownNow(); } assertNull(failure.get(), "concurrent add threw: " + failure.get()); - assertTrue(store.getByOrigin(URI.create("https://example.com/")).size() <= maxEntries); + assertTrue(store.getByOrigin(partition, URI.create("https://example.com/")).size() <= maxEntries); } @Test void getByHashReturnsNonNullAfterAdd() { final BasicCompressionDictionaryStore store = new BasicCompressionDictionaryStore(); final CompressionDictionary dict = dictionary(5); - store.add(dict); - assertNotNull(store.getByHash(dict.getSha256())); + store.add(partition, dict); + assertNotNull(store.getByHash(partition, URI.create("https://example.com/"), dict.getSha256())); } } diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/entity/compress/TestCompressionDictionary.java b/httpclient5/src/test/java/org/apache/hc/client5/http/entity/compress/TestCompressionDictionary.java index 6c0520145d..7341bad361 100644 --- a/httpclient5/src/test/java/org/apache/hc/client5/http/entity/compress/TestCompressionDictionary.java +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/entity/compress/TestCompressionDictionary.java @@ -38,6 +38,7 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.Instant; +import java.util.Arrays; import org.junit.jupiter.api.Test; @@ -86,6 +87,43 @@ void constructorRejectsBlankMatch() { new CompressionDictionary(CONTENT, SOURCE, " ", ID, STORED_AT, VALID_UNTIL)); } + @Test + void constructorRejectsNonHttpsOrRelativeSource() { + assertThrows(IllegalArgumentException.class, () -> + new CompressionDictionary(CONTENT, URI.create("http://example.com/dict"), + MATCH, ID, STORED_AT, VALID_UNTIL)); + assertThrows(IllegalArgumentException.class, () -> + new CompressionDictionary(CONTENT, URI.create("/dict"), + MATCH, ID, STORED_AT, VALID_UNTIL)); + } + + @Test + void constructorRejectsInvalidStructuredFieldStrings() { + assertThrows(IllegalArgumentException.class, () -> + new CompressionDictionary(CONTENT, SOURCE, "/caf\u00e9", ID, STORED_AT, VALID_UNTIL)); + assertThrows(IllegalArgumentException.class, () -> + new CompressionDictionary(CONTENT, SOURCE, MATCH, "bad\nid", STORED_AT, VALID_UNTIL)); + assertThrows(IllegalArgumentException.class, () -> + new CompressionDictionary(CONTENT, SOURCE, MATCH, + Arrays.asList("document", null), ID, "raw", STORED_AT, VALID_UNTIL)); + } + + @Test + void constructorRejectsOversizedId() { + final char[] chars = new char[1025]; + Arrays.fill(chars, 'x'); + assertThrows(IllegalArgumentException.class, () -> + new CompressionDictionary(CONTENT, SOURCE, MATCH, + new String(chars), STORED_AT, VALID_UNTIL)); + } + + @Test + void constructorRejectsInvalidTypeToken() { + assertThrows(IllegalArgumentException.class, () -> + new CompressionDictionary(CONTENT, SOURCE, MATCH, + null, ID, "raw type", STORED_AT, VALID_UNTIL)); + } + @Test void constructorRejectsNullStoredAt() { assertThrows(NullPointerException.class, () -> diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/examples/AsyncClientCompressionDictionary.java b/httpclient5/src/test/java/org/apache/hc/client5/http/examples/AsyncClientCompressionDictionary.java index d92dba3624..e29d2e953a 100644 --- a/httpclient5/src/test/java/org/apache/hc/client5/http/examples/AsyncClientCompressionDictionary.java +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/examples/AsyncClientCompressionDictionary.java @@ -37,6 +37,7 @@ import org.apache.hc.client5.http.entity.compress.BasicCompressionDictionaryStore; import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; import org.apache.hc.client5.http.impl.async.HttpAsyncClients; +import org.apache.hc.client5.http.protocol.HttpClientContext; import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpHeaders; import org.apache.hc.core5.http.HttpResponse; @@ -68,6 +69,7 @@ public final class AsyncClientCompressionDictionary { public static void main(final String[] args) throws Exception { final BasicCompressionDictionaryStore dictionaryStore = new BasicCompressionDictionaryStore(); + final HttpClientContext clientContext = HttpClientContext.create(); try (final CloseableHttpAsyncClient client = HttpAsyncClients.custom() .setCompressionDictionaryStore(dictionaryStore) @@ -76,7 +78,7 @@ public static void main(final String[] args) throws Exception { client.start(); final Message manifestResponse = - execute(client, MANIFEST_URI); + execute(client, clientContext, MANIFEST_URI); final String manifest = new String( manifestResponse.getBody(), @@ -107,7 +109,7 @@ public static void main(final String[] args) throws Exception { System.out.println("Fetching dictionary..."); final Message dictionaryResponse = - execute(client, dictionaryUri); + execute(client, clientContext, dictionaryUri); final HttpResponse dictionaryHead = dictionaryResponse.getHead(); @@ -127,13 +129,14 @@ public static void main(final String[] args) throws Exception { : 0)); System.out.println("Stored dictionaries: " - + dictionaryStore.getByOrigin(dictionaryUri).size()); + + dictionaryStore.getByOrigin( + clientContext.getCookieStore(), dictionaryUri).size()); System.out.println(); System.out.println("Fetching current resource..."); final Message resourceResponse = - execute(client, resourceUri); + execute(client, clientContext, resourceUri); final HttpResponse resourceHead = resourceResponse.getHead(); @@ -163,6 +166,7 @@ public static void main(final String[] args) throws Exception { private static Message execute( final CloseableHttpAsyncClient client, + final HttpClientContext context, final URI uri) throws Exception { final SimpleHttpRequest request = SimpleRequestBuilder.get(uri) @@ -171,6 +175,7 @@ private static Message execute( final Future> future = client.execute( new BasicRequestProducer(request, null), new BasicResponseConsumer<>(new BasicAsyncEntityConsumer()), + context, null); return future.get(); @@ -215,4 +220,4 @@ private static List extractDeployPaths(final String manifest) { return paths; } -} \ No newline at end of file +} diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/examples/AsyncClientServerDictionaryBrotli.java b/httpclient5/src/test/java/org/apache/hc/client5/http/examples/AsyncClientServerDictionaryBrotli.java index f3ab55ecc7..52ade1bd0b 100644 --- a/httpclient5/src/test/java/org/apache/hc/client5/http/examples/AsyncClientServerDictionaryBrotli.java +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/examples/AsyncClientServerDictionaryBrotli.java @@ -59,6 +59,7 @@ import org.apache.hc.client5.http.impl.async.HttpAsyncClients; import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager; import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.client5.http.protocol.HttpClientContext; import org.apache.hc.client5.http.ssl.DefaultClientTlsStrategy; import org.apache.hc.core5.http.ClassicHttpRequest; import org.apache.hc.core5.http.ClassicHttpResponse; @@ -212,6 +213,7 @@ public static void main(final String[] args) throws Exception { final BasicCompressionDictionaryStore dictionaryStore = new BasicCompressionDictionaryStore(); + final HttpClientContext clientContext = HttpClientContext.create(); final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder.create() @@ -229,7 +231,7 @@ public static void main(final String[] args) throws Exception { System.out.println(" " + dictionaryUri); final Message dictionaryResponse = - execute(client, dictionaryUri); + execute(client, clientContext, dictionaryUri); final HttpResponse dictionaryHead = dictionaryResponse.getHead(); @@ -249,14 +251,15 @@ public static void main(final String[] args) throws Exception { + dictionaryResponse.getBody().length); System.out.println("Stored dictionaries: " - + dictionaryStore.getByOrigin(dictionaryUri).size()); + + dictionaryStore.getByOrigin( + clientContext.getCookieStore(), dictionaryUri).size()); System.out.println(); System.out.println("Fetching DCB resource:"); System.out.println(" " + resourceUri); final Message resourceResponse = - execute(client, resourceUri); + execute(client, clientContext, resourceUri); final HttpResponse resourceHead = resourceResponse.getHead(); @@ -299,6 +302,7 @@ public static void main(final String[] args) throws Exception { private static Message execute( final CloseableHttpAsyncClient client, + final HttpClientContext context, final URI uri) throws Exception { final SimpleHttpRequest request = @@ -309,6 +313,7 @@ private static Message execute( new BasicRequestProducer(request, null), new BasicResponseConsumer<>( new BasicAsyncEntityConsumer()), + context, null); return future.get(); @@ -588,4 +593,4 @@ private static SSLContext createClientSslContext( return sslContext; } -} \ No newline at end of file +} diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestCompressionDictionaryCookieStore.java b/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestCompressionDictionaryCookieStore.java new file mode 100644 index 0000000000..8d8980f608 --- /dev/null +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestCompressionDictionaryCookieStore.java @@ -0,0 +1,69 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.impl.async; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.time.Instant; + +import org.apache.hc.client5.http.cookie.BasicCookieStore; +import org.apache.hc.client5.http.cookie.CookieStore; +import org.apache.hc.client5.http.entity.compress.BasicCompressionDictionaryStore; +import org.apache.hc.client5.http.entity.compress.CompressionDictionary; +import org.apache.hc.client5.http.impl.cookie.BasicClientCookie; +import org.junit.jupiter.api.Test; + +class TestCompressionDictionaryCookieStore { + + @Test + void testClearRemovesCookiesAndCompressionDictionaries() { + final BasicCompressionDictionaryStore dictionaryStore = + new BasicCompressionDictionaryStore(); + final CookieStore cookieStore = new CompressionDictionaryCookieStore( + new BasicCookieStore(), dictionaryStore); + cookieStore.addCookie(new BasicClientCookie("name", "value")); + final Instant now = Instant.now(); + final URI source = URI.create("https://example.com/dictionary"); + dictionaryStore.add(cookieStore, new CompressionDictionary( + "dictionary".getBytes(StandardCharsets.UTF_8), + source, + "/*", + "", + now, + now.plusSeconds(60))); + assertFalse(cookieStore.getCookies().isEmpty()); + + cookieStore.clear(); + + assertTrue(cookieStore.getCookies().isEmpty()); + assertTrue(dictionaryStore.getByOrigin(cookieStore, source).isEmpty()); + } + +} diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestCompressionDictionaryFreshness.java b/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestCompressionDictionaryFreshness.java new file mode 100644 index 0000000000..589d578fe6 --- /dev/null +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestCompressionDictionaryFreshness.java @@ -0,0 +1,120 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.hc.client5.http.impl.async; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.time.Instant; + +import org.apache.hc.core5.http.HttpHeaders; +import org.apache.hc.core5.http.message.BasicHttpResponse; +import org.junit.jupiter.api.Test; + +class TestCompressionDictionaryFreshness { + + private static final Instant REQUEST_TIME = Instant.parse("2026-08-21T11:59:58Z"); + private static final Instant RESPONSE_TIME = Instant.parse("2026-08-21T12:00:00Z"); + + @Test + void testMaxAgeUsesCorrectedInitialAge() { + final BasicHttpResponse response = new BasicHttpResponse(200); + response.addHeader(HttpHeaders.CACHE_CONTROL, "max-age=60"); + response.addHeader(HttpHeaders.DATE, "Fri, 21 Aug 2026 11:59:50 GMT"); + response.addHeader(HttpHeaders.AGE, "5"); + + assertEquals(Instant.parse("2026-08-21T12:00:50Z"), + CompressionDictionaryFreshness.determineValidUntil( + response, REQUEST_TIME, RESPONSE_TIME)); + } + + @Test + void testExpiresSuppliesFreshnessLifetime() { + final BasicHttpResponse response = new BasicHttpResponse(200); + response.addHeader(HttpHeaders.DATE, "Fri, 21 Aug 2026 12:00:00 GMT"); + response.addHeader(HttpHeaders.EXPIRES, "Fri, 21 Aug 2026 12:01:00 GMT"); + + assertEquals(Instant.parse("2026-08-21T12:00:58Z"), + CompressionDictionaryFreshness.determineValidUntil( + response, REQUEST_TIME, RESPONSE_TIME)); + } + + @Test + void testObsoleteHttpDateFormatsAreAccepted() { + final BasicHttpResponse rfc850 = new BasicHttpResponse(200); + rfc850.addHeader(HttpHeaders.DATE, "Friday, 21-Aug-26 12:00:00 GMT"); + rfc850.addHeader(HttpHeaders.EXPIRES, "Friday, 21-Aug-26 12:01:00 GMT"); + assertEquals(Instant.parse("2026-08-21T12:00:58Z"), + CompressionDictionaryFreshness.determineValidUntil( + rfc850, REQUEST_TIME, RESPONSE_TIME)); + + final BasicHttpResponse asctime = new BasicHttpResponse(200); + asctime.addHeader(HttpHeaders.DATE, "Fri Aug 21 12:00:00 2026"); + asctime.addHeader(HttpHeaders.EXPIRES, "Fri Aug 21 12:01:00 2026"); + assertEquals(Instant.parse("2026-08-21T12:00:58Z"), + CompressionDictionaryFreshness.determineValidUntil( + asctime, REQUEST_TIME, RESPONSE_TIME)); + } + + @Test + void testNoStoreIsNotStorable() { + final BasicHttpResponse response = new BasicHttpResponse(200); + response.addHeader(HttpHeaders.CACHE_CONTROL, "max-age=60, no-store"); + + assertNull(CompressionDictionaryFreshness.determineValidUntil( + response, REQUEST_TIME, RESPONSE_TIME)); + } + + @Test + void testNoCacheIsNotFresh() { + final BasicHttpResponse response = new BasicHttpResponse(200); + response.addHeader(HttpHeaders.CACHE_CONTROL, "max-age=60, no-cache"); + + assertNull(CompressionDictionaryFreshness.determineValidUntil( + response, REQUEST_TIME, RESPONSE_TIME)); + } + + @Test + void testMalformedMaxAgeFailsClosed() { + final BasicHttpResponse response = new BasicHttpResponse(200); + response.addHeader(HttpHeaders.CACHE_CONTROL, "max-age=invalid"); + + assertNull(CompressionDictionaryFreshness.determineValidUntil( + response, REQUEST_TIME, RESPONSE_TIME)); + } + + @Test + void testAlreadyStaleResponseIsNotEligible() { + final BasicHttpResponse response = new BasicHttpResponse(200); + response.addHeader(HttpHeaders.CACHE_CONTROL, "max-age=5"); + response.addHeader(HttpHeaders.DATE, "Fri, 21 Aug 2026 11:59:50 GMT"); + + assertNull(CompressionDictionaryFreshness.determineValidUntil( + response, REQUEST_TIME, RESPONSE_TIME)); + } + +} diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestContentCompressionAsyncExecDictionary.java b/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestContentCompressionAsyncExecDictionary.java index 15a585a7f8..7d55d8c697 100644 --- a/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestContentCompressionAsyncExecDictionary.java +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestContentCompressionAsyncExecDictionary.java @@ -45,13 +45,17 @@ import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.Base64; +import java.util.LinkedHashMap; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.UnaryOperator; import org.apache.hc.client5.http.HttpRoute; import org.apache.hc.client5.http.async.AsyncExecCallback; import org.apache.hc.client5.http.async.AsyncExecChain; import org.apache.hc.client5.http.async.AsyncExecRuntime; +import org.apache.hc.client5.http.cookie.BasicCookieStore; +import org.apache.hc.client5.http.cookie.CookieStore; import org.apache.hc.client5.http.entity.compress.BasicCompressionDictionaryStore; import org.apache.hc.client5.http.entity.compress.CompressionDictionary; import org.apache.hc.client5.http.entity.compress.CompressionDictionaryStore; @@ -99,6 +103,7 @@ class TestContentCompressionAsyncExecDictionary { private HttpClientContext context; private AsyncExecChain.Scope scope; private BasicCompressionDictionaryStore store; + private CookieStore cookieStore; private ContentCompressionAsyncExec impl; @BeforeEach @@ -108,7 +113,6 @@ void init() { final HttpHost target = new HttpHost("https", "example.com", 443); final HttpRequest originalRequest = new BasicHttpRequest(Method.GET, "/"); context = HttpClientContext.create(); - scope = new AsyncExecChain.Scope( "test", new HttpRoute(target), @@ -120,6 +124,8 @@ void init() { new AtomicInteger()); store = new BasicCompressionDictionaryStore(); + cookieStore = new CompressionDictionaryCookieStore(new BasicCookieStore(), store); + context.setCookieStore(cookieStore); impl = new ContentCompressionAsyncExec((CompressionDictionaryStore) store); } @@ -148,7 +154,7 @@ private static String expectedAvailableDictionary(final CompressionDictionary di @Test void testFreshDictionaryAddsAvailableDictionaryHeader() throws Exception { final CompressionDictionary dictionary = freshDictionary("/*", "dict-1"); - store.add(dictionary); + store.add(cookieStore, dictionary); final HttpRequest request = new BasicHttpRequest(Method.GET, URI.create(ORIGIN + "/page.html")); executeAndCapture(request); @@ -160,7 +166,7 @@ void testFreshDictionaryAddsAvailableDictionaryHeader() throws Exception { @Test void testFreshDictionaryAddsDictionaryIdHeaderWhenIdPresent() throws Exception { - store.add(freshDictionary("/*", "dict-1")); + store.add(cookieStore, freshDictionary("/*", "dict-1")); final HttpRequest request = new BasicHttpRequest(Method.GET, URI.create(ORIGIN + "/page.html")); executeAndCapture(request); @@ -171,7 +177,7 @@ void testFreshDictionaryAddsDictionaryIdHeaderWhenIdPresent() throws Exception { @Test void testDictionaryIdHeaderOmittedWhenIdEmpty() throws Exception { - store.add(freshDictionary("/*", "")); + store.add(cookieStore, freshDictionary("/*", "")); final HttpRequest request = new BasicHttpRequest(Method.GET, URI.create(ORIGIN + "/page.html")); executeAndCapture(request); @@ -182,7 +188,7 @@ void testDictionaryIdHeaderOmittedWhenIdEmpty() throws Exception { @Test void testAcceptEncodingIncludesDictionaryTokensWhenAvailable() throws Exception { - store.add(freshDictionary("/*", "dict-1")); + store.add(cookieStore, freshDictionary("/*", "dict-1")); final HttpRequest request = new BasicHttpRequest(Method.GET, URI.create(ORIGIN + "/page.html")); executeAndCapture(request); @@ -206,7 +212,7 @@ private static boolean tokenPresent(final String headerValue, final String token @Test void testNoDictionaryWhenRequestAlreadyHasAvailableDictionary() throws Exception { - store.add(freshDictionary("/*", "dict-1")); + store.add(cookieStore, freshDictionary("/*", "dict-1")); final HttpRequest request = new BasicHttpRequest(Method.GET, URI.create(ORIGIN + "/page.html")); request.addHeader(AVAILABLE_DICTIONARY, ":preset:"); @@ -224,7 +230,7 @@ void testNoDictionaryWhenRequestAlreadyHasAvailableDictionary() throws Exception @Test void testNoDictionaryWhenRequestUriNotHttps() throws Exception { - store.add(freshDictionary("/*", "dict-1")); + store.add(cookieStore, freshDictionary("/*", "dict-1")); final HttpRequest request = new BasicHttpRequest(Method.GET, URI.create("http://example.com/page.html")); executeAndCapture(request); @@ -292,12 +298,36 @@ void testUseAsDictionaryWrapsDownstreamAndCapturesBody() throws Exception { assertTrue(downstream.ended); - final List stored = store.getByOrigin(URI.create(ORIGIN + "/page.html")); + final List stored = + store.getByOrigin(cookieStore, URI.create(ORIGIN + "/page.html")); assertEquals(1, stored.size()); assertEquals("/page.html", stored.get(0).getMatch()); assertArrayEquals(body, stored.get(0).getContent()); } + @Test + void testUseAsDictionaryCombinesMultipleFieldLines() throws Exception { + final HttpRequest request = new BasicHttpRequest(Method.GET, URI.create(ORIGIN + "/page.html")); + final AsyncExecCallback cb = executeAndCapture(request); + final HttpResponse rsp = new BasicHttpResponse(200, "OK"); + rsp.addHeader(USE_AS_DICTIONARY, "match=\"/page.html\""); + rsp.addHeader(USE_AS_DICTIONARY, "id=\"split-field\""); + rsp.addHeader(HttpHeaders.CACHE_CONTROL, "max-age=3600"); + final EntityDetails details = mock(EntityDetails.class); + when(details.getContentEncoding()).thenReturn(null); + final DrainingConsumer downstream = new DrainingConsumer(); + when(originalCb.handleResponse(same(rsp), any(EntityDetails.class))).thenReturn(downstream); + + final AsyncDataConsumer wrapped = cb.handleResponse(rsp, details); + wrapped.consume(ByteBuffer.wrap("dictionary".getBytes(StandardCharsets.UTF_8))); + wrapped.streamEnd(null); + + final List stored = + store.getByOrigin(cookieStore, URI.create(ORIGIN + "/page.html")); + assertEquals(1, stored.size()); + assertEquals("split-field", stored.get(0).getId()); + } + @Test void testUseAsDictionaryNotWrappedWithoutFreshnessDirective() throws Exception { final HttpRequest request = new BasicHttpRequest(Method.GET, URI.create(ORIGIN + "/page.html")); @@ -318,6 +348,63 @@ void testUseAsDictionaryNotWrappedWithoutFreshnessDirective() throws Exception { assertSame(downstream, wrapped); } + @Test + void testDictionaryIsNotVisibleFromAnotherCookiePartition() throws Exception { + final CompressionDictionary dictionary = freshDictionary("/*", "dict-1"); + store.add(cookieStore, dictionary); + context.setCookieStore(new CompressionDictionaryCookieStore(new BasicCookieStore(), store)); + + final HttpRequest request = new BasicHttpRequest(Method.GET, URI.create(ORIGIN + "/page.html")); + executeAndCapture(request); + + assertFalse(request.containsHeader(AVAILABLE_DICTIONARY)); + assertFalse(request.containsHeader(DICTIONARY_ID)); + } + + @Test + void testClearingCookiesClearsDictionaryPartition() throws Exception { + final CompressionDictionary dictionary = freshDictionary("/*", "dict-1"); + store.add(cookieStore, dictionary); + final HttpRequest request = new BasicHttpRequest(Method.GET, URI.create(ORIGIN + "/page.html")); + executeAndCapture(request); + + cookieStore.clear(); + + assertTrue(store.getByOrigin(cookieStore, URI.create(ORIGIN)).isEmpty()); + } + + @Test + void testUnmanagedCookieStoreDisablesDictionaryTransport() throws Exception { + final CookieStore unmanagedPartition = new BasicCookieStore(); + context.setCookieStore(unmanagedPartition); + store.add(unmanagedPartition, freshDictionary("/*", "dict-1")); + + final HttpRequest request = new BasicHttpRequest(Method.GET, URI.create(ORIGIN + "/page.html")); + executeAndCapture(request); + + assertFalse(request.containsHeader(AVAILABLE_DICTIONARY)); + assertFalse(request.containsHeader(DICTIONARY_ID)); + } + + @Test + void testCustomDcbDecoderIsUsed() throws Exception { + final LinkedHashMap> decoders = new LinkedHashMap<>(); + final AsyncDataConsumer custom = mock(AsyncDataConsumer.class); + decoders.put("DCB", downstream -> custom); + impl = new ContentCompressionAsyncExec(decoders, store); + store.add(cookieStore, freshDictionary("/*", "dict-1")); + + final HttpRequest request = new BasicHttpRequest(Method.GET, URI.create(ORIGIN + "/page.html")); + final AsyncExecCallback cb = executeAndCapture(request); + final HttpResponse rsp = new BasicHttpResponse(200, "OK"); + final EntityDetails details = mock(EntityDetails.class); + when(details.getContentEncoding()).thenReturn("dcb"); + when(originalCb.handleResponse(same(rsp), any(EntityDetails.class))) + .thenReturn(mock(AsyncDataConsumer.class)); + + assertSame(custom, cb.handleResponse(rsp, details)); + } + private static final class DrainingConsumer implements AsyncDataConsumer { private boolean ended; diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestDefaultCompressionDictionaryMatcher.java b/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestDefaultCompressionDictionaryMatcher.java index 82611fe8dc..83a41685f3 100644 --- a/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestDefaultCompressionDictionaryMatcher.java +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestDefaultCompressionDictionaryMatcher.java @@ -29,6 +29,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.net.URI; import java.time.Clock; @@ -38,6 +39,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.List; import org.apache.hc.client5.http.entity.compress.CompressionDictionary; import org.junit.jupiter.api.BeforeEach; @@ -59,11 +61,22 @@ private static CompressionDictionary dictionary( final String match, final Instant storedAt, final Instant validUntil) { + return dictionary(source, match, Collections.emptyList(), storedAt, validUntil); + } + + private static CompressionDictionary dictionary( + final String source, + final String match, + final List matchDest, + final Instant storedAt, + final Instant validUntil) { return new CompressionDictionary( new byte[] {1, 2, 3}, URI.create(source), match, + matchDest, "id", + "raw", storedAt, validUntil); } @@ -113,13 +126,12 @@ void httpsSchemeIsCaseInsensitive() { } @Test - void nonHttpsDictionarySourceIsNotMatched() { - final CompressionDictionary dictionary = dictionary( + void nonHttpsDictionarySourceIsRejected() { + assertThrows(IllegalArgumentException.class, () -> dictionary( "http://example.com/", "/*", NOW.minus(Duration.ofMinutes(1)), - NOW.plus(Duration.ofHours(1))); - assertNull(matcher.match(URI.create("https://example.com/app"), list(dictionary))); + NOW.plus(Duration.ofHours(1)))); } @Test @@ -175,21 +187,40 @@ void dictionaryExpiringAtTheClockInstantIsExcluded() { } @Test - void patternMustStartWithSlash() { + void relativePatternUsesOutboundRequestAsBase() { final CompressionDictionary dictionary = fresh("app/*"); assertNull(matcher.match(URI.create("https://example.com/app/x"), list(dictionary))); } @Test - void patternWithQueryCharNeverMatches() { + void relativePatternWithParentSegmentUsesOutboundRequestAsBase() { + final CompressionDictionary dictionary = fresh("../app/*"); + assertSame(dictionary, matcher.match(URI.create("https://example.com/app/x"), list(dictionary))); + } + + @Test + void patternWithQueryMatchesQueryComponent() { final CompressionDictionary dictionary = fresh("/app?x"); - assertNull(matcher.match(URI.create("https://example.com/app?x"), list(dictionary))); + assertSame(dictionary, matcher.match(URI.create("https://example.com/app?x"), list(dictionary))); + } + + @Test + void wildcardQuestionMarkIsAGroupModifier() { + final CompressionDictionary dictionary = fresh("https://example.com/*?foo"); + assertNull(matcher.match(URI.create("https://example.com/?foo"), list(dictionary))); + } + + @Test + void escapedQuestionMarkSeparatesTheSearchComponent() { + final CompressionDictionary dictionary = fresh("https://example.com/*\\?foo"); + assertSame(dictionary, + matcher.match(URI.create("https://example.com/?foo"), list(dictionary))); } @Test - void patternWithFragmentCharNeverMatches() { + void patternWithFragmentMatchesFragmentComponent() { final CompressionDictionary dictionary = fresh("/app#x"); - assertNull(matcher.match(URI.create("https://example.com/app"), list(dictionary))); + assertSame(dictionary, matcher.match(URI.create("https://example.com/app#x"), list(dictionary))); } @Test @@ -199,17 +230,108 @@ void patternWithParenthesesNeverMatches() { } @Test - void patternWithBracesNeverMatches() { + void rfcPathPrefixExampleMatches() { + final CompressionDictionary dictionary = fresh("/product/*"); + assertSame(dictionary, + matcher.match(URI.create("https://example.com/product/123"), list(dictionary))); + } + + @Test + void rfcVersionedDirectoriesExampleMatches() { + final CompressionDictionary dictionary = fresh("/app/*/main.js"); + assertSame(dictionary, + matcher.match(URI.create("https://example.com/app/v1/main.js"), list(dictionary))); + } + + @Test + void percentEncodedPathMatchesAtHttpLevel() { + final CompressionDictionary dictionary = fresh("/d%C3%BCsseldorf"); + assertSame(dictionary, + matcher.match(URI.create("https://example.com/d%C3%BCsseldorf"), list(dictionary))); + } + + @Test + void unbalancedPatternBracesNeverMatch() { assertNull(matcher.match(URI.create("https://example.com/app"), list(fresh("/app{")))); assertNull(matcher.match(URI.create("https://example.com/app"), list(fresh("/app}")))); } + @Test + void optionalBraceGroupMatches() { + final CompressionDictionary dictionary = fresh("/app{/v1}?/*"); + assertSame(dictionary, + matcher.match(URI.create("https://example.com/app/v1/main.js"), list(dictionary))); + assertSame(dictionary, + matcher.match(URI.create("https://example.com/app/main.js"), list(dictionary))); + } + @Test void patternWithColonNeverMatches() { final CompressionDictionary dictionary = fresh("/app:x"); assertNull(matcher.match(URI.create("https://example.com/app"), list(dictionary))); } + @Test + void namedGroupMatchesOnePathSegment() { + final CompressionDictionary dictionary = fresh("/app/:name"); + assertSame(dictionary, + matcher.match(URI.create("https://example.com/app/main.js"), list(dictionary))); + assertNull(matcher.match(URI.create("https://example.com/app/a/main.js"), list(dictionary))); + } + + @Test + void absoluteSameOriginPatternMatches() { + final CompressionDictionary dictionary = fresh("https://example.com/app/*"); + assertSame(dictionary, + matcher.match(URI.create("https://example.com/app/main.js?version=1"), list(dictionary))); + } + + @Test + void absoluteCrossOriginPatternNeverMatches() { + final CompressionDictionary dictionary = fresh("https://other.example/app/*"); + assertNull(matcher.match(URI.create("https://example.com/app/main.js"), list(dictionary))); + } + + @Test + void absoluteCrossOriginPatternIsValidButCannotMatchDictionaryOrigin() { + final CompressionDictionaryUrlPatternMatcher patternMatcher = + new DefaultCompressionDictionaryUrlPatternMatcher(); + assertTrue(patternMatcher.isValid( + "https://other.example/app/*", URI.create("https://example.com/dictionary"))); + assertNull(matcher.match(URI.create("https://example.com/app/main.js"), + list(fresh("https://other.example/app/*")))); + } + + @Test + void patternedHostnameCanMatchWithinDictionaryOrigin() { + final CompressionDictionary dictionary = dictionary( + "https://www.example.com/", + "https://*.example.com/app/*", + NOW.minus(Duration.ofMinutes(1)), + NOW.plus(Duration.ofHours(1))); + assertSame(dictionary, + matcher.match(URI.create("https://www.example.com/app/main.js"), list(dictionary))); + } + + @Test + void bracedAuthorityPatternMatchesWithinDictionaryOrigin() { + final CompressionDictionary dictionary = fresh( + "https://{sub.}?example{.com/}foo"); + assertSame(dictionary, + matcher.match(URI.create("https://example.com/foo"), list(dictionary))); + } + + @Test + void escapedUserInfoDelimiterMatches() { + final CompressionDictionary dictionary = dictionary( + "https://foo:bar@example.com/dictionary", + "https://foo\\:bar@example.com", + NOW.minus(Duration.ofMinutes(1)), + NOW.plus(Duration.ofHours(1))); + assertSame(dictionary, + matcher.match(URI.create("https://foo:bar@example.com"), list(dictionary))); + } + @Test void globPrefixMatchesSubPath() { final CompressionDictionary dictionary = fresh("/app/*"); @@ -260,6 +382,44 @@ void longestMatchWins() { matcher.match(URI.create("https://example.com/app/main.js"), list(narrow, broad))); } + @Test + void matchingDestinationTakesPrecedence() { + final CompressionDictionary destinationAgnostic = fresh("/app/*"); + final CompressionDictionary destinationSpecific = dictionary( + "https://example.com/", + "/*", + Collections.singletonList("script"), + NOW.minus(Duration.ofMinutes(10)), + NOW.plus(Duration.ofHours(1))); + assertSame(destinationSpecific, + matcher.match(URI.create("https://example.com/app/main.js"), "script", + list(destinationAgnostic, destinationSpecific))); + } + + @Test + void nonMatchingDestinationIsExcluded() { + final CompressionDictionary dictionary = dictionary( + "https://example.com/", + "/*", + Collections.singletonList("document"), + NOW.minus(Duration.ofMinutes(10)), + NOW.plus(Duration.ofHours(1))); + assertNull(matcher.match( + URI.create("https://example.com/app/main.js"), "script", list(dictionary))); + } + + @Test + void unsupportedRequestDestinationsMatchAll() { + final CompressionDictionary dictionary = dictionary( + "https://example.com/", + "/*", + Collections.singletonList("document"), + NOW.minus(Duration.ofMinutes(10)), + NOW.plus(Duration.ofHours(1))); + assertSame(dictionary, + matcher.match(URI.create("https://example.com/app/main.js"), null, list(dictionary))); + } + @Test void equalLengthLaterStoredAtWins() { final CompressionDictionary older = dictionary( diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestDictionaryCapturingAsyncDataConsumer.java b/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestDictionaryCapturingAsyncDataConsumer.java index 205d336105..30b7c59671 100644 --- a/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestDictionaryCapturingAsyncDataConsumer.java +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestDictionaryCapturingAsyncDataConsumer.java @@ -30,6 +30,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -43,6 +45,8 @@ import java.util.Collections; import java.util.List; +import org.apache.hc.client5.http.cookie.BasicCookieStore; +import org.apache.hc.client5.http.cookie.CookieStore; import org.apache.hc.client5.http.entity.compress.BasicCompressionDictionaryStore; import org.apache.hc.client5.http.entity.compress.CompressionDictionary; import org.apache.hc.client5.http.entity.compress.CompressionDictionaryStore; @@ -61,11 +65,13 @@ class TestDictionaryCapturingAsyncDataConsumer { private RecordingDataConsumer downstream; private CompressionDictionaryStore store; + private CookieStore partition; @BeforeEach void setUp() { downstream = new RecordingDataConsumer(); store = new BasicCompressionDictionaryStore(); + partition = new BasicCookieStore(); } private static UseAsDictionary rawDirective() throws Exception { @@ -78,64 +84,71 @@ private static UseAsDictionary unsupportedDirective() throws Exception { private DictionaryCapturingAsyncDataConsumer consumer(final int maxSize) throws Exception { return new DictionaryCapturingAsyncDataConsumer( - downstream, store, SOURCE, rawDirective(), STORED_AT, VALID_UNTIL, maxSize); + downstream, store, partition, SOURCE, rawDirective(), STORED_AT, VALID_UNTIL, maxSize); } @Test void testConstructorRejectsNullDownstream() throws Exception { final UseAsDictionary directive = rawDirective(); assertThrows(NullPointerException.class, () -> new DictionaryCapturingAsyncDataConsumer( - null, store, SOURCE, directive, STORED_AT, VALID_UNTIL, 1024)); + null, store, partition, SOURCE, directive, STORED_AT, VALID_UNTIL, 1024)); } @Test void testConstructorRejectsNullStore() throws Exception { final UseAsDictionary directive = rawDirective(); assertThrows(NullPointerException.class, () -> new DictionaryCapturingAsyncDataConsumer( - downstream, null, SOURCE, directive, STORED_AT, VALID_UNTIL, 1024)); + downstream, null, partition, SOURCE, directive, STORED_AT, VALID_UNTIL, 1024)); + } + + @Test + void testConstructorRejectsNullPartition() throws Exception { + final UseAsDictionary directive = rawDirective(); + assertThrows(NullPointerException.class, () -> new DictionaryCapturingAsyncDataConsumer( + downstream, store, null, SOURCE, directive, STORED_AT, VALID_UNTIL, 1024)); } @Test void testConstructorRejectsNullSource() throws Exception { final UseAsDictionary directive = rawDirective(); assertThrows(NullPointerException.class, () -> new DictionaryCapturingAsyncDataConsumer( - downstream, store, null, directive, STORED_AT, VALID_UNTIL, 1024)); + downstream, store, partition, null, directive, STORED_AT, VALID_UNTIL, 1024)); } @Test void testConstructorRejectsNullDirective() { assertThrows(NullPointerException.class, () -> new DictionaryCapturingAsyncDataConsumer( - downstream, store, SOURCE, null, STORED_AT, VALID_UNTIL, 1024)); + downstream, store, partition, SOURCE, null, STORED_AT, VALID_UNTIL, 1024)); } @Test void testConstructorRejectsNullStoredAt() throws Exception { final UseAsDictionary directive = rawDirective(); assertThrows(NullPointerException.class, () -> new DictionaryCapturingAsyncDataConsumer( - downstream, store, SOURCE, directive, null, VALID_UNTIL, 1024)); + downstream, store, partition, SOURCE, directive, null, VALID_UNTIL, 1024)); } @Test void testConstructorRejectsNullValidUntil() throws Exception { final UseAsDictionary directive = rawDirective(); assertThrows(NullPointerException.class, () -> new DictionaryCapturingAsyncDataConsumer( - downstream, store, SOURCE, directive, STORED_AT, null, 1024)); + downstream, store, partition, SOURCE, directive, STORED_AT, null, 1024)); } @Test void testConstructorRejectsNonPositiveMaxSize() throws Exception { final UseAsDictionary directive = rawDirective(); assertThrows(IllegalArgumentException.class, () -> new DictionaryCapturingAsyncDataConsumer( - downstream, store, SOURCE, directive, STORED_AT, VALID_UNTIL, 0)); + downstream, store, partition, SOURCE, directive, STORED_AT, VALID_UNTIL, 0)); assertThrows(IllegalArgumentException.class, () -> new DictionaryCapturingAsyncDataConsumer( - downstream, store, SOURCE, directive, STORED_AT, VALID_UNTIL, -1)); + downstream, store, partition, SOURCE, directive, STORED_AT, VALID_UNTIL, -1)); } @Test void testUpdateCapacityDelegates() throws Exception { final AsyncDataConsumer spy = mock(AsyncDataConsumer.class); final DictionaryCapturingAsyncDataConsumer mocked = new DictionaryCapturingAsyncDataConsumer( - spy, store, SOURCE, rawDirective(), STORED_AT, VALID_UNTIL, 1024); + spy, store, partition, SOURCE, rawDirective(), STORED_AT, VALID_UNTIL, 1024); final CapacityChannel channel = mock(CapacityChannel.class); mocked.updateCapacity(channel); verify(spy).updateCapacity(channel); @@ -165,7 +178,7 @@ void testStreamEndStoresDictionaryMatchingInputs() throws Exception { assertTrue(downstream.streamEnded()); - final List stored = store.getByOrigin(SOURCE); + final List stored = store.getByOrigin(partition, SOURCE); assertEquals(1, stored.size()); final CompressionDictionary dictionary = stored.get(0); assertArrayEquals("abcdefgh".getBytes(StandardCharsets.UTF_8), dictionary.getContent()); @@ -186,7 +199,7 @@ void testExceedingMaxSizeDiscardsButDelegatesAllData() throws Exception { c.consume(ByteBuffer.wrap(part2)); c.streamEnd(Collections.
emptyList()); - assertTrue(store.getByOrigin(SOURCE).isEmpty()); + assertTrue(store.getByOrigin(partition, SOURCE).isEmpty()); assertArrayEquals("abcdefgh".getBytes(StandardCharsets.UTF_8), downstream.consumedBytes()); assertTrue(downstream.streamEnded()); } @@ -194,17 +207,29 @@ void testExceedingMaxSizeDiscardsButDelegatesAllData() throws Exception { @Test void testUnsupportedDirectiveStoresNothing() throws Exception { final DictionaryCapturingAsyncDataConsumer c = new DictionaryCapturingAsyncDataConsumer( - downstream, store, SOURCE, unsupportedDirective(), STORED_AT, VALID_UNTIL, 1024); + downstream, store, partition, SOURCE, unsupportedDirective(), STORED_AT, VALID_UNTIL, 1024); final byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); c.consume(ByteBuffer.wrap(payload)); c.streamEnd(Collections.
emptyList()); - assertTrue(store.getByOrigin(SOURCE).isEmpty()); + assertTrue(store.getByOrigin(partition, SOURCE).isEmpty()); assertArrayEquals(payload, downstream.consumedBytes()); assertTrue(downstream.streamEnded()); } + @Test + void testDownstreamFailurePreventsStore() throws Exception { + final AsyncDataConsumer failing = mock(AsyncDataConsumer.class); + doThrow(new IOException("failure")).when(failing).streamEnd(anyList()); + final DictionaryCapturingAsyncDataConsumer c = new DictionaryCapturingAsyncDataConsumer( + failing, store, partition, SOURCE, rawDirective(), STORED_AT, VALID_UNTIL, 1024); + c.consume(ByteBuffer.wrap("payload".getBytes(StandardCharsets.UTF_8))); + + assertThrows(IOException.class, () -> c.streamEnd(Collections.
emptyList())); + assertTrue(store.getByOrigin(partition, SOURCE).isEmpty()); + } + @Test void testReleaseResourcesDelegatesAndPreventsStore() throws Exception { final DictionaryCapturingAsyncDataConsumer c = consumer(1024); @@ -215,7 +240,7 @@ void testReleaseResourcesDelegatesAndPreventsStore() throws Exception { c.streamEnd(Collections.
emptyList()); assertTrue(downstream.released()); - assertTrue(store.getByOrigin(SOURCE).isEmpty()); + assertTrue(store.getByOrigin(partition, SOURCE).isEmpty()); } /** diff --git a/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestUseAsDictionary.java b/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestUseAsDictionary.java index 1b629361f4..759a623ac1 100644 --- a/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestUseAsDictionary.java +++ b/httpclient5/src/test/java/org/apache/hc/client5/http/impl/async/TestUseAsDictionary.java @@ -68,9 +68,9 @@ void typeOtherIsNotSupported() throws ParseException { } @Test - void typeRawIsCaseInsensitive() throws ParseException { + void typeTokenIsCaseSensitive() throws ParseException { final UseAsDictionary dictionary = UseAsDictionary.parse("match=\"/app/*\", type=RAW"); - assertTrue(dictionary.isSupported()); + assertFalse(dictionary.isSupported()); } @Test @@ -174,10 +174,18 @@ void commaInsideQuotesIsNotSeparator() throws ParseException { } @Test - void commaInsideParensIsNotSeparator() throws ParseException { - final UseAsDictionary dictionary = UseAsDictionary.parse("dummy=(a, b), match=\"/x\", type=raw"); - assertEquals("/x", dictionary.getMatch()); - assertTrue(dictionary.isSupported()); + void commaInsideInnerListIsInvalidStructuredFieldSyntax() { + assertThrows(ParseException.class, + () -> UseAsDictionary.parse("dummy=(a, b), match=\"/x\", type=raw")); + } + + @Test + void matchDestParsesInnerListOfStrings() throws ParseException { + final UseAsDictionary dictionary = UseAsDictionary.parse( + "match=\"/x\", match-dest=(\"document\" \"script\")"); + assertEquals(2, dictionary.getMatchDest().size()); + assertEquals("document", dictionary.getMatchDest().get(0)); + assertEquals("script", dictionary.getMatchDest().get(1)); } @Test @@ -192,6 +200,12 @@ void parametersAfterSemicolonAreIgnored() throws ParseException { assertTrue(dictionary.isSupported()); } + @Test + void spaceAfterParameterSemicolonIsAccepted() throws ParseException { + final UseAsDictionary dictionary = UseAsDictionary.parse("match=\"/x\"; foo=bar"); + assertEquals("/x", dictionary.getMatch()); + } + @Test void parametersAfterSemicolonAreIgnoredForUnsupportedType() throws ParseException { final UseAsDictionary dictionary = UseAsDictionary.parse("match=\"/x\", type=other;foo=bar"); @@ -207,4 +221,40 @@ void emptyTypeTokenThrows() { void invalidTypeTokenCharacterThrows() { assertThrows(ParseException.class, () -> UseAsDictionary.parse("match=\"/x\", type=a b")); } + + @Test + void integerLongerThanFifteenDigitsInvalidatesWholeField() { + assertThrows(ParseException.class, + () -> UseAsDictionary.parse("match=\"/x\", extension=1234567890123456")); + } + + @Test + void decimalIntegerPartLongerThanTwelveDigitsInvalidatesWholeField() { + assertThrows(ParseException.class, + () -> UseAsDictionary.parse("match=\"/x\", extension=1234567890123.1")); + } + + @Test + void dateLongerThanFifteenDigitsInvalidatesWholeField() { + assertThrows(ParseException.class, + () -> UseAsDictionary.parse("match=\"/x\", extension=@1234567890123456")); + } + + @Test + void malformedUtf8DisplayStringInvalidatesWholeField() { + assertThrows(ParseException.class, + () -> UseAsDictionary.parse("match=\"/x\", extension=%\"%ff\"")); + } + + @Test + void validUtf8DisplayStringIsAcceptedAsAnExtension() throws ParseException { + final UseAsDictionary dictionary = UseAsDictionary.parse( + "match=\"/x\", extension=%\"caf%c3%a9\""); + assertEquals("/x", dictionary.getMatch()); + } + + @Test + void leadingTabIsNotAccepted() { + assertThrows(ParseException.class, () -> UseAsDictionary.parse("\tmatch=\"/x\"")); + } }