From 8edef9d9b60bfc15efe5d8326b7f976f2afdcb91 Mon Sep 17 00:00:00 2001 From: Jean-Frederic Clere Date: Thu, 29 Oct 2026 13:51:08 +0100 Subject: [PATCH 1/2] Add switchabe debug for CID for a possible eBDF usage. Arrange the logic to limit the error when using h3bench. --- modules/http3/h3_threads.c | 67 ++++++++++--- modules/http3/quic/detail/h3q_addr.c | 144 +++++++++++++++++++++++---- modules/http3/quic/h3q.c | 16 ++- 3 files changed, 186 insertions(+), 41 deletions(-) diff --git a/modules/http3/h3_threads.c b/modules/http3/h3_threads.c index 137e9531123..7d186e0fa39 100644 --- a/modules/http3/h3_threads.c +++ b/modules/http3/h3_threads.c @@ -20,6 +20,7 @@ #include #include +#include #include "h3.h" #include "h3_io.h" @@ -29,6 +30,8 @@ #include "quic/h3q.h" #include "quic/h3q_conn.h" +#define H3_PUMP_BUDGET 32 + void* APR_THREAD_FUNC h3_event_thread(apr_thread_t* thread, void* data) { (void)thread; @@ -48,21 +51,27 @@ void* APR_THREAD_FUNC h3_event_thread(apr_thread_t* thread, void* data) wait_for_event(io); } work_pending = 0; - int pumped = h3q_engine_pump(io->qengine); - if (pumped < 0) { - if (!listener_failed) - { - listener_failed = 1; - ap_log_error(APLOG_MARK, APLOG_ERR, 0, io->server, "QUIC listener event processing failed; no further datagrams will be handled"); - } - /* Once latched, unread datagrams keep POLLIN hot; do not spin on them. */ - apr_sleep(apr_time_from_msec(H3_LISTENER_FAILED_BACKOFF_MS)); - } - else - { - listener_failed = 0; - work_pending = pumped; + int pump_rounds = 0; + int pumped; + do { + pumped = h3q_engine_pump(io->qengine); + if (pumped < 0) + { + if (!listener_failed) + { + listener_failed = 1; + ap_log_error(APLOG_MARK, APLOG_ERR, 0, io->server, "QUIC listener event processing failed; no further datagrams will be handled"); + } + apr_sleep(apr_time_from_msec(H3_LISTENER_FAILED_BACKOFF_MS)); + break; + } + listener_failed = 0; + if (pumped > 0) + { + work_pending = 1; + } + } while (pumped > 0 && ++pump_rounds < H3_PUMP_BUDGET); } if (io->thread_running && !io->draining) @@ -92,8 +101,36 @@ void* APR_THREAD_FUNC h3_event_thread(apr_thread_t* thread, void* data) for (int i = 0; i < io->active_sessions->nelts;) { + if (i > 0 && (i % 8) == 0) + { + int p = h3q_engine_pump(io->qengine); + if (p > 0) + { + work_pending = 1; + } + } + h3_session* session = ((h3_session**)io->active_sessions->elts)[i]; - if (service_session_pass(io, session)) + + /* Skip sessions with no work: no active worker tasks, no streams, + * and not draining/stopping. Just check if the connection died. */ + if (!io->draining && io->thread_running + && apr_atomic_read32(&session->active_tasks) == 0 + && apr_hash_count(session->streams) == 0 + && !session->aborted) + { + if (h3q_conn_is_closed(session->qconn)) + { + session->aborted = 1; + } + else + { + i++; + continue; + } + } + + if (!session->aborted && service_session_pass(io, session)) { work_pending = 1; } diff --git a/modules/http3/quic/detail/h3q_addr.c b/modules/http3/quic/detail/h3q_addr.c index 5317ff6c7ad..0018a2cc953 100644 --- a/modules/http3/quic/detail/h3q_addr.c +++ b/modules/http3/quic/detail/h3q_addr.c @@ -19,6 +19,7 @@ #include #endif +#include #include #include @@ -28,6 +29,75 @@ #include "h3_os.h" #include "quic/detail/h3q_addr.h" +#ifdef H3Q_DCID_DEBUG + +static void h3q_hex(char* out, const unsigned char* data, size_t len) +{ + size_t i; + for (i = 0; i < len; i++) + sprintf(out + i * 2, "%02x", data[i]); + out[len * 2] = '\0'; +} + +static void h3q_trace_quic(const char* direction, const unsigned char* data, size_t data_len) +{ + unsigned char dcid[20]; + unsigned char scid[20]; + size_t dcid_len = 0; + size_t scid_len = 0; + size_t off; + char dcid_hex[41]; + char scid_hex[41]; + FILE* f; + + if (data_len < 1) + return; + + if (data[0] & 0x80) { + /* Long header: [flags(1)][version(4)][dcid_len(1)][dcid...][scid_len(1)][scid...] */ + if (data_len < 6) + return; + dcid_len = data[5]; + if (dcid_len > 20 || data_len < 6 + dcid_len) + return; + memcpy(dcid, data + 6, dcid_len); + + off = 6 + dcid_len; + if (data_len > off) { + scid_len = data[off]; + if (scid_len <= 20 && data_len >= off + 1 + scid_len) + memcpy(scid, data + off + 1, scid_len); + else + scid_len = 0; + } + + h3q_hex(dcid_hex, dcid, dcid_len); + h3q_hex(scid_hex, scid, scid_len); + f = fopen("/tmp/h3q_dcid.log", "a"); + if (f) { + fprintf(f, "[h3q] %s long DCID(%zu): %s SCID(%zu): %s\n", + direction, dcid_len, dcid_hex, scid_len, scid_hex); + fclose(f); + } + } else { + /* Short header: [flags(1)][dcid...] */ + dcid_len = 8; + if (data_len < 1 + dcid_len) + return; + memcpy(dcid, data + 1, dcid_len); + + h3q_hex(dcid_hex, dcid, dcid_len); + f = fopen("/tmp/h3q_dcid.log", "a"); + if (f) { + fprintf(f, "[h3q] %s short DCID(%zu): %s\n", + direction, dcid_len, dcid_hex); + fclose(f); + } + } +} + +#endif /* H3Q_DCID_DEBUG */ + struct h3q_datagram { unsigned char* data; @@ -81,6 +151,9 @@ static int h3q_queue_fill(h3q_engine* engine, BIO_MSG* msg, size_t stride, size_ h3q_peer_addr_queue_clear(engine); return 0; } +#ifdef H3Q_DCID_DEBUG + h3q_trace_quic("RECV", source->data, source->data_len); +#endif item->data = OPENSSL_memdup(source->data, source->data_len); item->data_len = source->data_len; item->peer = source->peer ? BIO_ADDR_dup(source->peer) : NULL; @@ -143,6 +216,18 @@ long h3q_peer_addr_bio_ctrl(BIO* bio, int cmd, long num, void* ptr) int h3q_peer_addr_bio_sendmmsg(BIO* bio, BIO_MSG* msg, size_t stride, size_t num_msg, uint64_t flags, size_t* msgs_processed) { BIO* next = BIO_next(bio); + +#ifdef H3Q_DCID_DEBUG + { + size_t i; + for (i = 0; i < num_msg; i++) { + BIO_MSG* m = (BIO_MSG*)((unsigned char*)msg + i * stride); + if (m->data && m->data_len > 0) + h3q_trace_quic("SEND", m->data, m->data_len); + } + } +#endif + return next ? BIO_sendmmsg(next, msg, stride, num_msg, flags, msgs_processed) : 0; } @@ -150,47 +235,62 @@ int h3q_peer_addr_bio_recvmmsg(BIO* bio, BIO_MSG* msg, size_t stride, size_t num { h3q_engine* engine = BIO_get_data(bio); BIO* next = BIO_next(bio); + size_t count; + if (!engine || !next || !msg || !msgs_processed || num_msg == 0) { return 0; } BIO_ADDR_clear(engine->current_peer_addr); - if (engine->peer_rx_head) + + if (!engine->peer_rx_head) { - *msgs_processed = 0; - if (!h3q_queue_pop(engine, msg)) + size_t received = 0; + int rv = BIO_recvmmsg(next, msg, stride, num_msg, flags, &received); + if (!rv || received == 0) { - return 0; + *msgs_processed = received; + return rv; } - *msgs_processed = 1; - if (msg->peer) + if (!h3q_queue_fill(engine, msg, stride, received)) { - BIO_ADDR_copy(engine->current_peer_addr, msg->peer); + *msgs_processed = 0; + return 0; } - return 1; } - size_t received = 0; - int rv = BIO_recvmmsg(next, msg, stride, num_msg, flags, &received); - if (rv && received > 0) + /* Pop from queue: batch short-header packets (existing connections); + * long-header packets go one at a time for peer-address capture. */ + count = 0; + while (count < num_msg && engine->peer_rx_head) { - if (!h3q_queue_fill(engine, msg, stride, received) || !h3q_queue_pop(engine, msg)) + h3q_datagram* head = engine->peer_rx_head; + int is_long = (head->data_len > 0 && (head->data[0] & 0x80)); + + if (is_long && count > 0) { - *msgs_processed = 0; - return 0; + break; + } + + BIO_MSG* dst = (BIO_MSG*)((unsigned char*)msg + count * stride); + if (!h3q_queue_pop(engine, dst)) + { + break; } - *msgs_processed = 1; - if (msg->peer) + count++; + + if (is_long) { - BIO_ADDR_copy(engine->current_peer_addr, msg->peer); + if (dst->peer) + { + BIO_ADDR_copy(engine->current_peer_addr, dst->peer); + } + break; } } - else - { - *msgs_processed = received; - } - return rv; + *msgs_processed = count; + return count > 0 ? 1 : 0; } int h3q_peer_addr_bio_destroy(BIO* bio) diff --git a/modules/http3/quic/h3q.c b/modules/http3/quic/h3q.c index 9b38736c4d6..10f88b4a842 100644 --- a/modules/http3/quic/h3q.c +++ b/modules/http3/quic/h3q.c @@ -31,6 +31,8 @@ #include "quic/detail/h3q_tls.h" #include "quic/h3q.h" +#define H3Q_PUMP_INNER_BUDGET 64 + h3q_engine* h3q_engine_create(const h3q_config* cfg, int udp_fd, char* err, size_t errlen) { CHECK(cfg); @@ -144,24 +146,30 @@ void h3q_engine_destroy(h3q_engine* engine) int h3q_engine_pump(h3q_engine* engine) { + int rounds = 0; + int had_queued; + if (!engine || !engine->ssl_listener) { return 0; } - int work = 0; + had_queued = (engine->peer_rx_head != NULL); if (SSL_handle_events(engine->ssl_listener) != 1) { return -1; } - while (engine->peer_rx_head) + while (engine->peer_rx_head && ++rounds < H3Q_PUMP_INNER_BUDGET) { if (SSL_handle_events(engine->ssl_listener) != 1) { return -1; } - work = 1; } - return work; + if (engine->peer_rx_head) + { + return 1; + } + return (had_queued || rounds > 0) ? 1 : 0; } void h3q_engine_want(h3q_engine* engine, int* want_read, int* want_write, int* timeout_ms) From a8b91ee8519b8a54783eaf6c777478320045d257 Mon Sep 17 00:00:00 2001 From: Jean-Frederic Clere Date: Tue, 3 Nov 2026 17:07:59 +0100 Subject: [PATCH 2/2] Don't use the legacy it gives problens. --- modules/http3/quic/detail/h3q_tls.c | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/http3/quic/detail/h3q_tls.c b/modules/http3/quic/detail/h3q_tls.c index 1d6b6d215ba..1ed6125b0d2 100644 --- a/modules/http3/quic/detail/h3q_tls.c +++ b/modules/http3/quic/detail/h3q_tls.c @@ -88,6 +88,7 @@ SSL_CTX* h3q_tls_ctx_create(const h3q_config* cfg, char* err, size_t errlen) return NULL; } + SSL_CTX_set_domain_flags(ssl_ctx, SSL_DOMAIN_FLAG_MULTI_THREAD); SSL_CTX_set_min_proto_version(ssl_ctx, TLS1_3_VERSION); SSL_CTX_set_max_proto_version(ssl_ctx, TLS1_3_VERSION);