diff --git a/.cspell.json b/.cspell.json index 601633f..c9ec414 100644 --- a/.cspell.json +++ b/.cspell.json @@ -1,9 +1,5 @@ { "words": [ - "dedent", - "doxygen", - "literalinclude", - "toctree", "64kminus1", "ALTSVC", "anyhttp", @@ -11,6 +7,7 @@ "ASIO", "autotuning", "AWSLC", + "BINDIR", "boringssl", "callgrind", "capy", @@ -22,10 +19,14 @@ "datagrams", "dcid", "dcidlen", + "dedent", "demuxed", + "DESTDIR", "docroot", + "doxygen", "ECDH", "Eisenlohr", + "endforeach", "epoll", "gencert", "gmock", @@ -33,6 +34,7 @@ "GTEST", "HCAT", "HPACK", + "INCLUDEDIR", "IOCP", "IWYU", "jemalloc", @@ -40,6 +42,7 @@ "LIBCXX", "linearisation", "linearise", + "literalinclude", "logd", "loge", "logi", @@ -48,6 +51,7 @@ "malloc", "marp", "mawk", + "misaligns", "mlogd", "mloge", "mlogi", @@ -57,10 +61,12 @@ "nghttpd", "nghttpd's", "ngtcp", + "noexcept", "nullptr", "ossl", "osslclient", "osslserver", + "Postconditions", "PROFDATA", "PROFRAW", "psedoc", @@ -85,8 +91,11 @@ "syscalls", "testcases", "TLSEXT", + "toctree", "TSAN", - "WOULDBLOCK" + "UBSAN", + "WOULDBLOCK", + "zstd" ], "ignorePaths": [ ".devcontainer/*", diff --git a/.github/instructions/anyhttp.instructions.md b/.github/instructions/anyhttp.instructions.md index c81d2e4..5549327 100644 --- a/.github/instructions/anyhttp.instructions.md +++ b/.github/instructions/anyhttp.instructions.md @@ -4,7 +4,7 @@ applyTo: '**' AnyHTTP is an experimental, low-level C++23 HTTP client and server library with a type-erasing interface: one API for HTTP/1.1, HTTP/2 and HTTP/3 (QUIC), hence the *any* in the name -- plus plain QUIC streams next to HTTP/3. None of the protocols are implemented from scratch; it wraps -Boost.Beast (parser and serializer), nghttp2, and ngtcp2/nghttp3. +Boost.HTTP (cppalliance/http: parser, serializer and header fields), nghttp2, and ngtcp2/nghttp3. Everything is asynchronous and written with C++20 coroutines. Every operation is a coroutine (`read_some`, `write`, `write_eof`, `submit`, `get_response`, `connect`, `get`) that yields a @@ -24,7 +24,7 @@ the runtime only through `anyhttp/runtime.hpp`, `anyhttp/net.hpp` and `anyhttp/f includes nothing else from Boost.Asio but ``. Each protocol has a directory of its own, again under both `include/anyhttp/` and `src/`: `h1/` -(Beast), `h2/` (nghttp2), `quic/` (ngtcp2) and `h3/` (nghttp3, on top of `quic/`). Only files in +(Boost.HTTP), `h2/` (nghttp2), `quic/` (ngtcp2) and `h3/` (nghttp3, on top of `quic/`). Only files in `h2/` include ``, only `quic/` and `h3/` ``, only `h3/` ``. Generic code reaches a protocol through that protocol's `backend.hpp`. Keep these boundaries. diff --git a/CLAUDE.md b/CLAUDE.md index 3b21945..30ef905 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,6 +13,15 @@ benchmark with the latter, never with `build/`. Both use clang, which builds aga the libraries in `/opt/libc++` (`USE_LIBCXX`, `LIBCXX_ROOT`); `build-gcc/` is GCC with libstdc++ and `/usr/local`. Keep `build/` on clang: clangd reads its `compile_commands.json`. +HTTP/1.1 is [Boost.HTTP](https://github.com/cppalliance/http) (cppalliance/http, not in a Boost +release yet), and its `fields` are anyhttp's `Fields`. Every tree FetchContents it, and capy with +it, at a pinned SHA of `develop`; `cmake/http.cmake` builds its protocol core as `anyhttp_http`, +with the patches in `cmake/patches/` applied, each of them proposed upstream. Its parser and +serializer are sans-I/O: `h1/io.hpp` has the loops that drive them on the runtime layer, as +Boost.HTTP's own (`parser::read()`, `serializer::sink`) need a capy stream. Leave chunking and +`Content-Length` to the library, even where it copies: a body is copied into the serializer's +buffer, there is no zero-copy path yet. + `build-corosio/` is the COROSIO I/O runtime (`-DANYHTTP_API=COROSIO -DTLS_LIBRARY=OpenSSL`, see [docs/corosio-port-plan.md](docs/corosio-port-plan.md)); capy and corosio are FetchContent'd at pinned SHAs. Both runtimes build the same library sources and test files; what belongs to one @@ -139,7 +148,7 @@ buffer (see `WriteMode`), and `build-h2v2-asan/` is the version 2 ASAN tree for ## Source layout -Each protocol lives in a directory of its own, `h1/` (Beast/HTTP/1.1), `h2/` (nghttp2), `quic/` +Each protocol lives in a directory of its own, `h1/` (Boost.HTTP), `h2/` (nghttp2), `quic/` (ngtcp2) and `h3/` (nghttp3, on top of `quic/`), in both `include/anyhttp/` and `src/`. **Only files in `h2/` may include ``, only files in `h2v2/` ``, only files in `quic/` and `h3/` ``, only files in `h3/` ``.** What belongs to one async @@ -206,8 +215,9 @@ Only objects in `h2/`, `quic/` and `h3/` of the library may appear in the first in `h3/` in the second: that is what proves the QUIC transport has no HTTP in it. (Test objects may: the h2c upgrade and formatter tests drive nghttp2 by hand.) -**Generic and protocol code include nothing from Boost.Asio but ``**, which -Beast's parser, serializer and `Fields` bring anyway, and nothing from capy or corosio. They reach +**Generic and protocol code include nothing from Boost.Asio but ``**, and +nothing from capy or corosio but what Boost.HTTP brings along: its parser and serializer are +written in terms of capy's buffers, which `h1/io.hpp` converts at the boundary. They reach the runtime through `anyhttp/runtime.hpp`, `anyhttp/net.hpp` and `anyhttp/formatter.hpp`, which include the `asio/` or `corosio/` header of the build's runtime. Addresses and endpoints are the runtime's own (`IpAddress`, `TcpEndpoint`, `UdpEndpoint`); where the two differ, `net.hpp` declares diff --git a/CMakeLists.txt b/CMakeLists.txt index 1ed194a..133e322 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -151,29 +151,40 @@ elseif (ANYHTTP_API STREQUAL "COROSIO") message(FATAL_ERROR "ANYHTTP_API=COROSIO needs TLS_LIBRARY=OpenSSL: " "corosio's openssl_stream does not build against AWS-LC") endif() +else() + message(FATAL_ERROR "ANYHTTP_API must be ASIO or COROSIO, not '${ANYHTTP_API}'") +endif() - # - # Pinned: both are pre-release and change their API on 'develop' without notice. Moving a pin - # is a commit of its own. - # - include(FetchContent) - foreach (option TESTS EXAMPLES BENCH) - set(BOOST_CAPY_BUILD_${option} OFF CACHE BOOL "" FORCE) - set(BOOST_COROSIO_BUILD_${option} OFF CACHE BOOL "" FORCE) - endforeach() - FetchContent_Declare(capy - GIT_REPOSITORY https://github.com/cppalliance/capy.git - GIT_TAG d62f5de1684389682ff4201a6171eec646165036 - EXCLUDE_FROM_ALL) +# +# capy, Boost.HTTP (cppalliance/http) and, for COROSIO, corosio. Every tree needs the first two: +# HTTP/1.1 is Boost.HTTP's parser and serializer, and those are written in terms of capy's buffers +# and errors, whatever the runtime. See cmake/http.cmake. +# +# Pinned: all three are pre-release and change their API on 'develop' without notice. Moving a pin +# is a commit of its own, and Boost.HTTP's has to build against capy's. +# +include(FetchContent) +foreach (option TESTS EXAMPLES BENCH) + set(BOOST_CAPY_BUILD_${option} OFF CACHE BOOL "" FORCE) + set(BOOST_COROSIO_BUILD_${option} OFF CACHE BOOL "" FORCE) +endforeach() +FetchContent_Declare(capy + GIT_REPOSITORY https://github.com/cppalliance/capy.git + GIT_TAG d62f5de1684389682ff4201a6171eec646165036 + EXCLUDE_FROM_ALL) +include(cmake/http.cmake) +if (ANYHTTP_COROSIO) FetchContent_Declare(corosio GIT_REPOSITORY https://github.com/cppalliance/corosio.git GIT_TAG 13bcf933adf8c5ce8576bd3349ae58d2e46e5c53 EXCLUDE_FROM_ALL) - FetchContent_MakeAvailable(capy corosio) + FetchContent_MakeAvailable(capy http corosio) message(STATUS "Using I/O runtime: COROSIO (capy/corosio)") else() - message(FATAL_ERROR "ANYHTTP_API must be ASIO or COROSIO, not '${ANYHTTP_API}'") + FetchContent_MakeAvailable(capy http) + message(STATUS "Using I/O runtime: ASIO") endif() +anyhttp_add_http() # -------------------------------------------------------------------------------------------------- diff --git a/README.md b/README.md index 54f513e..e37de93 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ It supports HTTP/1.x, HTTP/2 and HTTP/3 behind a common, type-erasing interface, None of those protocols are implemented from scratch. Instead, it is a wrapper around the following well-established libraries: -* [beast](https://github.com/boostorg/beast) (its parser and serializer) +* [Boost.HTTP](https://github.com/cppalliance/http) (its parser, serializer and header fields), pre-release, built from its `develop` branch * [nghttp2](https://github.com/nghttp2/nghttp2), or [nghttp2 version 2](https://github.com/nghttp2/nghttp2-v2), its rewrite * [ngtcp2](https://github.com/ngtcp2/ngtcp2) / [nghttp3](https://github.com/ngtcp2/nghttp3) @@ -106,7 +106,6 @@ An installed package works as well, on ASIO, see For now, this section contains just a set of random links collected during development. -* [Beast Example using Type Erasure](https://www.boost.org/doc/libs/develop/boost/beast/http/message_generator.hpp) * [Development container](docs/devcontainer.rst) -- how the build environment is put together * [Capturing traffic](docs/capture.rst) -- tcpdump and tshark, decrypting TLS and QUIC with a key log * [cpp-devcontainer](https://github.com/pgit/cpp-devcontainer) -- the generic C++ base image it builds on diff --git a/cmake/http.cmake b/cmake/http.cmake new file mode 100644 index 0000000..5e6b5eb --- /dev/null +++ b/cmake/http.cmake @@ -0,0 +1,69 @@ +# +# Boost.HTTP (https://github.com/cppalliance/http), the HTTP/1.1 parser and serializer of h1/ and +# the header field container that is anyhttp's Fields. Not part of a Boost release yet, so it comes +# like capy and corosio, by FetchContent at a pinned commit of 'develop'. +# +# Only its protocol core is built, as a target of our own, anyhttp_http (anyhttp::http when +# installed). Upstream's CMakeLists.txt builds all of src/ -- the router, the server, bcrypt and +# JSON -- and asks for Boost.JSON, which nothing here needs; SOURCE_SUBDIR keeps +# FetchContent_MakeAvailable() from adding it. The compression codecs are left out as well: the +# parser and serializer look them up as services at run time and do without them unless asked to +# (de)compress, which anyhttp never does. Their error categories are part of the core all the same. +# +# The patches in cmake/patches/ are proposed upstream. Each one says what it is for. +# +# http-release-buffered-data.patch parser::release_buffered_data(), a stub on 'develop': the +# h2c upgrade hands what follows the request to HTTP/2 +# http-reserve-bytes-alignment.patch fields_base::reserve_bytes() took any capacity, which +# misaligns the table of fields at the end of the buffer +# (UBSAN: "reference binding to misaligned address") +# +set(anyhttp_http_patches + ${CMAKE_CURRENT_LIST_DIR}/patches/http-release-buffered-data.patch + ${CMAKE_CURRENT_LIST_DIR}/patches/http-reserve-bytes-alignment.patch) + +# +# The patch step runs when the source is fetched, and again whenever its command changes -- not when +# a patch file does, so each command carries the hash of its patch. Every run starts from the +# pristine files: a patch that has changed does not apply over its earlier version. +# +set(anyhttp_http_patch_command COMMAND git checkout --quiet -- .) +foreach (patch IN LISTS anyhttp_http_patches) + file(SHA256 ${patch} anyhttp_http_patch_hash) + list(APPEND anyhttp_http_patch_command COMMAND sh -c + "git apply '${patch}' # ${anyhttp_http_patch_hash}") +endforeach() +unset(anyhttp_http_patch_hash) +list(POP_FRONT anyhttp_http_patch_command) # the first COMMAND is PATCH_COMMAND itself + +FetchContent_Declare(http + GIT_REPOSITORY https://github.com/cppalliance/http.git + GIT_TAG 1f58e9791afd6c7351805f3074d4d3bbff8189f9 + PATCH_COMMAND ${anyhttp_http_patch_command} + SOURCE_SUBDIR do-not-add-upstream-cmake + EXCLUDE_FROM_ALL) + +function(anyhttp_add_http) + set(dir ${http_SOURCE_DIR}) + file(GLOB sources + ${dir}/src/*.cpp + ${dir}/src/core/*.cpp + ${dir}/src/detail/*.cpp + ${dir}/src/rfc/*.cpp + ${dir}/src/rfc/detail/*.cpp + ${dir}/src/brotli/error.cpp + ${dir}/src/zlib/error.cpp + ${dir}/src/zstd/error.cpp) + list(FILTER sources EXCLUDE REGEX "/src/(application|datastore)\\.cpp$|/file_win32\\.cpp$") + + add_library(anyhttp_http STATIC ${sources}) + add_library(anyhttp::http ALIAS anyhttp_http) + set_target_properties(anyhttp_http PROPERTIES EXPORT_NAME http) + target_compile_features(anyhttp_http PUBLIC cxx_std_20) + target_include_directories(anyhttp_http SYSTEM PUBLIC + $ + $) + target_include_directories(anyhttp_http PRIVATE ${dir}) + target_compile_definitions(anyhttp_http PUBLIC BOOST_HTTP_STATIC_LINK PRIVATE BOOST_HTTP_SOURCE) + target_link_libraries(anyhttp_http PUBLIC Boost::capy Boost::headers Boost::url) +endfunction() diff --git a/cmake/install.cmake b/cmake/install.cmake index b7f90a9..34f4bc9 100644 --- a/cmake/install.cmake +++ b/cmake/install.cmake @@ -51,7 +51,13 @@ endif() set(anyhttp_cmake_dir ${CMAKE_INSTALL_LIBDIR}/cmake/anyhttp) -install(TARGETS anyhttp +# +# Boost.HTTP's core is a library of its own, see cmake/http.cmake, and its headers come along: Fields +# is its container. So does capy, which it is written against: both are FetchContent'd, and a +# subproject added EXCLUDE_FROM_ALL installs nothing of its own, so they are part of this package, +# as anyhttp::http and anyhttp::capy. +# +install(TARGETS anyhttp anyhttp_http boost_capy EXPORT anyhttp-targets ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT Development LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT Development @@ -77,6 +83,9 @@ install(DIRECTORY ${PROJECT_SOURCE_DIR}/include/anyhttp PATTERN "*_impl.hpp" EXCLUDE PATTERN "any_async_stream.hpp" EXCLUDE PATTERN "buffer_array.hpp" EXCLUDE) +install(DIRECTORY ${http_SOURCE_DIR}/include/boost ${capy_SOURCE_DIR}/include/boost + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT Development + FILES_MATCHING PATTERN "*.hpp") install(FILES ${PROJECT_BINARY_DIR}/include/anyhttp/config.hpp DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/anyhttp COMPONENT Development) diff --git a/cmake/patches/http-release-buffered-data.patch b/cmake/patches/http-release-buffered-data.patch new file mode 100644 index 0000000..5056365 --- /dev/null +++ b/cmake/patches/http-release-buffered-data.patch @@ -0,0 +1,93 @@ +diff --git a/include/boost/http/parser.hpp b/include/boost/http/parser.hpp +index 2edf97b..dfe0c9c 100644 +--- a/include/boost/http/parser.hpp ++++ b/include/boost/http/parser.hpp +@@ -336,6 +336,19 @@ public: + to retrieve protocol-dependent data that + follows the HTTP message. + ++ The octets are released: they are no longer ++ part of what the parser holds, and a later ++ @ref has_buffered_data returns false. The body ++ of the message, if any, is left as it is. ++ ++ @par Preconditions ++ `this->is_complete() == true`, otherwise the ++ returned view is empty. ++ ++ @par Postconditions ++ The returned view is invalidated by any ++ modifying member function. ++ + @return A string view of leftover data. + + @see @ref metadata::upgrade, @ref metadata::connection. +diff --git a/src/parser.cpp b/src/parser.cpp +index e73cb3b..29aac77 100644 +--- a/src/parser.cpp ++++ b/src/parser.cpp +@@ -33,6 +33,7 @@ + #include "src/detail/buffer_utils.hpp" + #include "src/detail/zlib_filter_base.hpp" + ++#include + #include + #include + +@@ -1303,6 +1304,45 @@ public: + body_avail_); + } + ++ core::string_view ++ release_buffered_data() noexcept ++ { ++ if(state_ != state::complete) ++ return {}; ++ ++ // the octets past the message are at the end of cb0_, ++ // behind what is left of a plain body ++ auto const skip = is_plain() ? body_avail_ : 0; ++ if(cb0_.size() <= skip) ++ return {}; ++ ++ // make them contiguous: rotating the storage puts the ++ // second piece of the circular buffer behind the first ++ auto cbp = cb0_.data(); ++ if(cbp[1].size() != 0) ++ { ++ auto* base = static_cast( ++ const_cast(cbp[1].data())); ++ auto* first = static_cast( ++ const_cast(cbp[0].data())); ++ auto const cap = cb0_.max_size(); ++ auto const size = cb0_.size(); ++ std::rotate(base, first, first + cbp[0].size()); ++ cb0_ = { base, cap, size }; ++ cbp = cb0_.data(); ++ } ++ ++ auto const* p = static_cast(cbp[0].data()); ++ core::string_view leftover(p + skip, cb0_.size() - skip); ++ ++ // released: no longer part of what the parser holds, but ++ // the storage stays as it is until the next modifying call. ++ // What is left is the body, and that is all the buffer ++ // may claim: p is not the base of the storage. ++ cb0_ = { const_cast(p), skip, skip }; ++ return leftover; ++ } ++ + bool + has_buffered_data() const noexcept + { +@@ -1564,8 +1604,8 @@ core::string_view + parser:: + release_buffered_data() noexcept + { +- // TODO +- return {}; ++ BOOST_ASSERT(impl_); ++ return impl_->release_buffered_data(); + } + + bool diff --git a/cmake/patches/http-reserve-bytes-alignment.patch b/cmake/patches/http-reserve-bytes-alignment.patch new file mode 100644 index 0000000..43643ea --- /dev/null +++ b/cmake/patches/http-reserve-bytes-alignment.patch @@ -0,0 +1,17 @@ +diff --git a/src/fields_base.cpp b/src/fields_base.cpp +index dc4b16b..0d87141 100644 +--- a/src/fields_base.cpp ++++ b/src/fields_base.cpp +@@ -161,6 +161,12 @@ op_t:: + reserve( + std::size_t n) + { ++ // the table of entries is at the end of the ++ // buffer, so the capacity has to keep it ++ // aligned, as header::bytes_needed() does ++ auto const al = alignof(entry); ++ n = (n + al - 1) & ~(al - 1); ++ + // TODO: consider using a growth factor + if(n > self_.max_cap_) + { diff --git a/docs/design.rst b/docs/design.rst index f6c53ac..448b86c 100644 --- a/docs/design.rst +++ b/docs/design.rst @@ -27,7 +27,7 @@ ngtcp2 and nghttp3 from being compile-time dependencies of anyone who uses anyht - On top of * - ``h1/`` - HTTP/1.1 - - Beast's parser and serializer + - Boost.HTTP's parser and serializer * - ``h2/`` - HTTP/2 - nghttp2 diff --git a/docs/examples/hello_client.cpp b/docs/examples/hello_client.cpp index 9366d80..46d151c 100644 --- a/docs/examples/hello_client.cpp +++ b/docs/examples/hello_client.cpp @@ -35,7 +35,7 @@ static Task fetch(client::Client& client, boost::urls::url url, int& resul co_return; } - std::println("{} over {}: {}", message.result_int(), to_string(session.protocol()), + std::println("{} over {}: {}", message.status_int(), to_string(session.protocol()), message.body()); result = 0; } diff --git a/docs/index.rst b/docs/index.rst index 7fc2017..5c50001 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,8 +11,8 @@ protocols of your own. It runs on either `Asio **THIS REPOSITORY IS PURELY EXPERIMENTAL.** It is not meant for production use, and anything in it can change or break at any time, without notice. -None of the protocols is implemented from scratch. The library is a wrapper around `Beast -`__ (its parser and serializer), `nghttp2 +None of the protocols is implemented from scratch. The library is a wrapper around `Boost.HTTP +`__ (its parser, serializer and header fields), `nghttp2 `__ or `nghttp2 version 2 `__, and `ngtcp2 `__ with `nghttp3 `__. diff --git a/docs/io-runtimes.rst b/docs/io-runtimes.rst index 5213f02..320f752 100644 --- a/docs/io-runtimes.rst +++ b/docs/io-runtimes.rst @@ -67,7 +67,7 @@ What the names stand for - ``server``, ``client`` - ``server`` -Buffers are Asio's on both, headers are Beast's :cpp:type:`Fields`, URLs Boost.URL's. +Buffers are Asio's on both, headers are Boost.HTTP's :cpp:type:`Fields`, URLs Boost.URL's. The reference shows what the headers have for ASIO, as Doxygen is told to read them that way. So the reference has the operations with completion tokens that a COROSIO build does not have, and the types @@ -143,7 +143,7 @@ Where the two runtimes spell an error differently, the namespace ``anyhttp::erro - ``boost::asio::error::eof`` - ``boost::capy::error::eof`` * - :cpp:var:`errors::partial_message` - - ``beast::http::error::partial_message`` + - ``boost::http::error::incomplete`` - ``boost::capy::error::stream_truncated`` * - :cpp:var:`errors::canceled` - ``errc::operation_canceled`` @@ -155,7 +155,10 @@ Where the two runtimes spell an error differently, the namespace ``anyhttp::erro - ``boost::asio::error::connection_aborted`` - ``errc::connection_aborted`` * - :cpp:var:`errors::header_limit` - - ``beast::http::error::header_limit`` + - ``boost::http::error::headers_limit`` + - the same + * - :cpp:var:`errors::end_of_stream` + - ``boost::http::error::end_of_stream`` - the same Anything else compares against ``errc::`` (``boost::system::errc`` or ``std::errc``), with one diff --git a/docs/programmers-guide.rst b/docs/programmers-guide.rst index a3891fe..92ae0e4 100644 --- a/docs/programmers-guide.rst +++ b/docs/programmers-guide.rst @@ -178,8 +178,9 @@ Making requests One request, one operation ^^^^^^^^^^^^^^^^^^^^^^^^^^ -:cpp:func:`Session::get` does a whole GET as one operation, and hands back the response as a plain -Beast message, with status, fields and the body as a ``std::string``: +:cpp:func:`Session::get` does a whole GET as one operation, and hands back the response as a +:cpp:class:`client::Message`: a plain ``boost::http::response``, with status and fields, and the +body as a ``std::string``: .. literalinclude:: snippets.cpp :language: cpp diff --git a/docs/reference.rst b/docs/reference.rst index 7ebe438..f5d3c10 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -28,7 +28,8 @@ Client .. doxygenclass:: anyhttp::client::Response :members: -.. doxygentypedef:: anyhttp::client::Message +.. doxygenclass:: anyhttp::client::Message + :members: Session ------- @@ -90,6 +91,8 @@ Headers and protocols .. doxygenfunction:: anyhttp::fields +.. doxygenfunction:: anyhttp::reason_phrase + .. doxygenclass:: anyhttp::FieldValue :members: @@ -239,6 +242,8 @@ against ``errc::``, which is ``boost::system::errc`` on ASIO and ``std::errc`` o .. doxygenvariable:: anyhttp::errors::header_limit +.. doxygenvariable:: anyhttp::errors::end_of_stream + Coroutines ^^^^^^^^^^ diff --git a/docs/snippets.cpp b/docs/snippets.cpp index 1c03522..aef57ee 100644 --- a/docs/snippets.cpp +++ b/docs/snippets.cpp @@ -152,7 +152,7 @@ Task get_body(Session& session, boost::urls::url url) if (ec) throw_error(ec); - std::println("{}: {} bytes", message.result_int(), message.body().size()); + std::println("{}: {} bytes", message.status_int(), message.body().size()); co_return std::move(message.body()); } // [get-end] @@ -169,7 +169,7 @@ Task read_at_most(Session session, boost::urls::url url, size_t limit) if (response_ec) co_return 0; std::println("{} {}", response.status_code(), - std::string_view(response.fields()["content-type"])); + std::string_view(response.fields().value_or("content-type", ""))); size_t total = 0; std::array buffer; diff --git a/docs/testing.rst b/docs/testing.rst index 170c9bf..7b706ba 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -80,7 +80,7 @@ A ``ClientAsync`` test body does not do any I/O itself. It only assigns up to tw clientSession = [this](Session session) -> Task { auto [ec, request] = co_await session.submit(url); auto [response_ec, response] = co_await request.get_response(); - EXPECT_EQ(response_ec, boost::beast::http::error::end_of_stream); + EXPECT_EQ(response_ec, errors::end_of_stream); }; } diff --git a/include/anyhttp/asio/any_async_stream.hpp b/include/anyhttp/asio/any_async_stream.hpp index 1478abd..44b8716 100644 --- a/include/anyhttp/asio/any_async_stream.hpp +++ b/include/anyhttp/asio/any_async_stream.hpp @@ -16,9 +16,6 @@ #include #include -#include -#include - #include #include @@ -163,8 +160,6 @@ class any_async_stream std::unique_ptr impl; }; -static_assert(boost::beast::is_async_stream::value); - // ------------------------------------------------------------------------------------------------- // diff --git a/include/anyhttp/asio/net.hpp b/include/anyhttp/asio/net.hpp index 8847235..86e645d 100644 --- a/include/anyhttp/asio/net.hpp +++ b/include/anyhttp/asio/net.hpp @@ -3,12 +3,12 @@ // // The network half of the runtime layer for Boost.Asio, see anyhttp/net.hpp. // -// The sessions are templates over the stream they run on, and there are four of those: a plain -// TCP socket, a TLS stream on top of one, beast's tcp_stream and the type-erased any_async_stream. -// Beyond the async read and write operations, which all of them have in common already, a session -// needs three more things from its stream: the underlying socket, to shut it down or close it, an -// executor to run its loops on, and whether it is encrypted (which tells h2 from h2c). None of them -// is spelled the same way by all four, so they are reached through this trait instead. Ending the +// The sessions are templates over the stream they run on, and there are three of those: a plain +// TCP socket, a TLS stream on top of one and the type-erased any_async_stream. Beyond the async +// read and write operations, which all of them have in common already, a session needs three more +// things from its stream: the underlying socket, to shut it down or close it, an executor to run +// its loops on, and whether it is encrypted (which tells h2 from h2c). None of them is spelled the +// same way by all three, so they are reached through this trait instead. Ending the // stream itself, which only TLS has to do, is async and comes as a free function below. // @@ -25,7 +25,6 @@ #include #include #include -#include #include #include @@ -95,17 +94,6 @@ struct stream_traits> static bool is_tls(const stream_type&) noexcept { return true; } }; -/// Beast's stream, which wraps a socket to add timeouts and a rate policy. -template -struct stream_traits> -{ - using stream_type = boost::beast::basic_stream; - - static auto& get_socket(stream_type& stream) noexcept { return stream.socket(); } - static auto get_executor(stream_type& stream) noexcept { return stream.get_executor(); } - static bool is_tls(const stream_type&) noexcept { return false; } -}; - /// The type-erased stream already offers all three, its implementation has to provide them. template <> struct stream_traits @@ -246,7 +234,6 @@ Task async_teardown(Stream& stream) static_assert(SocketStream); static_assert(SocketStream>); -static_assert(SocketStream); static_assert(SocketStream); static_assert(!SocketStream); // rvalues only, see above diff --git a/include/anyhttp/asio/runtime.hpp b/include/anyhttp/asio/runtime.hpp index a82cb80..b75b342 100644 --- a/include/anyhttp/asio/runtime.hpp +++ b/include/anyhttp/asio/runtime.hpp @@ -35,7 +35,7 @@ #include #include -#include +#include #include #include @@ -133,9 +133,10 @@ namespace errors inline const error_code eof = boost::asio::error::eof; /// An incoming body that ended before it was complete: a reset stream, a lost connection. -/// \runtimes{beast::http::error::partial_message,capy::error::stream_truncated} +/// \runtimes{boost::http::error::incomplete,capy::error::stream_truncated} /// \hideinitializer -inline const error_code partial_message = boost::beast::http::error::partial_message; +inline const error_code partial_message = + boost::http::make_error_code(boost::http::error::incomplete); /// An operation that was cancelled before it completed. /// \runtimes{errc::operation_canceled,capy::error::canceled} @@ -163,9 +164,16 @@ inline const error_code bad_descriptor = boost::asio::error::bad_descriptor; inline const error_code already_started = boost::asio::error::already_started; /// A header section larger than Config::max_header_size. -/// \runtimes{beast::http::error::header_limit,beast::http::error::header_limit} +/// \runtimes{boost::http::error::headers_limit,boost::http::error::headers_limit} /// \hideinitializer -inline const error_code header_limit = boost::beast::http::error::header_limit; +inline const error_code header_limit = + boost::http::make_error_code(boost::http::error::headers_limit); + +/// A connection that ended where a message could have begun: before a response, or between two. +/// \runtimes{boost::http::error::end_of_stream,boost::http::error::end_of_stream} +/// \hideinitializer +inline const error_code end_of_stream = + boost::http::make_error_code(boost::http::error::end_of_stream); } // namespace errors /// \cond internal diff --git a/include/anyhttp/client.hpp b/include/anyhttp/client.hpp index 5d2c394..680a390 100644 --- a/include/anyhttp/client.hpp +++ b/include/anyhttp/client.hpp @@ -6,8 +6,7 @@ #include -#include -#include +#include #include @@ -110,11 +109,26 @@ struct Config /** * A response received in one piece: status, header fields and the whole body as a string. * - * This is a plain Beast message, with nothing of anyhttp left in it -- what \c Session::async_get() - * hands back. Its version is 11 whatever the protocol was: HTTP/2 and HTTP/3 have no version on - * the wire, and a Beast message has nowhere else to put one. + * The status line and header fields are a plain \c boost::http::response, with nothing of anyhttp + * left in them -- \c status_int(), \c reason(), \c value_or(), iteration -- and the body comes + * with it. That is what \c Session::async_get() hands back. Its version is HTTP/1.1 whatever the + * protocol was: HTTP/2 and HTTP/3 have no version on the wire, and a response has nowhere else to + * put one. */ -using Message = boost::beast::http::response; +class Message : public boost::http::response +{ +public: + using boost::http::response::response; + + /// \return The body of the response. + std::string& body() noexcept { return body_; } + + /// \return The body of the response. + const std::string& body() const noexcept { return body_; } + +private: + std::string body_; +}; // ------------------------------------------------------------------------------------------------- @@ -227,8 +241,6 @@ class Request : public Writer Impl& pimpl() const noexcept; }; -// static_assert(boost::beast::is_async_write_stream::value); - // ================================================================================================= /// The completion signature of \c Client::async_connect: the error and the session. diff --git a/include/anyhttp/common.hpp b/include/anyhttp/common.hpp index 84082b5..68d6da4 100644 --- a/include/anyhttp/common.hpp +++ b/include/anyhttp/common.hpp @@ -3,10 +3,9 @@ #include #include -#include - -#include -#include +#include +#include +#include #include #include #include @@ -44,9 +43,28 @@ std::ostream& operator<<(std::ostream& str, Protocol protocol); // ================================================================================================= -/// The header fields of a message, in both directions: Beast's container for them. -using Fields = boost::beast::http::fields; -static_assert(boost::beast::http::is_fields::value); +/** + * The header fields of a message, in both directions: Boost.HTTP's container for them. + * + * It checks what goes into it: a name that is not a token, or a value with a line break or a NUL + * in it, makes \c set() and \c append() throw -- or report it, with the overloads that take an + * error code. Values lose their leading and trailing whitespace. + */ +using Fields = boost::http::fields; + +/** + * The reason phrase HTTP/1.1 has for a status code, for the logs: HTTP/2 and HTTP/3 have none. + * + * \param status_code The status code. + * \return The phrase, or an empty string for a code that has none. + */ +inline std::string_view reason_phrase(unsigned int status_code) +{ + const auto status = boost::http::int_to_status(status_code); + if (status == boost::http::status::unknown) + return {}; + return boost::http::to_string(status); +} /// Default for \c server::Config::max_header_size and \c client::Config::max_header_size. inline constexpr size_t default_max_header_size = 64 * 1024; @@ -70,7 +88,7 @@ constexpr size_t header_field_size(std::string_view name, std::string_view value * \c std::format can turn into a string, so sizes and counts need no conversion at the call site. * * String-like values are only referenced, never copied: a FieldValue lives just long enough for - * \c fields() to hand the bytes to Beast, which copies them. Only formatted values need storage. + * \c fields() to hand the bytes to the Fields, which copy them. Only formatted values need storage. */ class FieldValue { @@ -112,8 +130,8 @@ class FieldValue }; /** - * Beast's fields have no initializer-list constructor, so building a small, fixed set of headers - * takes a statement per header. This lets it be spelled inline: + * Fields have no initializer-list constructor, so building a small, fixed set of headers takes a + * statement per header. This lets it be spelled inline: * * \code * fields({{"Content-Length", body.size()}, {"Content-Type", "text/plain"}}) diff --git a/include/anyhttp/corosio/runtime.hpp b/include/anyhttp/corosio/runtime.hpp index 38e9706..2acaf33 100644 --- a/include/anyhttp/corosio/runtime.hpp +++ b/include/anyhttp/corosio/runtime.hpp @@ -35,10 +35,11 @@ #include #include -// the buffer vocabulary stays Boost.Asio's, which Beast needs anyway +// the buffer vocabulary stays Boost.Asio's, which the generic and protocol code is written in #include -#include +#include #include +#include #include #include @@ -119,7 +120,9 @@ inline const error_code bad_descriptor = std::make_error_code(std::errc::bad_fil inline const error_code already_started = std::make_error_code(std::errc::connection_already_in_progress); inline const error_code header_limit = - boost::system::error_code(boost::beast::http::error::header_limit); + boost::http::make_error_code(boost::http::error::headers_limit); +inline const error_code end_of_stream = + boost::http::make_error_code(boost::http::error::end_of_stream); } // namespace errors /// Whether the runtime supports running a server or client on several threads (with strands). diff --git a/include/anyhttp/detail/flat_buffer.hpp b/include/anyhttp/detail/flat_buffer.hpp new file mode 100644 index 0000000..54cce57 --- /dev/null +++ b/include/anyhttp/detail/flat_buffer.hpp @@ -0,0 +1,119 @@ +#pragma once + +// +// The bytes a session starts with, and what an HTTP/2 session reads into: a dynamic buffer in one +// piece of memory, with the prepare()/commit()/consume() interface of Asio's DynamicBuffer_v1. +// Beast's flat_buffer did this before; Boost.HTTP keeps its input inside its parser and has no +// such thing to offer. +// + +#include + +#include +#include +#include +#include +#include + +namespace anyhttp::detail +{ + +// ================================================================================================= + +class FlatBuffer +{ +public: + using const_buffers_type = boost::asio::const_buffer; + using mutable_buffers_type = boost::asio::mutable_buffer; + + FlatBuffer() = default; + FlatBuffer(FlatBuffer&& other) noexcept { *this = std::move(other); } + FlatBuffer& operator=(FlatBuffer&& other) noexcept + { + storage_ = std::move(other.storage_); + capacity_ = std::exchange(other.capacity_, 0); + begin_ = std::exchange(other.begin_, 0); + end_ = std::exchange(other.end_, 0); + prepared_ = std::exchange(other.prepared_, 0); + return *this; + } + + /// The readable bytes. + size_t size() const noexcept { return end_ - begin_; } + + /// How many bytes the buffer can hold without allocating. + size_t capacity() const noexcept { return capacity_; } + + const_buffers_type data() const noexcept { return {storage_.get() + begin_, size()}; } + mutable_buffers_type data() noexcept { return {storage_.get() + begin_, size()}; } + + /// Makes room for at least \p n bytes in all, without changing what is readable. + void reserve(size_t n) + { + if (n > capacity_) + grow(n); + } + + /// Space for \p n more bytes behind the readable ones, valid until the next modifying call. + mutable_buffers_type prepare(size_t n) + { + if (capacity_ - end_ < n) + { + if (capacity_ - size() >= n) + compact(); + else + grow(std::max(size() + n, 2 * capacity_)); + } + prepared_ = n; + return {storage_.get() + end_, n}; + } + + /// Makes \p n of the bytes last prepared readable. + void commit(size_t n) noexcept + { + end_ += std::min(n, prepared_); + prepared_ = 0; + } + + /// Drops the first \p n readable bytes. + void consume(size_t n) noexcept + { + begin_ += std::min(n, size()); + if (begin_ == end_) + begin_ = end_ = 0; + } + + void clear() noexcept { begin_ = end_ = prepared_ = 0; } + +private: + void compact() noexcept + { + if (begin_ == 0) + return; + if (size()) + std::memmove(storage_.get(), storage_.get() + begin_, size()); + end_ -= begin_; + begin_ = 0; + } + + void grow(size_t n) + { + auto storage = std::make_unique_for_overwrite(n); + if (size()) + std::memcpy(storage.get(), storage_.get() + begin_, size()); + end_ -= begin_; + begin_ = 0; + storage_ = std::move(storage); + capacity_ = n; + } + + std::unique_ptr storage_; + size_t capacity_ = 0; + size_t begin_ = 0; ///< first readable byte + size_t end_ = 0; ///< one past the last readable byte + size_t prepared_ = 0; ///< what prepare() handed out last +}; + +// ================================================================================================= + +} // namespace anyhttp::detail diff --git a/include/anyhttp/formatter.hpp b/include/anyhttp/formatter.hpp index c0c15be..ac8dc47 100644 --- a/include/anyhttp/formatter.hpp +++ b/include/anyhttp/formatter.hpp @@ -8,8 +8,8 @@ #include "anyhttp/asio/formatter.hpp" #endif -#include #include +#include #include #include #include @@ -58,12 +58,12 @@ struct std::formatter : public std::formatter -struct std::formatter +struct std::formatter { constexpr auto parse(std::format_parse_context& ctx) { return ctx.begin(); } template - auto format(const boost::beast::http::field& field, FormatContext& ctx) const + auto format(const boost::http::field& field, FormatContext& ctx) const { return std::format_to(ctx.out(), "{}", to_string(field)); } diff --git a/include/anyhttp/h1/backend.hpp b/include/anyhttp/h1/backend.hpp index f1acc68..c98f20b 100644 --- a/include/anyhttp/h1/backend.hpp +++ b/include/anyhttp/h1/backend.hpp @@ -3,7 +3,7 @@ // // The HTTP/1.1 backend as seen from the generic server and client: factories that turn a stream // that is ready to carry HTTP/1.1 into a Session::Impl. Everything else about the backend -- -// beast's parser, serializer and the session templates driving them -- stays in h1/session.hpp +// Boost.HTTP's parser, serializer and the session templates driving them -- stays in h1/session.hpp // and h1/session.cpp, which are the only places instantiating them. // @@ -24,7 +24,7 @@ namespace anyhttp::h1 // connection's strand if it has one: a corosio socket knows its context, not its strand. // // These are defined in src/h1/session.cpp and explicitly instantiated there for each of the -// stream types below, so that beast's HTTP machinery is instantiated in that one place only. +// stream types below, so that the sessions are instantiated in that one place only. // template diff --git a/include/anyhttp/h1/io.hpp b/include/anyhttp/h1/io.hpp index a0d6dc2..22f95af 100644 --- a/include/anyhttp/h1/io.hpp +++ b/include/anyhttp/h1/io.hpp @@ -1,264 +1,344 @@ #pragma once // -// HTTP/1.1 over a stream, with Beast's parser and serializer, but without Beast's asynchronous -// operations, which need an ASIO stream. These are the same algorithms (see read.hpp and write.hpp -// in boost/beast/http/impl), written as coroutines on the runtime layer so that they run on either -// runtime. +// HTTP/1.1 over a stream, with Boost.HTTP's parser and serializer. Both are sans-I/O: the parser +// takes input through prepare()/commit() and hands out the body with pull_body()/consume_body(), +// the serializer takes the body through stream_prepare()/stream_commit() and hands out what goes +// on the wire with prepare()/consume(). Boost.HTTP's own loops around them (parser::read(), +// serializer::sink) are written for capy's streams, so these are the same loops on the runtime +// layer, to run on either runtime. The framing -- chunks, Content-Length, the end of the body -- +// stays the library's. // -// Beast's parser and serializer say what they need through error codes of their own. need_more -// (more input) is handled in here. need_buffer (the body buffer is full, or used up) is what these -// are called to get to, so it comes out as success. Any other error comes out as the runtime's -// error_code, Beast's own ones (end_of_stream, header_limit, ...) included -- except for -// partial_message, which is errors::partial_message, see to_error_code(). +// What comes out of here is the runtime's error_code. Boost.HTTP's own codes stay what they are, +// with three exceptions, see to_error_code(): a message cut short is errors::partial_message, as +// the API has it, and every one of the parser's limits on the header section is +// errors::header_limit. // #include "anyhttp/common.hpp" #include "anyhttp/net.hpp" +#include "anyhttp/session_impl.hpp" -#include -#include -#include -#include -#include -#include -#include - +#include #include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include #include namespace anyhttp::h1 { -namespace http = boost::beast::http; +namespace http = boost::http; /** - * A Beast error as the runtime's error code. A body that ended before it was complete is - * \c errors::partial_message, as the API has it, which is Beast's own code only with ASIO. + * A Boost.HTTP error as the runtime's error code. A message that ended before it was complete is + * \c errors::partial_message, as the API has it. All of the parser's limits on the header section + * -- its total size, the start line, a single field, the number of fields -- are what + * Config::max_header_size is about, so they are all \c errors::header_limit. */ -inline error_code to_error_code(const boost::system::error_code& ec) +inline error_code to_error_code(const std::error_code& ec) { - if (ec == http::error::partial_message) + if (ec == http::error::incomplete) return errors::partial_message; - return ec; + if (ec == http::error::headers_limit || ec == http::error::start_line_limit || + ec == http::error::field_size_limit || ec == http::error::fields_limit) + return errors::header_limit; + return error_code(ec); } -// ================================================================================================= - /** - * Parses what \p buffer holds and, if the parser needs more, reads from \p stream into \p buffer - * first, until the parser has made progress. As Beast's \c http::async_read_some(), this yields the - * number of bytes the parser has used up. + * The parser's configuration for a header section of at most \p max_header_size bytes, see + * Config::max_header_size. * - * The end of the stream is \c http::error::end_of_stream if nothing of a message had arrived yet. - * Otherwise the parser decides: a message that ends with the connection is complete, any other one - * is a \c http::error::partial_message. + * Boost.HTTP limits the start line, every field and the number of fields on their own, HTTP/2 and + * HTTP/3 only the whole: the first two can be as large as the whole here, and there can be as many + * fields as fit when each one counts its name and value plus 32 bytes, as with HTTP/2 and HTTP/3 + * (see header_field_size()). The parser keeps a table entry for each field it may have, + * so the count is what bounds that table. * - * Over TLS, a connection that ends without a "close_notify" may have been cut by anyone, so no - * message can be complete because of it: that is a \c http::error::partial_message for everything - * that has begun. Between messages, it is just the end of the stream, as nothing can have been - * lost. + * There is no limit on the body: it is read piece by piece, never as a whole. */ -template -IoTask read_some(Stream& stream, DynamicBuffer& buffer, - http::basic_parser& parser) +inline http::shared_parser_config make_parser_config(bool server, size_t max_header_size) { - size_t parsed = 0; - boost::system::error_code ec; - for (bool read = buffer.size() == 0;; read = true) - { - if (read) - { - - const auto size = boost::beast::read_size(buffer, 65536); - if (size == 0) - co_return std::tuple{to_error_code(http::error::buffer_overflow), parsed}; - - auto [read_ec, n] = co_await io::read_some(stream, buffer.prepare(size)); - buffer.commit(n); - if (read_ec == errors::eof) - { - if (!parser.got_some()) - co_return std::tuple{to_error_code(http::error::end_of_stream), parsed}; - - // the caller sees the end of the stream with the next read - ec = {}; - parser.put_eof(ec); - co_return std::tuple{to_error_code(ec), parsed}; - } - if (io::is_truncated(read_ec)) - co_return std::tuple{to_error_code(parser.got_some() ? http::error::partial_message - : http::error::end_of_stream), - parsed}; - if (read_ec) - co_return std::tuple{read_ec, parsed}; - } - - const auto used = parser.put(buffer.data(), ec); - parsed += used; - buffer.consume(used); - if (ec != http::error::need_more) - break; - } - - if (ec == http::error::need_buffer) - ec = {}; - co_return std::tuple{to_error_code(ec), parsed}; + http::parser_config config(server); + config.headers.max_size = max_header_size; + config.headers.max_start_line = max_header_size; + config.headers.max_field = max_header_size; + config.headers.max_fields = std::max(max_header_size / header_field_size("", ""), 1); + config.body_limit = std::numeric_limits::max(); + return http::make_parser_config(config); } -/// Reads until the parser has the whole header, as Beast's \c http::async_read_header(). -template -IoTask read_header(Stream& stream, DynamicBuffer& buffer, - http::basic_parser& parser) +/** + * The serializer's configuration. Its buffer is what a write copies the body into, a piece at a + * time, framing included: the larger, the fewer writes a large body takes. + */ +inline http::shared_serializer_config make_serializer_config() { - size_t parsed = 0; - while (!parser.is_header_done()) - { - - auto [ec, n] = co_await h1::read_some(stream, buffer, parser); - if (ec) - co_return std::tuple{ec, parsed}; - parsed += n; - } - co_return std::tuple{error_code{}, parsed}; + http::serializer_config config; + config.payload_buffer = 64 * 1024; + return http::make_serializer_config(config); } -/// Reads until the parser has the whole message, as Beast's \c http::async_read() with a parser. -template -IoTask read(Stream& stream, DynamicBuffer& buffer, http::basic_parser& parser) +// ================================================================================================= + +/** + * Gives the parser more input: what is left of \p pending first, the bytes a session started with + * (see Session::Impl::do_session()), and only then what can be read from \p stream. + * + * The end of the stream goes to the parser, which knows whether a message can end with it: the + * next parse() says. Over TLS, a connection that ends without a "close_notify" may have been cut + * by anyone, so no message can be complete because of it: once the header is there, that is a + * \c errors::partial_message for the body. Before, the parser can tell nothing of a message from + * a message cut short, as with any other end of the stream. + */ +template +IoTask<> fill(Stream& stream, Buffer& pending, http::parser& parser) { - size_t parsed = 0; - while (!parser.is_done()) + auto buffers = parser.prepare(); + auto it = std::ranges::find_if(buffers, [](const auto& buffer) { return buffer.size() > 0; }); + if (it == buffers.end()) + co_return std::tuple{to_error_code(http::error::buffer_overflow)}; + const boost::asio::mutable_buffer buffer(it->data(), it->size()); + + if (pending.size()) { - auto [ec, n] = co_await h1::read_some(stream, buffer, parser); - if (ec) - co_return std::tuple{ec, parsed}; - parsed += n; + const auto n = boost::asio::buffer_copy(buffer, pending.data()); + pending.consume(n); + parser.commit(n); + co_return std::tuple{error_code{}}; } - co_return std::tuple{error_code{}, parsed}; -} -/// Reads a whole message into \p message, as Beast's \c http::async_read() with a message. -template -IoTask read_message(Stream& stream, DynamicBuffer& buffer, - http::message>& message) -{ - http::parser parser; - parser.eager(true); - auto [ec, n] = co_await h1::read(stream, buffer, parser); - if (!ec) - message = parser.release(); - co_return std::tuple{ec, n}; + auto [ec, n] = co_await io::read_some(stream, buffer); + if (ec == errors::eof) + parser.commit_eof(); + else if (io::is_truncated(ec)) + { + if (parser.got_header()) + co_return std::tuple{errors::partial_message}; + parser.commit_eof(); + } + else if (ec) + co_return std::tuple{ec}; + else + parser.commit(n); + co_return std::tuple{error_code{}}; } -// ------------------------------------------------------------------------------------------------- - /** - * Counts how many of the bytes written come from \c range, the caller's body buffer, which the - * serializer hands out by reference among the header and the chunk framing. That is how much of - * the body a write that stops short has taken. + * Reads until the parser has the whole header section. + * + * The end of the stream is \c errors::end_of_stream if nothing of a message had arrived yet, and a + * \c errors::partial_message otherwise. */ -struct BodyWatch +template +IoTask<> read_header(Stream& stream, Buffer& pending, http::parser& parser) { - boost::asio::const_buffer range; - size_t written = 0; -}; + for (;;) + { + std::error_code ec; + parser.parse(ec); + if (parser.got_header()) + co_return std::tuple{error_code{}}; + if (ec != http::condition::need_more_input) + co_return std::tuple{to_error_code(ec)}; + + if (auto [fill_ec] = co_await h1::fill(stream, pending, parser); fill_ec) + co_return std::tuple{fill_ec}; + } +} /** - * Writes what \p serializer has to offer next, as Beast's \c http::async_write_some(). Yields the - * number of bytes written, which is not the size of the body that went with them: \p watch, if - * given, counts those. - * - * With a \c buffer_body, a serializer that has used up the body buffer it was given offers nothing - * but \c http::error::need_buffer, which is passed on here. + * Reads some of the body into \p buffer, as much as is there, reading from the stream only if + * nothing is. Yields the number of bytes copied, and \c errors::eof once the parser has the whole + * message and all of the body has been taken. */ -template -IoTask write_some(Stream& stream, http::serializer& serializer, - BodyWatch* watch = nullptr) +template +IoTask read_body(Stream& stream, Buffer& pending, http::parser& parser, + boost::asio::mutable_buffer buffer) { - if (serializer.is_done()) - co_return std::tuple{error_code{}, size_t{0}}; - - // - // The buffers next() offers are valid until consume(), but the sequence it offers them in is - // not: take its descriptors out of the visitor. - // - boost::system::error_code ec; - boost::container::small_vector buffers; - serializer.next(ec, [&](boost::system::error_code& next_ec, const auto& sequence) { - next_ec = {}; - for (boost::asio::const_buffer buffer : boost::beast::buffers_range_ref(sequence)) - buffers.push_back(buffer); - }); - if (ec) - co_return std::tuple{to_error_code(ec), size_t{0}}; - - auto [write_ec, n] = co_await io::write(stream, buffers); - if (!write_ec) - serializer.consume(n); - - if (watch) + for (;;) { - const auto* begin = static_cast(watch->range.data()); - const auto* end = begin + watch->range.size(); - size_t left = n; - for (const auto& buffer : buffers) + std::error_code ec; + parser.parse(ec); + + size_t copied = 0; + for (const auto& body : parser.pull_body()) { - const auto size = std::min(left, buffer.size()); - const auto* data = static_cast(buffer.data()); - if (size && data >= begin && data + buffer.size() <= end) - watch->written += size; - left -= size; + const auto n = + boost::asio::buffer_copy(boost::asio::buffer(buffer + copied), + boost::asio::const_buffer(body.data(), body.size())); + copied += n; + if (n < body.size()) + break; } + parser.consume_body(copied); + if (copied) + co_return std::tuple{error_code{}, copied}; + + if (parser.is_complete()) + co_return std::tuple{errors::eof, size_t{0}}; + + // more input, or room for it, which taking what was there has just made + if (ec && ec != http::condition::need_more_input && ec != http::error::in_place_overflow) + co_return std::tuple{to_error_code(ec), size_t{0}}; + + if (auto [fill_ec] = co_await h1::fill(stream, pending, parser); fill_ec) + co_return std::tuple{fill_ec, size_t{0}}; } - co_return std::tuple{write_ec, n}; } +// ------------------------------------------------------------------------------------------------- + /** - * Writes what \p serializer has to offer until \p done says so. As \c need_buffer is not an error - * here, this also stops when a \c buffer_body has been written that is not the last. + * Writes what \p serializer has to offer, until it needs more of the body or is done. Yields the + * number of bytes written, framing and all. + * + * The serializer stops after the header of a message that expects "100 Continue" as well, which + * is no reason to stop here: whoever asked for that has decided to send the body right away. */ -template -IoTask write_until(Stream& stream, Serializer& serializer, Predicate done, - BodyWatch* watch = nullptr) +template +IoTask flush(Stream& stream, http::serializer& serializer) { size_t written = 0; - while (!done(serializer)) + while (!serializer.is_done()) { - auto [ec, n] = co_await h1::write_some(stream, serializer, watch); + auto prepared = serializer.prepare(); + if (prepared.has_error()) + { + if (prepared.error() == http::error::need_data) + break; + if (prepared.error() == http::error::expect_100_continue) + continue; + co_return std::tuple{to_error_code(prepared.error()), written}; + } + + // + // The buffers prepare() hands out are valid until consume(), the sequence it hands them out + // in is not: copy the descriptors. + // + boost::container::small_vector buffers; + for (const auto& buffer : *prepared) + if (buffer.size()) + buffers.emplace_back(buffer.data(), buffer.size()); + if (buffers.empty()) + { + serializer.consume(0); + continue; + } + + auto [ec, n] = co_await io::write(stream, buffers); + serializer.consume(n); written += n; - if (ec == to_error_code(http::error::need_buffer)) - break; if (ec) co_return std::tuple{ec, written}; } co_return std::tuple{error_code{}, written}; } -/// Writes the header, as Beast's \c http::async_write_header(). -template -IoTask write_header(Stream& stream, Serializer& serializer) +/** + * Starts \p message on \p serializer and writes its header, the body to follow by write_body(). + * A message that has no body -- \p body is false -- is done with this. + */ +template +IoTask<> write_header(Stream& stream, http::serializer& serializer, + const http::message_base& message, bool body = true) { - serializer.split(true); - return h1::write_until(stream, serializer, [](auto& sr) { return sr.is_header_done(); }); + serializer.reset(); + serializer.set_message(message); + if (body) + serializer.start_buffers(); + else + serializer.start(); + auto [ec, n] = co_await h1::flush(stream, serializer); + co_return std::tuple{ec}; } -/// Writes all there is, as Beast's \c http::async_write() with a serializer. -template -IoTask write(Stream& stream, Serializer& serializer, BodyWatch* watch = nullptr) +/** + * Writes \p buffer as body, and ends the body if \p eof says so. The serializer takes the body by + * copying it, framing it as it goes, as much at a time as its buffer holds: when \p eof is set, + * the end of the body goes with the last of it. + * + * Yields the number of bytes of \p buffer the serializer has taken. Those are on their way, but not + * necessarily on the wire when the write fails: with a stream that can not be written to any more, + * that is of no consequence. + */ +template +IoTask write_body(Stream& stream, http::serializer& serializer, + boost::asio::const_buffer buffer, bool eof) +{ + size_t taken = 0; + for (;;) + { + // + // One commit per prepare: it invalidates what prepare handed out, and with chunked encoding, + // it is what makes a chunk of what has been copied. + // + const auto space = serializer.stream_prepare(); + const std::array spaces{ + boost::asio::mutable_buffer(space[0].data(), space[0].size()), + boost::asio::mutable_buffer(space[1].data(), space[1].size())}; + const auto n = boost::asio::buffer_copy(spaces, buffer + taken); + serializer.stream_commit(n); + taken += n; + + const bool last = taken == buffer.size(); + if (last && eof) + serializer.stream_close(); + + auto [ec, written] = co_await h1::flush(stream, serializer); + if (ec || last) + co_return std::tuple{ec, taken}; + } +} + +// ------------------------------------------------------------------------------------------------- + +/** + * Writes all of \p message, with \p body as its body: the counterpart of read_message(), for + * talking HTTP/1.1 by hand. The message says how the body is framed, by 'Content-Length' or + * 'Transfer-Encoding: chunked'. + */ +template +IoTask<> write_message(Stream& stream, http::serializer& serializer, + const http::message_base& message, std::string_view body = {}) { - return h1::write_until(stream, serializer, [](auto& sr) { return sr.is_done(); }, watch); + if (auto [ec] = co_await h1::write_header(stream, serializer, message); ec) + co_return std::tuple{ec}; + auto [ec, n] = co_await h1::write_body(stream, serializer, boost::asio::buffer(body), true); + co_return std::tuple{ec}; } -/// Writes all of \p message, as Beast's \c http::async_write() with a message. -template -IoTask write_message(Stream& stream, http::message& message) +/** + * Reads a whole message with \p parser, which must have been started on it, and yields its body: + * the header is the parser's. For talking HTTP/1.1 by hand. + */ +template +IoTask read_message(Stream& stream, Buffer& pending, http::parser& parser) { - http::serializer serializer{message}; - co_return co_await h1::write(stream, serializer); + std::string body; + if (auto [ec] = co_await h1::read_header(stream, pending, parser); ec) + co_return std::tuple{ec, std::move(body)}; + + std::array buffer; + for (;;) + { + auto [ec, n] = co_await h1::read_body(stream, pending, parser, boost::asio::buffer(buffer)); + body.append(buffer.data(), n); + if (ec == errors::eof) + co_return std::tuple{error_code{}, std::move(body)}; + if (ec) + co_return std::tuple{ec, std::move(body)}; + } } // ================================================================================================= diff --git a/include/anyhttp/h1/session.hpp b/include/anyhttp/h1/session.hpp index 0ecd0fd..2f8a4ee 100644 --- a/include/anyhttp/h1/session.hpp +++ b/include/anyhttp/h1/session.hpp @@ -8,9 +8,9 @@ #include -#include -#include -#include +#include +#include +#include #include #include @@ -56,14 +56,20 @@ class StreamOp // ================================================================================================= +/** + * What the server and client sessions have in common: the stream, and what goes on and comes off + * it. That is one message at a time in each direction, so a session has one parser (in the role's + * own session) and one serializer, which the readers and writers use in turn: each of them + * allocates all the memory it will ever need when it is made, and keeps it for every message. + */ template -class BeastSession : public ::anyhttp::Session::Impl +class H1Session : public ::anyhttp::Session::Impl { protected: - BeastSession(std::string_view log_prefix, Executor executor, Stream&& stream); + H1Session(std::string_view log_prefix, Executor executor, Stream&& stream); public: - ~BeastSession() override; + ~H1Session() override; std::string_view log_prefix() const { return log_prefix_; } @@ -120,7 +126,12 @@ class BeastSession : public ::anyhttp::Session::Impl std::string log_prefix_; Executor executor_; Stream stream_; - Buffer buffer_; + + /// What the session started with, see do_session(), on its way into the parser. + Buffer pending_; + + boost::http::serializer serializer_; + bool closed_ = false; bool stream_closed_ = false; ///< set by end_connection(), with nothing left for destroy() to do std::shared_ptr stream_ops_ = std::make_shared(); @@ -150,16 +161,17 @@ class ServerSessionBase }; template -class ServerSession : public ServerSessionBase, public BeastSession +class ServerSession : public ServerSessionBase, public H1Session { - using super = BeastSession; + using super = H1Session; // FIXME: maybe use CRTP or something similar to avoid this? - using super::buffer_; using super::closed_; using super::detach_readers; using super::detach_writers; using super::log_prefix; + using super::pending_; + using super::serializer_; using super::stream_; public: @@ -170,6 +182,9 @@ class ServerSession : public ServerSessionBase, public BeastSession const Fields& headers) override; Task do_session(Buffer&& data) override; + /// The requests, one after the other. + boost::http::request_parser parser_; + private: /// Takes over the stream after an upgrade to h2c, see do_session(). std::shared_ptr upgraded_; @@ -230,15 +245,15 @@ class RequestWriter; * asking for its response -- getting any later response fails. */ template -class ClientSession : public ClientSessionBase, public BeastSession +class ClientSession : public ClientSessionBase, public H1Session { - using super = BeastSession; + using super = H1Session; // FIXME: maybe use CRTP or something similar to avoid this? - using super::buffer_; using super::detach_readers; using super::detach_writers; using super::log_prefix; + using super::pending_; using super::stream_; using super::stream_ops_; @@ -275,6 +290,13 @@ class ClientSession : public ClientSessionBase, public BeastSession bool send_failed_ = false; ///< set when a request could not be completed bool receive_failed_ = false; ///< set when a response could not be read completely bool shutting_down_ = false; ///< set by shutdown() + + /// The responses, one after the other. + boost::http::response_parser parser_; + + /// Whether the parser has been started on the response that is read next. A read of a header + /// that was cancelled before anything arrived leaves it started, see get_response(). + bool parser_started_ = false; }; // ================================================================================================= diff --git a/include/anyhttp/h2/session.hpp b/include/anyhttp/h2/session.hpp index 9a05058..0d3d26a 100644 --- a/include/anyhttp/h2/session.hpp +++ b/include/anyhttp/h2/session.hpp @@ -8,7 +8,6 @@ #include "anyhttp/session_impl.hpp" #include -#include #include #include diff --git a/include/anyhttp/h2/session_details.hpp b/include/anyhttp/h2/session_details.hpp index b0e7f81..d32b9e1 100644 --- a/include/anyhttp/h2/session_details.hpp +++ b/include/anyhttp/h2/session_details.hpp @@ -11,7 +11,6 @@ #include "anyhttp/net.hpp" #include -#include #include #include #include diff --git a/include/anyhttp/h2/stream.hpp b/include/anyhttp/h2/stream.hpp index f84fbfd..e91b9d7 100644 --- a/include/anyhttp/h2/stream.hpp +++ b/include/anyhttp/h2/stream.hpp @@ -8,7 +8,6 @@ #include "nghttp2/nghttp2.h" #include -#include #include #include #include diff --git a/include/anyhttp/h2v2/session.hpp b/include/anyhttp/h2v2/session.hpp index 1a74b2f..3e4aff7 100644 --- a/include/anyhttp/h2v2/session.hpp +++ b/include/anyhttp/h2v2/session.hpp @@ -17,7 +17,6 @@ #include "anyhttp/session_impl.hpp" #include -#include #include #include diff --git a/include/anyhttp/h2v2/stream.hpp b/include/anyhttp/h2v2/stream.hpp index 2d6a48c..9f34e32 100644 --- a/include/anyhttp/h2v2/stream.hpp +++ b/include/anyhttp/h2v2/stream.hpp @@ -13,7 +13,6 @@ #include #include -#include #include #include #include diff --git a/include/anyhttp/h3/stream.hpp b/include/anyhttp/h3/stream.hpp index 624c80d..81c05d7 100644 --- a/include/anyhttp/h3/stream.hpp +++ b/include/anyhttp/h3/stream.hpp @@ -6,7 +6,6 @@ #include "anyhttp/writer_impl.hpp" #include -#include #include #include diff --git a/include/anyhttp/session.hpp b/include/anyhttp/session.hpp index 7a82a8e..cf99bed 100644 --- a/include/anyhttp/session.hpp +++ b/include/anyhttp/session.hpp @@ -165,9 +165,9 @@ class Session * Any error along the way completes this operation: the ones \ref async_submit() describes, * \c errors::header_limit for a response header section over * \c client::Config::max_header_size, and \c errors::partial_message for a body cut - * short. The message that comes with an error is empty, and says \c status::unknown rather - * than the 200 a default-constructed Beast response would claim. A response that says 404, on - * the other hand, is not an error -- it is a response, and arrives as one. + * short. The message that comes with an error is empty, and has a status of 0 rather than + * the 200 a default-constructed response would claim. A response that says 404, on the other + * hand, is not an error -- it is a response, and arrives as one. * * \param url The URL of the request. * \param headers The header fields of the request. diff --git a/include/anyhttp/session_impl.hpp b/include/anyhttp/session_impl.hpp index b39865b..809471d 100644 --- a/include/anyhttp/session_impl.hpp +++ b/include/anyhttp/session_impl.hpp @@ -2,14 +2,15 @@ #include "session.hpp" -#include +#include "anyhttp/detail/flat_buffer.hpp" #include namespace anyhttp { -using Buffer = boost::beast::flat_buffer; +/// What a session starts with: the bytes read from the connection before it took over. +using Buffer = detail::FlatBuffer; // ================================================================================================= diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7360850..3a40b10 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -42,7 +42,7 @@ if (USE_LIBCXX) target_compile_options(anyhttp PUBLIC $<$:-stdlib=libc++>) target_link_options(anyhttp PUBLIC -stdlib=libc++) endif() -target_link_libraries(anyhttp PUBLIC Threads::Threads Boost::headers Boost::url) +target_link_libraries(anyhttp PUBLIC Threads::Threads Boost::headers Boost::url anyhttp::http) target_link_libraries(anyhttp PUBLIC spdlog::spdlog_header_only) target_link_libraries(anyhttp PRIVATE Boost::thread Boost::atomic Boost::filesystem) @@ -88,9 +88,3 @@ if (ANYHTTP_COROSIO) return() endif() - -# -# beast_issue_3032 -# -add_executable(beast_issue_3032) -target_sources(beast_issue_3032 PRIVATE "issues/beast_issue_3032.cpp") diff --git a/src/client_impl.cpp b/src/client_impl.cpp index 8f88fa1..d57f6d7 100644 --- a/src/client_impl.cpp +++ b/src/client_impl.cpp @@ -6,8 +6,6 @@ #include "anyhttp/h2/backend.hpp" #include "anyhttp/h3/backend.hpp" -#include -#include #include #include @@ -279,7 +277,7 @@ Session Client::Impl::start_session(Protocol protocol, Stream&& stream) }; // - // FIXME: Do we really need to "run" a session here? For nghttp2, yes. For beast, not so much, + // FIXME: Do we really need to "run" a session here? For nghttp2, yes. For HTTP/1.1, not so much, // as there is no communication outside the current request right now, but that may // still come with pipelining or if it is refactored to have the parser and serializer // in the session object itself. diff --git a/src/h1/session.cpp b/src/h1/session.cpp index fae8854..d088283 100644 --- a/src/h1/session.cpp +++ b/src/h1/session.cpp @@ -11,63 +11,73 @@ #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include #include #include +#include +#include #include #include -#include +#include #include #include #include using namespace std::chrono_literals; -using namespace boost::asio; -namespace beast = boost::beast; -namespace http = beast::http; - namespace anyhttp::h1 { // ================================================================================================= using namespace boost::asio; -using namespace boost::beast; +using http::field; /** * Adds the user's header fields to an outgoing message. A field replaces whatever the message * already has under that name, like a default set before, but repeated fields are all kept. */ -template -static void add_fields(http::message& message, const Fields& headers) +static void add_fields(http::message_base& message, const Fields& headers) { for (auto&& header : headers) - message.erase(header.name_string()); + message.erase(header.name); for (auto&& header : headers) - message.insert(header.name_string(), header.value()); + message.append(header.name, header.value); +} + +/// A copy of the header fields of \p message, which stay with the parser only until the next one. +static Fields copy_fields(const http::fields_base& message) +{ + Fields fields; + fields.reserve_bytes(message.buffer().size()); + for (auto&& header : message) + fields.append(header.name, header.value); + return fields; +} + +/// Whether \p message has a 'Content-Length', valid or not. +static bool has_content_length(const http::message_base& message) +{ + return message.metadata().content_length.count > 0; } -/// Converts Config::max_header_size into what a Beast parser takes as its header limit. -static std::uint32_t header_limit(size_t max_header_size) +/// Sets the 'Content-Length' of \p message, or removes it. +static void set_content_length(http::message_base& message, std::optional content_length) { - return static_cast( - std::min(max_header_size, std::numeric_limits::max())); + if (content_length) + message.set_content_length(*content_length); + else + message.erase(field::content_length); } // ================================================================================================= @@ -75,38 +85,41 @@ static std::uint32_t header_limit(size_t max_header_size) namespace { -template -class BeastReader : public Interface +/** + * The incoming half of a message, as a Reader. The parser is the session's, see H1Session: its + * header is good until the next message, so the reader keeps a copy of what it hands out. It reads + * through the parser while it is attached, and not at all any more once it is detached. + */ +template +class H1Reader : public Interface { public: - BeastReader(BeastSession& session_, Stream& stream_, Buffer& buffer_) - : session(&session_), stream(stream_), buffer(buffer_), + H1Reader(H1Session& session_, Parser& parser_) + : session(&session_), parser(&parser_), executor_(session_.get_executor()), // survives detach(), see get_executor() log_prefix_(session_.log_prefix()) { - // parser.header_limit(std::numeric_limits::max()); - parser.body_limit(std::numeric_limits::max()); session_.attach(*this); } void destroy() noexcept override { - mlogd("destroy: reader destroyed, is_done={}", parser.is_done()); - if (!parser.is_done() && session) + mlogd("destroy: reader destroyed, complete={}", complete); + if (!complete && session) { mlogw("destroy: reader destroyed, but parser not done yet... closing socket"); // // We could still try to send an error response here. // This breaks WHEN_server_discards_request_THEN_is_still_able_to_deliver_response. // - auto ec = io::shutdown(stream, io::Shutdown::receive); + auto ec = io::shutdown(session->stream_, io::Shutdown::receive); session->closed_ = true; logwd(ec != errc::not_connected, "[{}] destroy: shutdown: {}", log_prefix(), what(ec)); } finish(); } - ~BeastReader() override + ~H1Reader() override { assert(!reading); finish(); @@ -114,29 +127,45 @@ class BeastReader : public Interface session->release(*this); } + /// Takes what is needed of the header, once the parser has it. + void on_header() + { + const auto& message = parser->get(); + fields_ = copy_fields(message); + + // + // What the message announced as its body length. A response to HEAD has no payload whatever + // its 'Content-Length' says, but that is still what it announced, and what HTTP/2 and HTTP/3 + // report for it. + // + const auto& content_length = message.metadata().content_length; + if (message.payload() == http::payload::size) + content_length_ = message.payload_size(); + else if (content_length.count && !content_length.ec) + content_length_ = content_length.value; + else if (message.payload() == http::payload::none) + content_length_ = 0; + complete = parser->is_complete(); + } + /// Tells the session, once, that this reader is done with the stream, see reader_finished(). void finish() { if (session && !finished) { finished = true; - session->reader_finished(parser.is_done()); + session->reader_finished(complete); } } void detach() override { mlogd("detach"); session = nullptr; + parser = nullptr; } - const Fields& fields() const override { return parser.get(); } - std::optional content_length() const noexcept override - { - if (parser.content_length()) - return parser.content_length().value(); - else - return std::nullopt; - } + const Fields& fields() const override { return fields_; } + std::optional content_length() const noexcept override { return content_length_; } // // The I/O is done in a coroutine, see Reader::Impl::read_some(). As a task of its own, it is @@ -155,78 +184,58 @@ class BeastReader : public Interface // // Everything that can be answered without touching the connection, each case with the error // code the Reader contract prescribes for it (see common.hpp): a zero-length read is not a - // read and reports nothing, wherever the body stands; a parser that is done keeps reporting - // the end of the body, ASIO-style, session or no session; and a read past a detached, - // unfinished parser is a truncation the caller must hear about. + // read and reports nothing, wherever the body stands; a message that has been read keeps + // reporting the end of the body, ASIO-style, session or no session; and a read past a + // detached, unfinished one is a truncation the caller must hear about. // if (body_buffer.size() == 0) co_return std::tuple{error_code{}, size_t{0}}; - else if (parser.is_done()) + else if (eof) co_return std::tuple{errors::eof, size_t{0}}; else if (!session) - co_return std::tuple{errors::partial_message, size_t{0}}; - - buffer.reserve(64_k); // the buffer is the session's, so not before checking for it - mlogd("async_read_some: is_done={} size={} capacity={}", parser.is_done(), buffer.size(), - buffer.capacity()); + co_return std::tuple{complete ? errors::eof : errors::partial_message, size_t{0}}; auto self = Interface::shared_from_this(); reading = true; - for (;;) - { - parser.get().body().data = body_buffer.data(); - parser.get().body().size = body_buffer.size(); - error_code ec; - size_t n = 0; - try - { - const auto op = session->read_op(); - std::tie(ec, n) = co_await h1::read_some(stream, buffer, parser); - } - catch (const system_error& e) // a cancelled coroutine throws from its next co_await - { - ec = e.code(); - } - - auto& body = parser.get().body(); - size_t payload = body_buffer.size() - body.size; - mlogd("async_read_some: n={} (body={}) ({}) is_done={} size={} capacity={}", n, payload, - ec.message(), parser.is_done(), buffer.size(), buffer.capacity()); + error_code ec; + size_t n = 0; + try + { + const auto op = session->read_op(); + std::tie(ec, n) = + co_await h1::read_body(session->stream_, session->pending_, *parser, body_buffer); + } + catch (const system_error& e) // a cancelled coroutine throws from its next co_await + { + ec = e.code(); + } - if (parser.is_done()) - finish(); + if (parser) // still attached + complete = parser->is_complete(); + if (ec == errors::eof) + eof = true; + mlogd("async_read_some: n={} ({}) complete={}", n, ec.message(), complete); - // - // Nothing of the body came out of this round -- either the parser is done now, which - // is the end of the body, or it just needs more input. Either way there is nothing to - // hand to the caller yet. - // - if (!ec && payload == 0) - { - if (parser.is_done()) - ec = errors::eof; - else if (!session) - ec = errors::partial_message; - else - continue; - } + if (complete) + finish(); - reading = false; - co_return std::tuple{ec, payload}; - } + reading = false; + co_return std::tuple{ec, n}; } Executor get_executor() const noexcept override { return executor_; } std::string log_prefix() const override { return log_prefix_; } - BeastSession* session; - Stream& stream; - Buffer& buffer; - Parser parser; + H1Session* session; + Parser* parser; ///< the session's, see on_header() + Fields fields_; + std::optional content_length_; Executor executor_; // kept as a copy so a detached reader can still complete std::string log_prefix_; // likewise, for logging bool reading = false; + bool complete = false; ///< whether the parser has all of the message + bool eof = false; ///< whether all of the body has been read, too bool finished = false; // see finish() }; @@ -235,54 +244,53 @@ class BeastReader : public Interface // // The two roles a reader can be in. Everything above is the same for both; what they add is the // half of the incoming message that only their role has -- a request line, or a status code. -// Which of the two Beast parsers a role reads with follows from the role itself. +// Which of the two parsers a role reads with follows from the role itself. // -template -class BeastRequestReader final - : public BeastReader> +template +class RequestReader final : public H1Reader { - using Base = BeastReader>; + using Base = H1Reader; public: using Base::Base; - std::string_view method() const noexcept override { return this->parser.get().method_string(); } + std::string_view method() const noexcept override { return method_; } boost::url_view url() const override { return url_; } + /// Copied from the request line, see H1Reader::on_header(). + std::string method_; + /// Assembled from the request target, the host header and the kind of socket this arrived on, /// by the session, right after the header was parsed. boost::url url_; }; -template -class BeastResponseReader final - : public BeastReader> +template +class ResponseReader final : public H1Reader { - using Base = BeastReader>; + using Base = H1Reader; public: using Base::Base; - unsigned int status_code() const noexcept override { return this->parser.get().result_int(); } + unsigned int status_code() const noexcept override { return status_code_; } + + unsigned int status_code_ = 0; }; // ------------------------------------------------------------------------------------------------- /** - * Common implementation of server::Response and client::Request writer. + * Common implementation of server::Response and client::Request writer. The message is the + * writer's own, the serializer the session's: there is only one message at a time to write. */ -template > +template class WriterBase : public Parent { public: - WriterBase(BeastSession& session_, Stream& stream_) - : session(&session_), stream(stream_), + explicit WriterBase(H1Session& session_) + : session(&session_), executor_(session_.get_executor()), // survives detach(), see get_executor() log_prefix_(session_.log_prefix()) { @@ -308,7 +316,7 @@ class WriterBase : public Parent session = nullptr; } - /// The I/O is done in a coroutine, see Reader::Impl::read_some() and BeastReader. + /// The I/O is done in a coroutine, see Reader::Impl::read_some() and H1Reader. void async_write(WriteHandler&& handler, boost::asio::const_buffer buffer, bool eof) override { launch(get_executor(), write(buffer, eof), std::move(handler)); @@ -351,57 +359,42 @@ class WriterBase : public Parent writing = true; mlogd("async_write: {} bytes (eof={} chunked={} content_length={})", buffer.size(), eof, - message.chunked(), message.has_content_length()); + message.metadata().transfer_encoding.is_chunked, has_content_length(message)); // - // The last body buffer and the end of the body go into the same serializer pass: with - // 'more' cleared, beast emits the data and the terminating chunk (or just the data, for a - // content-length delimited body) in one go, so ending a body costs no extra write. - // -#if BOOST_BEAST_VERSION < 359 - // https://github.com/boostorg/beast/issues/3032 - // make sure to set 'nullptr' on empty size, otherwise beast may serialize an empty chunk - message.body().data = buffer.size() ? const_cast(buffer.data()) : nullptr; -#else - message.body().data = const_cast(buffer.data()); -#endif - message.body().size = buffer.size(); - message.body().more = !eof; - + // The last body buffer and the end of the body go into the same serializer pass, so ending + // a body costs no extra write: the data and the terminating chunk (or just the data, for a + // content-length delimited body) go out together. // - // With 'chunked' transfer encoding, the serializer will automatically emit a chunk as - // large as possible. This means that if the user writes a single large buffer, cancellation - // can not be done gracefully at chunk boundary any more. See 'Cancellation' testcase for an - // example of this. + // With 'chunked' transfer encoding, the serializer makes a chunk of as much as its buffer + // holds. So cancellation can not be done gracefully at a chunk boundary of the caller's. + // See 'Cancellation' testcase for an example of this. // auto self = Parent::shared_from_this(); const auto expected = buffer.size(); - // 'n' is the number of bytes written to the stream, 'body' counts those taken from the buffer + // 'taken' is how much of the buffer the serializer has copied, see h1::write_body() error_code ec; - size_t n = 0; - h1::BodyWatch body{buffer}; + size_t taken = 0; try { const auto op = session->write_op(); - std::tie(ec, n) = co_await h1::write(stream, serializer, &body); + std::tie(ec, taken) = + co_await h1::write_body(session->stream_, session->serializer_, buffer, eof); } catch (const system_error& e) // a cancelled coroutine throws from its next co_await { ec = e.code(); } - mlogd("async_write: n={} (\x1b[1;{}m{}\x1b[0m) done={} (body {})", n, ec ? 31 : 32, - ec.message(), serializer.is_done(), serializer.get().body().size); + mlogd("async_write: taken={} (\x1b[1;{}m{}\x1b[0m)", taken, ec ? 31 : 32, ec.message()); writing = false; if (ec == errc::operation_canceled) { // - // Cancellation is tricky, see e.g.: https://github.com/boostorg/beast/issues/2325. - // - // Main reason is that, depending on when the cancellation actually takes place, - // the stream is in an undefined state. For example, when writing a large chunk is + // Cancellation is tricky: depending on when the cancellation actually takes place, the + // stream is in an undefined state. For example, when writing a large chunk is // interrupted, there is no meaningful way to recover: The length of the chunk has // been written, but only part of the data. // @@ -410,25 +403,18 @@ class WriterBase : public Parent // TODO: We could try to support partial cancellation, but that would only work // at chunk boundaries. // - mlogi("async_write: canceled after writing {} of {} bytes", n, expected); + mlogi("async_write: canceled after taking {} of {} bytes", taken, expected); cancelled = true; if (session) // otherwise, the stream is gone already { mlogi("async_write: canceled, closing stream"); - io::shutdown(stream, io::Shutdown::send); + io::shutdown(session->stream_, io::Shutdown::send); } } else if (ec) { cancelled = true; } - /* - else if (!ec && n < expected) - { - mlogw("async_write: wrote {} bytes which is less than expected ({})", n, expected); - ec = make_error_code(errc::message_size); - } - */ // // Only now is the body really ended: a cancelled or failed EOF write never got its @@ -448,7 +434,7 @@ class WriterBase : public Parent // effect immediately, not only when this task is gone. // self.reset(); - co_return std::tuple{ec, ec ? body.written : expected}; + co_return std::tuple{ec, ec ? taken : expected}; } /// Called when a write has ended the body, before its handler is invoked. @@ -459,14 +445,15 @@ class WriterBase : public Parent // ---------------------------------------------------------------------------------------------- - /// Writes the header of the message, see submit_headers(). - IoTask<> write_head() + /// Writes the header of the message, see submit_headers(). With \p body cleared, that is all. + IoTask<> write_head(bool body = true) { error_code ec; try { const auto op = session->write_op(); - std::tie(ec, std::ignore) = co_await h1::write_header(stream, serializer); + std::tie(ec) = + co_await h1::write_header(session->stream_, session->serializer_, message, body); } catch (const system_error& e) // a cancelled coroutine throws from its next co_await { @@ -480,23 +467,27 @@ class WriterBase : public Parent */ void submit_headers(const Fields& headers) { - message.body().data = nullptr; - add_fields(message, headers); - if (!message.has_content_length()) - message.chunked(true); + if (!has_content_length(message)) + message.set_chunked(true); - mlogd("async_submit: chunked={} has_content_length={} length={}", message.chunked(), - message.has_content_length(), message.payload_size().value_or(0)); + mlogd("async_submit: chunked={} has_content_length={} length={}", + message.metadata().transfer_encoding.is_chunked, has_content_length(message), + message.metadata().content_length.value); + } + + /// Logs the header section of the message, start line excluded. + void log_fields() const + { + for (const auto& header : message) + mlogd(" \x1b[1;34m{}\x1b[0m: {}", truncated(header.name), truncated(header.value)); } // ---------------------------------------------------------------------------------------------- - BeastSession* session; - Stream& stream; + H1Session* session; Message message; - Serializer serializer{message}; Executor executor_; // kept as a copy so a detached writer can still complete std::string log_prefix_; // likewise, for logging bool writing = false; @@ -508,23 +499,19 @@ class WriterBase : public Parent // ------------------------------------------------------------------------------------------------- template -class ResponseWriter - : public WriterBase> +class ResponseWriter : public WriterBase { - using super = - WriterBase>; + using super = WriterBase; public: using super::cancelled; using super::log_prefix; using super::message; - using super::serializer; using super::session; - using super::stream; using super::submit_headers; public: - ResponseWriter(BeastSession& session_, Stream& stream_) : super(session_, stream_) {} + explicit ResponseWriter(H1Session& session_) : super(session_) {} // // A response that could not be written leaves the connection unusable, so the session must not @@ -536,13 +523,10 @@ class ResponseWriter void content_length(std::optional content_length) override { - if (content_length) - message.content_length(*content_length); - else - message.content_length(boost::none); + set_content_length(message, content_length); } - /// The I/O is done in a coroutine, see BeastReader. + /// The I/O is done in a coroutine, see H1Reader. void async_submit(StatusHandler&& handler, unsigned int status_code, const Fields& headers) override { @@ -557,10 +541,11 @@ class ResponseWriter co_return std::tuple{errors::connection_aborted}; } - message.result(status_code); + const auto status = static_cast(status_code); + message.set_start_line(status, reason_phrase(status)); - if (message.find(http::field::date) == message.end()) - message.set(http::field::date, std::string_view(http_date_now())); + if (!message.exists(field::date)) + message.set(field::date, std::string_view(http_date_now())); submit_headers(headers); @@ -573,13 +558,11 @@ class ResponseWriter if (!message.keep_alive()) session->closed_ = true; - if (message.find(http::field::server) == message.end()) - message.set(http::field::server, "anyhttp"); + if (!message.exists(field::server)) + message.set(field::server, "anyhttp"); - mlogd("{} {}", message.result_int(), message.reason()); - for (const auto& header : message) - mlogd(" \x1b[1;34m{}\x1b[0m: {}", truncated(header.name_string()), - truncated(header.value())); + mlogd("{} {}", message.status_int(), message.reason()); + super::log_fields(); // // TODO: For bundling writing the header and body, we should just post the writing here, @@ -600,11 +583,9 @@ class ResponseWriter } // namespace template -class RequestWriter - : public WriterBase> +class RequestWriter : public WriterBase { - using super = - WriterBase>; + using super = WriterBase; public: using super::cancelled; @@ -612,13 +593,11 @@ class RequestWriter using super::log_prefix; using super::message; using super::response_requested; - using super::serializer; using super::session; - using super::stream; using super::submit_headers; public: - RequestWriter(ClientSession& session_, Stream& stream_) : super(session_, stream_) {} + explicit RequestWriter(ClientSession& session_) : super(session_) {} ~RequestWriter() override { @@ -642,13 +621,12 @@ class RequestWriter */ bool has_body() const { - if (message.chunked()) + const auto& metadata = message.metadata(); + if (metadata.transfer_encoding.count) return true; - - auto value = message[http::field::content_length]; - size_t length = 0; - auto [end, ec] = std::from_chars(value.data(), value.data() + value.size(), length); - return ec != std::errc{} || end != value.data() + value.size() || length != 0; + if (!metadata.content_length.count) + return false; + return metadata.content_length.ec || metadata.content_length.value != 0; } /// Called when writing the header is done. A request without a body is complete by then. @@ -674,10 +652,7 @@ class RequestWriter void content_length(std::optional content_length) override { - if (content_length) - message.content_length(*content_length); - else - message.content_length(boost::none); + set_content_length(message, content_length); } using super::get_executor; @@ -697,11 +672,11 @@ class RequestWriter } submit_headers(headers); - message.method(http::verb::post); - co_return co_await super::write_head(); + message.set_method(http::method::post); + co_return co_await super::write_head(has_body()); } - /// The I/O is done in a coroutine, see BeastReader. + /// The I/O is done in a coroutine, see H1Reader. void async_get_response(client::Request::GetResponseHandler&& handler) override { launch(get_executor(), get_response(), std::move(handler)); @@ -743,21 +718,28 @@ class RequestWriter } response_requested = true; - auto& buffer = session->buffer_; - auto reader = - std::make_unique, decltype(buffer)>>( - *session, stream, buffer); - auto& parser = reader->parser; - parser.header_limit(header_limit(cs.client().config().max_header_size)); - // whatever its header field says, a response to HEAD is over after that (RFC 9112, 6.3) - parser.skip(message.method() == http::verb::head); - - mlogd("waiting for response (size={} capacity={})", buffer.size(), buffer.capacity()); - size_t len = 0; + // + // The parser is the session's. It is done with the previous response, or has not seen one + // yet, unless an earlier attempt to read this one was cancelled, see below: then it goes on + // where that one left off. + // + auto& parser = cs.parser_; + if (!std::exchange(cs.parser_started_, true)) + { + // whatever its header field says, a response to HEAD is over after that (RFC 9112, 6.3) + if (message.method() == http::method::head) + parser.start_head_response(); + else + parser.start(); + } + + auto reader = std::make_unique>(*session, parser); + + mlogd("waiting for response ({} bytes pending)", session->pending_.size()); try { const auto op = session->read_op(); - std::tie(ec, len) = co_await h1::read_header(stream, buffer, parser); + std::tie(ec) = co_await h1::read_header(session->stream_, session->pending_, parser); } catch (const system_error& e) // a cancelled coroutine throws from its next co_await { @@ -766,36 +748,39 @@ class RequestWriter if (!ec) { - auto& msg = parser.get(); - mlogd("{} {}", msg.result_int(), msg.reason()); + const auto& msg = parser.get(); + reader->status_code_ = msg.status_int(); + reader->on_header(); + mlogd("{} {}", msg.status_int(), msg.reason()); for (const auto& header : msg) - mlogd(" \x1b[1;34m{}\x1b[0m: {}", truncated(header.name_string()), - truncated(header.value())); + mlogd(" \x1b[1;34m{}\x1b[0m: {}", truncated(header.name), truncated(header.value)); // // An "Alt-Svc" on any response may point at an HTTP/3 endpoint to use for the next // connection (RFC 7838), see Client::Impl::on_alt_svc(). // - if (auto alt_svc = msg[http::field::alt_svc]; session && !alt_svc.empty()) + if (auto alt_svc = msg.value_or(field::alt_svc, ""); session && !alt_svc.empty()) client_session().client().on_alt_svc(std::string_view(alt_svc)); } else - mlogw("async_read_header: {} len={}", ec.message(), len); + mlogw("async_read_header: {}", ec.message()); // - // If reading the headers was cancelled before receiving anything, we can allow another - // attempt. TODO: If we move the parser into the session, we can even relax this further. + // If reading the header was cancelled before it was complete, another attempt can pick up + // where this one left off: the parser is the session's, and keeps what it has got so far. // - // As this reader has not taken anything from the connection, it does not count as having - // failed to read the response, either. + // As this reader has not taken anything from the connection that the next one would not + // take, it does not count as having failed to read the response, either. // - if (ec == errc::operation_canceled && !parser.got_some()) + if (ec == errc::operation_canceled && !parser.got_header()) { response_requested = false; reader->finished = true; } + else + cs.parser_started_ = false; - if (!ec && parser.is_done()) // a response without body is complete already + if (!ec && reader->complete) // a response without body is complete already reader->finish(); co_return std::tuple{ec, client::Response(std::move(reader))}; @@ -810,14 +795,15 @@ class RequestWriter // ================================================================================================= template -BeastSession::BeastSession(std::string_view prefix, Executor executor, Stream&& stream) - : executor_(std::move(executor)), log_prefix_(prefix), stream_(std::move(stream)) +H1Session::H1Session(std::string_view prefix, Executor executor, Stream&& stream) + : log_prefix_(prefix), executor_(std::move(executor)), stream_(std::move(stream)), + serializer_(h1::make_serializer_config()) { mlogd("session created"); } template -BeastSession::~BeastSession() +H1Session::~H1Session() { mlogd("session deleted"); if (!writers_.empty()) @@ -833,8 +819,10 @@ ServerSession::ServerSession(server::Server::Impl& parent, Executor exec Stream&& stream) : ServerSessionBase(parent), super(anyhttp::log_prefix(Role::server, "h1", io::remote_endpoint(stream)), - std::move(executor), std::move(stream)) + std::move(executor), std::move(stream)), + parser_(h1::make_parser_config(true, parent.config().max_header_size)) { + parser_.reset(); } template @@ -842,12 +830,14 @@ ClientSession::ClientSession(client::Client::Impl& parent, Executor exec Stream&& stream) : ClientSessionBase(parent), super(anyhttp::log_prefix(Role::client, "h1", io::remote_endpoint(stream)), - std::move(executor), std::move(stream)) + std::move(executor), std::move(stream)), + parser_(h1::make_parser_config(false, parent.config().max_header_size)) { + parser_.reset(); } template -Task BeastSession::end_connection(bool notify) +Task H1Session::end_connection(bool notify) { // // End the stream itself first: over TLS, that is the "close_notify" the peer needs to tell the @@ -878,7 +868,7 @@ Task BeastSession::end_connection(bool notify) // ------------------------------------------------------------------------------------------------- template -void BeastSession::destroy() noexcept +void H1Session::destroy() noexcept { if (stream_closed_) return; @@ -920,6 +910,67 @@ void ServerSession::destroy() noexcept // ================================================================================================= +/// Whether the comma-separated \p list has \p token in it, in any case. +static bool has_token(std::string_view list, std::string_view token) +{ + auto tokens = boost::urls::grammar::parse(list, http::list_rule(http::token_rule, 1)); + return tokens && std::ranges::any_of(*tokens, [&](auto item) { + return boost::urls::grammar::ci_is_equal(item, token); + }); +} + +/** + * Decodes \p text from base64url without padding (RFC 4648, section 5), what 'HTTP2-Settings' + * holds (RFC 7540, section 3.2.1). Fails on anything else. + */ +static std::optional base64url_decode(std::string_view text) +{ + const auto value = [](char c) -> int { + if (c >= 'A' && c <= 'Z') + return c - 'A'; + if (c >= 'a' && c <= 'z') + return c - 'a' + 26; + if (c >= '0' && c <= '9') + return c - '0' + 52; + if (c == '-') + return 62; + if (c == '_') + return 63; + return -1; + }; + + if (text.size() % 4 == 1) + return std::nullopt; + + std::string result; + result.reserve(text.size() * 3 / 4); + unsigned bits = 0; + int count = 0; + for (char c : text) + { + const int v = value(c); + if (v < 0) + return std::nullopt; + bits = (bits << 6) | static_cast(v); + if (++count == 4) + { + result.push_back(static_cast(bits >> 16)); + result.push_back(static_cast(bits >> 8)); + result.push_back(static_cast(bits)); + bits = 0; + count = 0; + } + } + if (count == 2) + result.push_back(static_cast(bits >> 4)); + else if (count == 3) + { + result.push_back(static_cast(bits >> 10)); + result.push_back(static_cast(bits >> 2)); + } + return result; +} + /** * Returns what the HTTP/2 session needs to continue \p request as stream 1, if the request asks * for an upgrade to h2c (RFC 7540, section 3.2) and it can be granted. Otherwise, the upgrade is @@ -930,7 +981,7 @@ void ServerSession::destroy() noexcept * negotiated by ALPN instead. */ static std::optional h2c_upgrade(std::string_view log_prefix, - const http::request& request, + const http::static_request& request, const boost::urls::url& url, bool complete) { // @@ -942,18 +993,11 @@ static std::optional h2c_upgrade(std::string_view log_prefix, if constexpr (ANYHTTP_H2_V2) return std::nullopt; - const auto has_token = [](std::string_view list, std::string_view token) { - for (auto item : http::token_list(list)) - if (beast::iequals(item, token)) - return true; - return false; - }; - - if (!has_token(request[http::field::upgrade], "h2c") || url.scheme() != "http") + if (!has_token(request.value_or(field::upgrade, ""), "h2c") || url.scheme() != "http") return std::nullopt; - if (!has_token(request[http::field::connection], "upgrade") || - !has_token(request[http::field::connection], "http2-settings")) + const auto connection = request.value_or(field::connection, ""); + if (!has_token(connection, "upgrade") || !has_token(connection, "http2-settings")) { logw("[{}] upgrade: ignoring h2c upgrade, 'Connection' misses 'Upgrade' or 'HTTP2-Settings'", log_prefix); @@ -961,7 +1005,7 @@ static std::optional h2c_upgrade(std::string_view log_prefix, } // exactly one HTTP2-Settings header, containing base64url without padding - if (request.count("HTTP2-Settings") != 1) + if (request.count(field::http2_settings) != 1) { logw("[{}] upgrade: ignoring h2c upgrade, need exactly one 'HTTP2-Settings' header", log_prefix); @@ -974,45 +1018,29 @@ static std::optional h2c_upgrade(std::string_view log_prefix, return std::nullopt; } - std::string encoded(request["HTTP2-Settings"]); - std::ranges::replace(encoded, '-', '+'); - std::ranges::replace(encoded, '_', '/'); - - h2::Upgrade upgrade; - upgrade.settings.resize(beast::detail::base64::decoded_size(encoded.size())); - auto [written, read] = - beast::detail::base64::decode(upgrade.settings.data(), encoded.data(), encoded.size()); - upgrade.settings.resize(written); - // a SETTINGS payload is a sequence of 6-byte entries - if (read != encoded.size() || upgrade.settings.size() % 6 != 0) + auto settings = base64url_decode(request.at(field::http2_settings)); + if (!settings || settings->size() % 6 != 0) { logw("[{}] upgrade: ignoring h2c upgrade, invalid 'HTTP2-Settings' header", log_prefix); return std::nullopt; } - upgrade.method = request.method_string(); + h2::Upgrade upgrade; + upgrade.settings = std::move(*settings); + upgrade.method = request.method_text(); upgrade.url = url; // // HTTP/2 has no connection-specific header fields (RFC 9113, section 8.2.2), and the upgrade // ones are used up by now. // - for (const auto& field : request) - { - switch (field.name()) - { - case http::field::connection: - case http::field::proxy_connection: - case http::field::keep_alive: - case http::field::transfer_encoding: - case http::field::upgrade: - case http::field::http2_settings: - break; - default: - upgrade.fields.insert(field.name_string(), field.value()); - } - } + static constexpr std::array connection_specific{field::connection, field::proxy_connection, + field::keep_alive, field::transfer_encoding, + field::upgrade, field::http2_settings}; + for (const auto& item : request) + if (!item.id || !std::ranges::contains(connection_specific, *item.id)) + upgrade.fields.append(item.name, item.value); return upgrade; } @@ -1022,39 +1050,52 @@ static std::optional h2c_upgrade(std::string_view log_prefix, * waiting again. * * Requests are served one at a time, also when the client pipelines them: those that follow wait in - * the buffer, and the next one is read only after the handler of the previous one has returned, so + * the parser, and the next one is read only after the handler of the previous one has returned, so * the responses go out in order. A response the handler has not completed by then is detached, and * writing to it fails with \c connection_aborted. * - * Requests and responses that outlive the session are detached as well, see ~BeastSession(). + * Requests and responses that outlive the session are detached as well, see ~H1Session(). */ template Task ServerSession::do_session(Buffer&& buffer) { - buffer_ = std::move(buffer); + pending_ = std::move(buffer); - mlogd("do_server_session, {} bytes in buffer", buffer_.size()); + mlogd("do_server_session, {} bytes in buffer", pending_.size()); size_t requestCounter = 0; while (!closed_) { detach_readers(); // the previous request, if still around, is done with the stream - auto reader = std::make_unique>( - *this, stream_, buffer_); + + // + // What is left of the previous message can not be told apart from whatever follows it on + // the connection, and a response that was begun and not ended leaves the client waiting for + // the rest of it: either way, nothing more can go over this connection. + // + if (requestCounter && !parser_.is_complete()) + { + mlogw("previous request not read completely, closing connection"); + break; + } + if (!serializer_.is_done()) + { + mlogw("previous response not complete, closing connection"); + break; + } + + parser_.start(); + auto reader = std::make_unique>(*this, parser_); logd(""); - mlogd("waiting for request (size={} capacity={})", buffer_.size(), buffer_.capacity()); - auto& parser = reader->parser; - parser.header_limit(header_limit(server().config().max_header_size)); - auto [ec, len] = co_await h1::read_header(stream_, buffer_, parser); + mlogd("waiting for request ({} bytes pending)", pending_.size()); + auto [ec] = co_await h1::read_header(stream_, pending_, parser_); if (!ec) - mlogd("async_read_header: len={} size={} capacity={} ec={}", len, buffer_.size(), - buffer_.capacity(), ec.message()); - else if (ec == h1::to_error_code(http::error::end_of_stream)) + mlogd("async_read_header: ok"); + else if (ec == errors::end_of_stream) mlogd("async_read_header: end of stream"); else - mlogw("async_read_header: len={} size={} capacity={} ec=\x1b[1;31m{}\x1b[0m", len, - buffer_.size(), buffer_.capacity(), ec.message()); + mlogw("async_read_header: \x1b[1;31m{}\x1b[0m", ec.message()); // // The rest of the request can not be told apart from whatever follows it on the connection, @@ -1062,25 +1103,26 @@ Task ServerSession::do_session(Buffer&& buffer) // if (ec == errors::header_limit) { - http::response res{http::status::request_header_fields_too_large, 11}; - res.set(http::field::server, "anyhttp"); - res.set(http::field::connection, "close"); - res.content_length(0); - if (auto [ec, n] = co_await h1::write_message(stream_, res); ec) + http::response res(http::status::request_header_fields_too_large); + res.set(field::server, "anyhttp"); + res.set(field::connection, "close"); + res.set_content_length(0); + if (auto [ec] = co_await h1::write_header(stream_, serializer_, res, false); ec) mlogw("writing 431 response: {}", ec.message()); } if (ec) break; requestCounter++; - auto& request = parser.get(); - const bool need_eof = request.need_eof(); + auto& request = parser_.get(); + const bool need_eof = !request.keep_alive(); + reader->method_ = request.method_text(); + reader->on_header(); - // if (auto url = boost::urls::parse_relative_ref(request.target()); url.has_value()) if (auto url = boost::urls::parse_uri_reference(request.target()); url.has_value()) reader->url_ = url.value(); else - mlogw("{} {}: invalid target: {}", request.method_string(), request.target(), + mlogw("{} {}: invalid target: {}", request.method_text(), request.target(), url.error().message()); // @@ -1097,44 +1139,52 @@ Task ServerSession::do_session(Buffer&& buffer) else reader->url_.set_scheme("http"); + const auto host = request.value_or(field::host, ""); try { - reader->url_.set_encoded_authority(request[http::field::host]); + reader->url_.set_encoded_authority(host); } catch (std::exception& ex) { - mlogw("ignoring invalid host header: {}", request[http::field::host]); + mlogw("ignoring invalid host header: {}", host); } - mlogd("{} {} (need_eof={})", request.method_string(), reader->url_.buffer(), need_eof); - for (auto& header : request) - mlogd(" \x1b[1;34m{}\x1b[0m: {}", truncated(header.name_string()), - truncated(header.value())); + mlogd("{} {} (need_eof={})", request.method_text(), reader->url_.buffer(), need_eof); + for (const auto& header : request) + mlogd(" \x1b[1;34m{}\x1b[0m: {}", truncated(header.name), truncated(header.value)); // // Upgrade to h2c, if requested: Answer with "101 Switching Protocols" and hand over the // stream to an HTTP/2 session, which continues this request as stream 1. Anything that - // follows in the buffer (the client preface) is already HTTP/2. + // follows the request (the client preface) is already HTTP/2: what the parser holds of it, + // and what it has not taken yet. // - if (auto upgrade = h2c_upgrade(log_prefix(), request, reader->url_, parser.is_done())) + if (auto upgrade = h2c_upgrade(log_prefix(), request, reader->url_, parser_.is_complete())) { - http::response res{http::status::switching_protocols, request.version()}; - res.set(http::field::connection, "Upgrade"); - res.set(http::field::upgrade, "h2c"); - reader.reset(); // owns the parser and thereby 'request' - - if (auto [ec, n] = co_await h1::write_message(stream_, res); ec) + Buffer data; + const auto leftover = parser_.release_buffered_data(); + buffer_copy(data.prepare(leftover.size()), const_buffer(leftover.data(), leftover.size())); + data.commit(leftover.size()); + buffer_copy(data.prepare(pending_.size()), pending_.data()); + data.commit(pending_.size()); + pending_.clear(); + reader.reset(); + + http::response res(http::status::switching_protocols); + res.set(field::connection, "Upgrade"); + res.set(field::upgrade, "h2c"); + if (auto [ec] = co_await h1::write_header(stream_, serializer_, res, false); ec) { mlogw("upgrade: writing 101 response: {}", ec.message()); break; } - mlogi("upgrading to h2c, {} bytes in buffer", buffer_.size()); + mlogi("upgrading to h2c, {} bytes in buffer", data.size()); // Stream is whatever this session runs on, but never a TLS one: h2c_upgrade() takes // cleartext requests only, as h2 over TLS is negotiated by ALPN instead. upgraded_ = h2::make_server_session(server(), super::get_executor(), std::move(stream_), std::move(*upgrade)); - co_await upgraded_->do_session(std::move(buffer_)); + co_await upgraded_->do_session(std::move(data)); mlogi("h2c session done, served {} requests before upgrade", requestCounter - 1); co_return; } @@ -1143,11 +1193,10 @@ Task ServerSession::do_session(Buffer&& buffer) // Prepare response. // detach_writers(); // likewise for the previous response - auto writer = std::make_shared>(*this, stream_); + auto writer = std::make_shared>(*this); - http::response& response = writer->message; - http::response_serializer& serializer = writer->serializer; - response.set(http::field::server, "anyhttp"); + http::response& response = writer->message; + response.set(field::server, "anyhttp"); // // Point the client at our HTTP/3 endpoint, see server::Config::alt_svc_max_age. Set before @@ -1155,7 +1204,7 @@ Task ServerSession::do_session(Buffer&& buffer) // simply pass its own field: submitting a response replaces the fields it names. // if (const auto& alt_svc = server().alt_svc(); !alt_svc.empty()) - response.set(http::field::alt_svc, alt_svc); + response.set(field::alt_svc, alt_svc); // // A client that asked for the connection to end -- or one speaking HTTP/1.0, which has no @@ -1163,7 +1212,7 @@ Task ServerSession::do_session(Buffer&& buffer) // is not allowed to sidestep this by keeping the connection alive. // if (need_eof) - response.keep_alive(false); + response.set_keep_alive(false); // // Call user-provided request handler. @@ -1192,14 +1241,7 @@ Task ServerSession::do_session(Buffer&& buffer) } } - // - // FIXME: Maybe we shouldn't hand out put the parser object to the request handler. If the - // request gets dropped, we don't know anything about the stream's state any more and - // whether or not we can try to read a new request. - // - // We should have a parser here, and call is_done() on it. - // - mlogd("request handler finished (size={} capacity={})", buffer_.size(), buffer_.capacity()); + mlogd("request handler finished ({} bytes pending)", pending_.size()); // // Honor 'Connection: close', even if the handler has not responded at all. A response that @@ -1224,9 +1266,9 @@ Task ServerSession::do_session(Buffer&& buffer) template Task ClientSession::do_session(Buffer&& buffer) { - buffer_ = std::move(buffer); + pending_ = std::move(buffer); - mlogd("do_client_session, {} bytes in buffer", buffer_.size()); + mlogd("do_client_session, {} bytes in buffer", pending_.size()); // Set the low-level TCP stream timeout. This is relevant for some testcases... // stream_.expires_after(5s); @@ -1236,7 +1278,7 @@ Task ClientSession::do_session(Buffer&& buffer) // and the session only keeps them in order (see ClientSession). Pipelining needs no queue // either, as a request is written as a whole before the next one may be submitted, and // responses are read in the order of their requests. Requests and responses that outlive the - // session are detached, see ~BeastSession(). + // session are detached, see ~H1Session(). // // @@ -1292,7 +1334,7 @@ void ServerSession::async_submit(SubmitHandler&& handler, std::string_vi assert(false); } -/// The I/O is done in a coroutine, see BeastReader. +/// The I/O is done in a coroutine, see H1Reader. template void ClientSession::async_submit(SubmitHandler&& handler, std::string_view method, boost::urls::url target, const Fields& headers) @@ -1322,32 +1364,39 @@ IoTask ClientSession::submit(std::string method, boost: co_return std::tuple{ec, client::Request{}}; } - auto writer = std::make_unique>(*this, stream_); + auto writer = std::make_unique>(*this); auto& request = writer->message; // // FIXME: https://datatracker.ietf.org/doc/html/rfc7230#section-5.3 // Usually, target should be in "origin form", but we need to support other forms, too. // - request.base().target(target.encoded_target()); - // request.base().target(url.buffer()); - request.method_string(method); - request.set(http::field::user_agent, "anyhttp"); - add_fields(request, headers); - if (request.find(http::field::host) == request.end()) - request.set(http::field::host, target.encoded_host_and_port()); - if (!request.has_content_length()) - request.chunked(true); - - mlogd("{} {}", request.method_string(), target.buffer()); - for (const auto& header : request) - mlogd(" \x1b[1;34m{}\x1b[0m: {}", truncated(header.name_string()), - truncated(header.value())); + // The request checks what it is given: a method or target with whitespace in it, or a header + // field that is not one, is not something to put on the wire. + // + try + { + request.set_start_line(method, target.encoded_target()); + request.set(field::user_agent, "anyhttp"); + add_fields(request, headers); + if (!request.exists(field::host)) + request.set(field::host, target.encoded_host_and_port()); + if (!has_content_length(request)) + request.set_chunked(true); + } + catch (const std::exception& ex) + { + mlogw("async_submit: invalid request: {}", ex.what()); + co_return std::tuple{make_error_code(errc::invalid_argument), client::Request{}}; + } + + mlogd("{} {}", request.method_text(), target.buffer()); + writer->log_fields(); writer->sequence = requests_sent_++; sending_ = writer.get(); - std::tie(ec) = co_await writer->write_head(); + std::tie(ec) = co_await writer->write_head(writer->has_body()); writer->header_written(ec); co_return std::tuple{ec, client::Request(std::move(writer))}; } @@ -1374,10 +1423,17 @@ template void ClientSession::request_released(RequestWriter& request) { // - // The connection is in the middle of this request, and nothing else can be sent any more. + // The connection is in the middle of this request, and nothing else can be sent any more. Nor + // is the rest of this one coming: the end of the stream is what tells the server, which would + // otherwise wait for it, and the client for the response, forever. // if (sending_ == &request) + { request_failed(request); + auto ec = io::shutdown(stream_, io::Shutdown::send); + logwd(ec && ec != errc::not_connected, "[{}] request #{} released: shutdown: {}", + log_prefix(), request.sequence, ec.message()); + } // // Its response will be coming, but nobody is going to read it. @@ -1409,7 +1465,8 @@ ANYHTTP_SERVER_STREAMS(ANYHTTP_H1_SESSION) // ================================================================================================= // Factories, see anyhttp/h1/backend.hpp. Instantiating the session templates is kept to this -// translation unit, so that the generic server and client stay free of beast's HTTP machinery. +// translation unit, so that the generic server and client stay free of Boost.HTTP's parser and +// serializer. // ================================================================================================= template diff --git a/src/h2/session.cpp b/src/h2/session.cpp index 50d3fa7..eeec6ac 100644 --- a/src/h2/session.cpp +++ b/src/h2/session.cpp @@ -12,11 +12,6 @@ #include -#include -#include -#include -#include - #include #include @@ -30,8 +25,6 @@ #include #include -namespace http = boost::beast::http; - // ================================================================================================= namespace anyhttp::h2 @@ -186,7 +179,7 @@ static int on_header_callback(nghttp2_session* session, const nghttp2_frame* fra } if (!name.starts_with(':')) - stream->fields.insert(name, value); + stream->fields.append(name, value); } catch (std::exception& ex) { @@ -311,12 +304,11 @@ static int on_frame_recv_callback(nghttp2_session* session, const nghttp2_frame* case NGHTTP2_HEADERS: { assert(stream); - using namespace boost::beast::http; if (frame->headers.cat == NGHTTP2_HCAT_REQUEST) logd("[{}] {} {}", stream->log_prefix(), stream->method, stream->url.buffer()); else if (frame->headers.cat == NGHTTP2_HCAT_RESPONSE && stream->status_code) logd("[{}] {} {}", stream->log_prefix(), *stream->status_code, - obsolete_reason(int_to_status(*stream->status_code))); + reason_phrase(*stream->status_code)); stream->log_received_headers(); if (frame->headers.cat == NGHTTP2_HCAT_REQUEST) @@ -327,7 +319,7 @@ static int on_frame_recv_callback(nghttp2_session* session, const nghttp2_frame* // A response may carry an alternative service as a header field instead of, or as well // as, in an ALTSVC frame -- the two say the same thing in the same syntax. // - if (auto alt_svc = stream->fields["alt-svc"]; !alt_svc.empty()) + if (auto alt_svc = stream->fields.value_or("alt-svc", ""); !alt_svc.empty()) handler->on_alt_svc(std::string_view(alt_svc)); stream->on_response(); @@ -555,11 +547,11 @@ void NGHttp2Session::async_submit(SubmitHandler&& handler, std::string_view meth for (auto&& item : headers) { - if (item.name_string().starts_with(':')) + if (item.name.starts_with(':')) logw("[{}] async_submit: invalid header '{}': setting pseudo headers is not allowed", - stream->log_prefix(), item.name_string()); + stream->log_prefix(), item.name); - nva.push_back(make_nv_ls(item.name_string(), item.value())); + nva.push_back(make_nv_ls(item.name, item.value)); } logd("[{}] {} {}", stream->log_prefix(), method_str, url.buffer()); @@ -726,7 +718,7 @@ void NGHttp2Session::close_stream(int32_t stream_id) if (stream->response_handler) { - auto ec = http::make_error_code(http::error::end_of_stream); // be compatible with beast + auto ec = errors::end_of_stream; // as HTTP/1.1 has it // ec = errc::make_error_code(errc::connection_reset); swap_and_invoke(stream->response_handler, ec, client::Response{nullptr}); } diff --git a/src/h2/stream.cpp b/src/h2/stream.cpp index 26937d8..e75c2da 100644 --- a/src/h2/stream.cpp +++ b/src/h2/stream.cpp @@ -11,9 +11,6 @@ #include -#include -#include - #include #include @@ -258,9 +255,7 @@ void NGHttp2Writer::async_submit(StatusHandler&& handler, unsigned int sta return; } - using namespace boost::beast::http; - logd("[{}] {} {}", stream->log_prefix(), status_code, - obsolete_reason(int_to_status(status_code))); + logd("[{}] {} {}", stream->log_prefix(), status_code, reason_phrase(status_code)); const auto status_code_str = std::to_string(status_code); const auto date = http_date_now(); @@ -280,10 +275,10 @@ void NGHttp2Writer::async_submit(StatusHandler&& handler, unsigned int sta for (auto&& item : headers) { - if (item.name_string().starts_with(':')) - logw("[{}] async_submit: invalid header '{}'", stream->log_prefix(), item.name_string()); + if (item.name.starts_with(':')) + logw("[{}] async_submit: invalid header '{}'", stream->log_prefix(), item.name); - nva.push_back(make_nv_ls(item.name_string(), item.value())); + nva.push_back(make_nv_ls(item.name, item.value)); } std::string length_str; diff --git a/src/h2v2/session.cpp b/src/h2v2/session.cpp index 32e647f..9c00bd8 100644 --- a/src/h2v2/session.cpp +++ b/src/h2v2/session.cpp @@ -12,10 +12,6 @@ #include -#include -#include -#include - #include #include @@ -29,8 +25,6 @@ #include #include -namespace http = boost::beast::http; - // ================================================================================================= namespace anyhttp::h2v2 @@ -128,7 +122,7 @@ static int on_recv_header(nghttp2_conn*, int64_t stream_id, int32_t /*token*/, n } if (!name.starts_with(':')) - stream->fields.insert(name, value); + stream->fields.append(name, value); } catch (std::exception& ex) { @@ -151,12 +145,11 @@ static int on_end_headers(nghttp2_conn* conn, int64_t /*stream_id*/, int /*fin*/ auto stream = static_cast(stream_user_data); assert(stream); - using namespace boost::beast::http; if (nghttp2_conn_is_server(conn)) logd("[{}] {} {}", stream->log_prefix(), stream->method, stream->url.buffer()); else if (stream->status_code) logd("[{}] {} {}", stream->log_prefix(), *stream->status_code, - obsolete_reason(int_to_status(*stream->status_code))); + reason_phrase(*stream->status_code)); stream->log_received_headers(); if (nghttp2_conn_is_server(conn)) @@ -167,7 +160,7 @@ static int on_end_headers(nghttp2_conn* conn, int64_t /*stream_id*/, int /*fin*/ // A response may carry an alternative service as a header field. Version 1 also accepted the // ALTSVC extension frame, which version 2 does not implement. // - if (auto alt_svc = stream->fields["alt-svc"]; !alt_svc.empty()) + if (auto alt_svc = stream->fields.value_or("alt-svc", ""); !alt_svc.empty()) handler->on_alt_svc(std::string_view(alt_svc)); stream->on_response(); @@ -450,11 +443,11 @@ void NGHttp2Session::async_submit(SubmitHandler&& handler, std::string_view meth for (auto&& item : headers) { - if (item.name_string().starts_with(':')) + if (item.name.starts_with(':')) logw("[{}] async_submit: invalid header '{}': setting pseudo headers is not allowed", - stream->log_prefix(), item.name_string()); + stream->log_prefix(), item.name); - nva.push_back(make_nv_ls(item.name_string(), item.value())); + nva.push_back(make_nv_ls(item.name, item.value)); } logd("[{}] {} {}", stream->log_prefix(), method_str, url.buffer()); @@ -639,7 +632,7 @@ void NGHttp2Session::close_stream(int64_t stream_id) if (stream->response_handler) { - auto ec = http::make_error_code(http::error::end_of_stream); // be compatible with beast + auto ec = errors::end_of_stream; // as HTTP/1.1 has it swap_and_invoke(stream->response_handler, ec, client::Response{nullptr}); } diff --git a/src/h2v2/stream.cpp b/src/h2v2/stream.cpp index 71dc20e..f8f3e73 100644 --- a/src/h2v2/stream.cpp +++ b/src/h2v2/stream.cpp @@ -9,9 +9,6 @@ #include -#include -#include - #include #include @@ -272,9 +269,7 @@ void NGHttp2Writer::async_submit(StatusHandler&& handler, unsigned int sta return; } - using namespace boost::beast::http; - logd("[{}] {} {}", stream->log_prefix(), status_code, - obsolete_reason(int_to_status(status_code))); + logd("[{}] {} {}", stream->log_prefix(), status_code, reason_phrase(status_code)); const auto status_code_str = std::to_string(status_code); const auto date = http_date_now(); @@ -294,10 +289,10 @@ void NGHttp2Writer::async_submit(StatusHandler&& handler, unsigned int sta for (auto&& item : headers) { - if (item.name_string().starts_with(':')) - logw("[{}] async_submit: invalid header '{}'", stream->log_prefix(), item.name_string()); + if (item.name.starts_with(':')) + logw("[{}] async_submit: invalid header '{}'", stream->log_prefix(), item.name); - nva.push_back(make_nv_ls(item.name_string(), item.value())); + nva.push_back(make_nv_ls(item.name, item.value)); } std::string length_str; diff --git a/src/h3/client.cpp b/src/h3/client.cpp index af0ccc4..5901ffe 100644 --- a/src/h3/client.cpp +++ b/src/h3/client.cpp @@ -20,9 +20,6 @@ #include "anyhttp/quic/session.hpp" #include "anyhttp/session_impl.hpp" -#include -#include - #include #include @@ -210,8 +207,7 @@ void Http3ClientStream::on_headers_complete() return; } - using namespace boost::beast::http; - mlogd("{} {}", status_code, obsolete_reason(int_to_status(status_code))); + mlogd("{} {}", status_code, reason_phrase(status_code)); log_headers(log_prefix_, std::exchange(received_headers, {})); deliver_response(); } @@ -223,8 +219,7 @@ void Http3ClientStream::on_failed(error_code ec) // ever arrived if headers were never received -- never report success with a null Response. // if (!failure_ec) // the first reason is the one to report, see on_headers_complete() - failure_ec = - ec ? ec : error_code(boost::system::error_code(boost::beast::http::error::end_of_stream)); + failure_ec = ec ? ec : errors::end_of_stream; deliver_failure(); } @@ -259,11 +254,11 @@ bool Http3ClientStream::submit_request(std::string_view method, const boost::url for (auto&& item : headers) { - if (item.name_string().starts_with(':')) + if (item.name.starts_with(':')) mlogw("async_submit: invalid header '{}': setting pseudo headers is not allowed", - item.name_string()); + item.name); - nva.push_back(make_nv(item.name_string(), item.value())); + nva.push_back(make_nv(item.name, item.value)); } mlogd("{} {}", method_str, request_url.buffer()); diff --git a/src/h3/server.cpp b/src/h3/server.cpp index 2cea4ed..d53cfb0 100644 --- a/src/h3/server.cpp +++ b/src/h3/server.cpp @@ -19,7 +19,6 @@ #include "anyhttp/request_handlers.hpp" // IWYU pragma: keep #include "anyhttp/server_impl.hpp" -#include #include #include @@ -211,16 +210,15 @@ void Http3ServerStream::submit_response(unsigned int status, const Fields& user_ for (auto&& item : response_fields) { - if (item.name_string().starts_with(':')) + if (item.name.starts_with(':')) { - mlogw("submit_response: dropping pseudo-header '{}'", item.name_string()); + mlogw("submit_response: dropping pseudo-header '{}'", item.name); continue; } - nva.push_back(make_nv(item.name_string(), item.value())); + nva.push_back(make_nv(item.name, item.value)); } - using namespace boost::beast::http; - mlogd("{} {}", response_status, obsolete_reason(int_to_status(response_status))); + mlogd("{} {}", response_status, reason_phrase(response_status)); if (submit_headers(nva, false /* response */)) session.wake_write(); diff --git a/src/h3/stream.cpp b/src/h3/stream.cpp index b926069..3b5bafc 100644 --- a/src/h3/stream.cpp +++ b/src/h3/stream.cpp @@ -9,7 +9,6 @@ #include "anyhttp/h3/common.hpp" #include "anyhttp/quic/session.hpp" -#include #include #include @@ -145,7 +144,7 @@ void Http3Stream::on_header(std::string_view name, std::string_view value) content_length = len; } - fields.insert(name, value); // insert, not set: repeated fields must all be kept + fields.append(name, value); // append, not set: repeated fields must all be kept } catch (const std::exception& ex) { diff --git a/src/issues/beast_issue_3032.cpp b/src/issues/beast_issue_3032.cpp deleted file mode 100644 index 70cff51..0000000 --- a/src/issues/beast_issue_3032.cpp +++ /dev/null @@ -1,84 +0,0 @@ -// -// https://github.com/boostorg/beast/issues/3032 -// https://godbolt.org/z/WaonxPoYr -// -#include -#include -#include - -#include - -using namespace std::string_view_literals; - -namespace asio = boost::asio; -namespace http = boost::beast::http; -using boost::system::error_code; - -/// Dummy stream satisfying the AsyncWriteStream requirements -struct StringWriteStream -{ - template - requires boost::beast::is_const_buffer_sequence::value - size_t write_some(const ConstBufferSequence& buffers, error_code&) - { - size_t written = 0; - for (auto it : buffers) - { - buffer += std::string_view(static_cast(it.data()), it.size()); - written += it.size(); - } - return written; - } - - template - requires boost::beast::is_const_buffer_sequence::value - size_t write_some(const ConstBufferSequence& buffers) - { - error_code ec; - auto written = write_some(buffers, ec); - if (ec) - BOOST_THROW_EXCEPTION(boost::system::system_error{ec}); - }; - - std::string buffer; -}; - -static_assert(boost::beast::is_sync_write_stream::value); - -static void test(bool with_nullptr) -{ - StringWriteStream stream; - - using Serializer = http::response_serializer; - using Message = std::remove_const_t; - Message message; - Serializer serializer{message}; - - message.chunked(true); - - error_code ec; - auto text = asio::buffer("Hello, World!"sv); - message.body().data = const_cast(text.data()); - message.body().size = text.size(); - message.body().more = true; - http::write(stream, serializer, ec); - assert(ec == http::error::need_buffer); - - if (with_nullptr) - message.body().data = nullptr; - message.body().size = 0; - message.body().more = false; - http::write(stream, serializer, ec); - assert(!ec); - - std::cout << stream.buffer; -} - -int main() -{ - std::cout << "--- GOOD -----------------\n"; - test(true); - std::cout << "--- BAD ------------------\n"; - test(false); - std::cout << "--------------------------\n"; -} diff --git a/src/request_handlers.cpp b/src/request_handlers.cpp index 36729da..3f5a1a8 100644 --- a/src/request_handlers.cpp +++ b/src/request_handlers.cpp @@ -76,7 +76,7 @@ Task dump(server::Request request, server::Response response) std::println(str, "headers:"); for (const auto& field : request.fields()) - std::println(str, " {}: {}", field.name_string(), EscapedString(field.value())); + std::println(str, " {}: {}", field.name, EscapedString(field.value)); auto body = str.str(); if (auto [ec] = co_await response.submit( diff --git a/src/server_impl.cpp b/src/server_impl.cpp index 2afa92e..da92311 100644 --- a/src/server_impl.cpp +++ b/src/server_impl.cpp @@ -7,8 +7,6 @@ #include "anyhttp/h3/backend.hpp" #include "anyhttp/quic/backend.hpp" -#include - #include #include @@ -218,7 +216,7 @@ Task Server::Impl::handle_connection(Executor executor, TcpSocket socket) // there is one line per connection, whatever the protocol. One that fails before that gets // the line telling why instead. // - auto buffer = boost::beast::flat_buffer(); + auto buffer = Buffer(); auto [ec, detected] = co_await detail::detect(socket, buffer); if (ec) { diff --git a/src/session.cpp b/src/session.cpp index 7aa3b37..7dcc895 100644 --- a/src/session.cpp +++ b/src/session.cpp @@ -66,10 +66,10 @@ namespace // Fields framed(std::string_view method, Fields headers) { - if ((method == "GET" || method == "HEAD") && - !headers.count(boost::beast::http::field::content_length) && - !headers.count(boost::beast::http::field::transfer_encoding)) - headers.set(boost::beast::http::field::content_length, "0"); + using boost::http::field; + if ((method == "GET" || method == "HEAD") && !headers.count(field::content_length) && + !headers.count(field::transfer_encoding)) + headers.set(field::content_length, "0"); return headers; } @@ -100,12 +100,12 @@ IoTask get_message(std::shared_ptr session, boos Fields headers) { // - // What an error leaves behind: an empty message that says status::unknown, rather than the 200 - // a default-constructed Beast response would claim. + // What an error leaves behind: an empty message with a status of 0, rather than the 200 a + // default-constructed response would claim. // auto failed = [](error_code ec) { client::Message message; - message.result(boost::beast::http::status::unknown); + message.set_start_line(0, ""); return std::tuple{ec, std::move(message)}; }; @@ -122,9 +122,10 @@ IoTask get_message(std::shared_ptr session, boos co_return failed(response_ec); client::Message message; - message.result(static_cast(response.status_code())); + const auto status_code = static_cast(response.status_code()); + message.set_start_line(status_code, reason_phrase(status_code)); for (auto&& field : response.fields()) - message.insert(field.name_string(), field.value()); // insert(), so repeated fields survive + message.append(field.name, field.value); // append(), so repeated fields survive auto& body = message.body(); std::array buffer; @@ -139,7 +140,7 @@ IoTask get_message(std::shared_ptr session, boos co_return failed(ec); } - logd("async_get: {} {}, {} bytes", message.result_int(), message.reason(), body.size()); + logd("async_get: {} {}, {} bytes", message.status_int(), message.reason(), body.size()); co_return std::tuple{error_code{}, std::move(message)}; } diff --git a/test/package/consumer.cpp b/test/package/consumer.cpp index 7280cd9..3e72cba 100644 --- a/test/package/consumer.cpp +++ b/test/package/consumer.cpp @@ -54,8 +54,8 @@ static Task get(client::Client& client, std::optional& ser auto [get_ec, message] = co_await session.get(url); if (get_ec) result = std::format("get: {}", get_ec.message()); - else if (message.result_int() != 200 || message.body() != body) - result = std::format("{}: '{}'", message.result_int(), message.body()); + else if (message.status_int() != 200 || message.body() != body) + result = std::format("{}: '{}'", message.status_int(), message.body()); else result = "ok"; } diff --git a/test/test_alt_svc.cpp b/test/test_alt_svc.cpp index 1a2caa6..d9b44b2 100644 --- a/test/test_alt_svc.cpp +++ b/test/test_alt_svc.cpp @@ -4,7 +4,7 @@ #include "anyhttp/alt_svc.hpp" -#include +#include #include @@ -13,7 +13,7 @@ using namespace testing; -namespace http = boost::beast::http; +namespace http = boost::http; // ================================================================================================= // Parsing the field value, which is the same for the header field and the HTTP/2 ALTSVC frame. @@ -128,7 +128,7 @@ constexpr auto http3_server = "anyhttp-quic/0.1"sv; static bool served_over_http3(const client::Message& message) { - return message[http::field::server] == http3_server; + return message.value_or(http::field::server, "") == http3_server; } class AltSvcUpgrade : public ClientAsync @@ -155,14 +155,14 @@ TEST_P(AltSvcUpgrade, WHEN_the_server_advertises_h3_THEN_the_next_connection_use { clientSession = [this](Session session) -> Task { auto first = check(co_await session.get(echo())); - EXPECT_EQ(first.result_int(), 200); + EXPECT_EQ(first.status_int(), 200); EXPECT_FALSE(served_over_http3(first)); - EXPECT_THAT(std::string(first[http::field::alt_svc]), HasSubstr("h3=")); + EXPECT_THAT(std::string(first.value_or(http::field::alt_svc, "")), HasSubstr("h3=")); auto second = check(co_await check(co_await client->connect()).get(echo())); - EXPECT_EQ(second.result_int(), 200); + EXPECT_EQ(second.status_int(), 200); EXPECT_TRUE(served_over_http3(second)) << "the second connection is not HTTP/3"; - EXPECT_EQ(second[http::field::alt_svc], "") + EXPECT_EQ(second.value_or(http::field::alt_svc, ""), "") << "HTTP/3 has no alternative to advertise, it is the alternative"; }; } @@ -188,11 +188,13 @@ TEST_P(AltSvcUpgrade, WHEN_the_server_clears_the_alternative_THEN_it_is_not_used }; clientSession = [this](Session session) -> Task { - EXPECT_THAT(std::string((check(co_await session.get(echo())))[http::field::alt_svc]), - HasSubstr("h3=")); + EXPECT_THAT( + std::string(check(co_await session.get(echo())).value_or(http::field::alt_svc, "")), + HasSubstr("h3=")); auto cleared = check(co_await session.get(url)); // "/custom", the handler above - EXPECT_EQ(cleared[http::field::alt_svc], "clear") << "the handler's field beats ours"; + EXPECT_EQ(cleared.value_or(http::field::alt_svc, ""), "clear") + << "the handler's field beats ours"; auto second = check(co_await check(co_await client->connect()).get(echo())); EXPECT_FALSE(served_over_http3(second)) << "the alternative should have been forgotten"; @@ -221,7 +223,7 @@ TEST_P(AltSvcWithEndpoints, WHEN_endpoints_are_given_THEN_the_alternative_is_not { clientSession = [this](Session session) -> Task { auto first = check(co_await session.get(echo())); - EXPECT_THAT(std::string(first[http::field::alt_svc]), HasSubstr("h3=")); + EXPECT_THAT(std::string(first.value_or(http::field::alt_svc, "")), HasSubstr("h3=")); auto second = check(co_await check(co_await client->connect()).get(echo())); EXPECT_FALSE(served_over_http3(second)); @@ -248,7 +250,7 @@ TEST_P(AltSvcIgnored, WHEN_the_client_does_not_follow_alt_svc_THEN_it_keeps_its_ target.set_path("/echo"); auto first = check(co_await session.get(target)); - EXPECT_THAT(std::string(first[http::field::alt_svc]), HasSubstr("h3=")); + EXPECT_THAT(std::string(first.value_or(http::field::alt_svc, "")), HasSubstr("h3=")); auto second = check(co_await check(co_await client->connect()).get(target)); EXPECT_FALSE(served_over_http3(second)); @@ -277,7 +279,7 @@ TEST_P(AltSvcDisabled, WHEN_the_server_advertises_nothing_THEN_the_client_stays_ target.set_path("/echo"); auto first = check(co_await session.get(target)); - EXPECT_EQ(first[http::field::alt_svc], ""); + EXPECT_EQ(first.value_or(http::field::alt_svc, ""), ""); auto second = check(co_await check(co_await client->connect()).get(target)); EXPECT_FALSE(served_over_http3(second)); @@ -402,11 +404,11 @@ TEST_P(AltSvcFrame, WHEN_an_altsvc_frame_arrives_THEN_the_next_connection_uses_i // from the frame that arrived before the request was even sent. // auto first = check(co_await check(co_await client.connect()).get(target)); - EXPECT_EQ(first.result_int(), 200); - EXPECT_EQ(first[http::field::alt_svc], ""); + EXPECT_EQ(first.status_int(), 200); + EXPECT_EQ(first.value_or(http::field::alt_svc, ""), ""); auto second = check(co_await check(co_await client.connect()).get(target)); - EXPECT_EQ(second.result_int(), 200); + EXPECT_EQ(second.status_int(), 200); EXPECT_TRUE(served_over_http3(second)) << "the second connection is not HTTP/3"; }; launch(client.get_executor(), client_main(), [&](const std::exception_ptr& ex) { diff --git a/test/test_asio.cpp b/test/test_asio.cpp index 747bf43..93ccd02 100644 --- a/test/test_asio.cpp +++ b/test/test_asio.cpp @@ -153,7 +153,7 @@ TEST_P(AsioTokens, WHEN_connect_and_get_take_cancel_after_THEN_results_pass_thro auto [get_ec, message] = co_await session.async_get(url.set_path("missing"), {}, cancel_after(never, as_tuple)); EXPECT_FALSE(get_ec) << get_ec.message(); - EXPECT_EQ(message.result_int(), 404); + EXPECT_EQ(message.status_int(), 404); }; } @@ -234,7 +234,7 @@ TEST_P(AsioTokens, WHEN_server_gives_a_callback_THEN_it_is_not_called_before_the clientSession = [this](Session session) -> Task { auto [ec, message] = co_await session.async_get(url, {}, as_tuple); EXPECT_FALSE(ec) << ec.message(); - EXPECT_EQ(message.result_int(), 200); + EXPECT_EQ(message.status_int(), 200); }; } diff --git a/test/test_client_async.cpp b/test/test_client_async.cpp index 9edc350..031f065 100644 --- a/test/test_client_async.cpp +++ b/test/test_client_async.cpp @@ -17,9 +17,8 @@ using namespace testing; INSTANTIATE_TEST_SUITE_P(ClientAsync, ClientAsync, ValuesIn(protocols()), NameGenerator); -/// Beast's end_of_stream, as the runtime's error code. -static const error_code end_of_stream = - boost::system::error_code(boost::beast::http::error::end_of_stream); +/// A connection that ended before a response began. +static const error_code end_of_stream = errors::end_of_stream; /** * Cancels \p task after \p timeout, the way each runtime does it: ASIO with cancel_after() on the @@ -265,7 +264,7 @@ TEST_P(ClientAsync, WHEN_server_session_is_gone_THEN_request_head_is_still_there EXPECT_EQ(request.method(), method); EXPECT_EQ(request.url().buffer(), url); EXPECT_EQ(request.url().path(), "/custom/head"); - EXPECT_EQ(request.fields()["X-Test"], "42"); + EXPECT_EQ(request.fields().value_or("X-Test", ""), "42"); inspected->notify(); }; @@ -619,7 +618,7 @@ TEST_P(ClientAsync, HelloWorld) }; clientSession = [this](Session session) -> Task { auto message = check(co_await session.get(url)); - EXPECT_EQ(message.result_int(), 200); + EXPECT_EQ(message.status_int(), 200); EXPECT_EQ(message.body(), hello); }; } @@ -776,7 +775,7 @@ TEST_P(ClientAsync, ServerYieldFirst) check(co_await response.write_eof()); }; clientSession = [this](Session session) -> Task { - EXPECT_EQ((check(co_await session.get(url))).result_int(), 200); + EXPECT_EQ((check(co_await session.get(url))).status_int(), 200); }; } diff --git a/test/test_client_connect.cpp b/test/test_client_connect.cpp index 7b807e9..01ca98a 100644 --- a/test/test_client_connect.cpp +++ b/test/test_client_connect.cpp @@ -367,7 +367,7 @@ class ClientEndpoints : public ClientAsync { EXPECT_EQ(session.protocol(), GetParam()); auto message = check(co_await session.get(url)); - EXPECT_EQ(message.result_int(), 200); + EXPECT_EQ(message.status_int(), 200); EXPECT_EQ(message.body(), url.encoded_host_and_port()); } }; diff --git a/test/test_client_tls.cpp b/test/test_client_tls.cpp index 807e24f..a7a94df 100644 --- a/test/test_client_tls.cpp +++ b/test/test_client_tls.cpp @@ -222,7 +222,7 @@ TEST_F(ClientTlsPeer, WHEN_framed_body_has_arrived_THEN_missing_close_notify_doe TEST_F(ClientTlsPeer, WHEN_connection_is_cut_before_the_response_THEN_that_is_the_end_of_stream) { get({.response = "", .close_notify = false}); - EXPECT_EQ(result, error_code{make_error_code(boost::beast::http::error::end_of_stream)}); + EXPECT_EQ(result, errors::end_of_stream); } // ================================================================================================= diff --git a/test/test_connection_close.cpp b/test/test_connection_close.cpp index 12182bd..51807b0 100644 --- a/test/test_connection_close.cpp +++ b/test/test_connection_close.cpp @@ -3,8 +3,9 @@ #include "anyhttp/h1/io.hpp" #include "anyhttp/net.hpp" -#include -#include +#include +#include +#include #if ANYHTTP_ASIO #include @@ -19,7 +20,7 @@ using namespace testing; -namespace http = boost::beast::http; +namespace http = boost::http; namespace h1 = anyhttp::h1; // ================================================================================================= @@ -35,8 +36,15 @@ namespace h1 = anyhttp::h1; class ConnectionClose : public Server { protected: - using Request = http::request; - using Response = http::response; + /// A request with a body, which a header alone has no place for. + struct Request : http::request + { + using http::request::request; + std::string body; + }; + + /// The response, header and body, as a client's get() has it. + using Response = client::Message; Task connect() { @@ -50,14 +58,35 @@ class ConnectionClose : public Server Task exchange(Stream& socket, Request request) { request.set(http::field::host, std::format("127.0.0.2:{}", port())); - request.prepare_payload(); - check(co_await h1::write_message(socket, request)); + if (!request.body.empty() || request.method() != http::method::get) + request.set_content_length(request.body.size()); + check(co_await h1::write_message(socket, serializer_, request, request.body)); + co_return co_await read_response(socket); + } - Response response; - check(co_await h1::read_message(socket, buffer_, response)); + /// Reads a response, which must arrive whole. + template + Task read_response(Stream& socket) + { + auto [ec, response] = co_await try_read_response(socket); + if (ec) + throw_error(ec); co_return response; } + /// Reads a response, as much of it as there is. + template + Task> try_read_response(Stream& socket) + { + parser_.start(); + auto [ec, body] = co_await h1::read_message(socket, buffer_, parser_); + Response response; + if (parser_.got_header()) + response = Response(parser_.get()); + response.body() = std::move(body); + co_return std::tuple{ec, std::move(response)}; + } + /** * Reads what follows the last response, which must be the end of the stream and nothing else. * Gives up after a while: a server that does not end the connection must fail a test, not hang @@ -66,7 +95,7 @@ class ConnectionClose : public Server template Task read_eof(Stream& socket) { - EXPECT_EQ(buffer_.size(), 0) << "unread data left over from the response"; + EXPECT_FALSE(parser_.has_buffered_data()) << "unread data left over from the response"; std::array buffer; Timer timer(context.get_executor()); @@ -97,7 +126,13 @@ class ConnectionClose : public Server Server::run(); } - boost::beast::flat_buffer buffer_; + Buffer buffer_; // input to go to the parser before the socket: none, everything is read by it + http::serializer serializer_{h1::make_serializer_config()}; + http::response_parser parser_ = [] { + http::response_parser parser(h1::make_parser_config(false, default_max_header_size)); + parser.reset(); + return parser; + }(); }; // ------------------------------------------------------------------------------------------------- @@ -107,11 +142,11 @@ TEST_F(ConnectionClose, WHEN_request_asks_to_close_THEN_response_says_so_and_str run([&]() -> Task { auto socket = co_await connect(); - Request request{http::verb::get, "/dump", 11}; + Request request(http::method::get, "/dump"); request.set(http::field::connection, "close"); auto response = co_await exchange(socket, std::move(request)); - EXPECT_EQ(response.result_int(), 200); + EXPECT_EQ(response.status_int(), 200); EXPECT_FALSE(response.keep_alive()) << "the last response has to announce itself as one"; EXPECT_EQ(co_await read_eof(socket), errors::eof); }()); @@ -122,12 +157,12 @@ TEST_F(ConnectionClose, WHEN_request_with_body_asks_to_close_THEN_body_is_served run([&]() -> Task { auto socket = co_await connect(); - Request request{http::verb::post, "/echo", 11}; - request.body() = "hello"; + Request request(http::method::post, "/echo"); + request.body = "hello"; request.set(http::field::connection, "close"); auto response = co_await exchange(socket, std::move(request)); - EXPECT_EQ(response.result_int(), 200); + EXPECT_EQ(response.status_int(), 200); EXPECT_EQ(response.body(), "hello"); EXPECT_FALSE(response.keep_alive()); EXPECT_EQ(co_await read_eof(socket), errors::eof); @@ -139,13 +174,13 @@ TEST_F(ConnectionClose, WHEN_request_does_not_ask_to_close_THEN_connection_takes run([&]() -> Task { auto socket = co_await connect(); - auto first = co_await exchange(socket, Request{http::verb::get, "/dump?first", 11}); - EXPECT_EQ(first.result_int(), 200); + auto first = co_await exchange(socket, Request(http::method::get, "/dump?first")); + EXPECT_EQ(first.status_int(), 200); EXPECT_TRUE(first.keep_alive()); EXPECT_THAT(first.body(), HasSubstr("query: first")); - auto second = co_await exchange(socket, Request{http::verb::get, "/dump?second", 11}); - EXPECT_EQ(second.result_int(), 200); + auto second = co_await exchange(socket, Request(http::method::get, "/dump?second")); + EXPECT_EQ(second.status_int(), 200); EXPECT_TRUE(second.keep_alive()); EXPECT_THAT(second.body(), HasSubstr("query: second")); }()); @@ -160,9 +195,10 @@ TEST_F(ConnectionClose, WHEN_request_is_http_1_0_THEN_stream_ends_after_the_resp // HTTP/1.0 has no persistent connections unless the client asks for one, so this response // is the last one even though nothing said "close". // - auto response = co_await exchange(socket, Request{http::verb::get, "/dump", 10}); + auto response = + co_await exchange(socket, Request(http::method::get, "/dump", http::version::http_1_0)); - EXPECT_EQ(response.result_int(), 200); + EXPECT_EQ(response.status_int(), 200); EXPECT_FALSE(response.keep_alive()); EXPECT_EQ(co_await read_eof(socket), errors::eof); }()); @@ -186,9 +222,8 @@ TEST_F(ConnectionClose, WHEN_requests_follow_one_that_asks_to_close_THEN_they_ar check(co_await io::write(socket, boost::asio::buffer(raw_get("Connection: close\r\n") + raw_get()))); - Response response; - check(co_await h1::read_message(socket, buffer_, response)); - EXPECT_EQ(response.result_int(), 200); + auto response = co_await read_response(socket); + EXPECT_EQ(response.status_int(), 200); EXPECT_FALSE(response.keep_alive()); EXPECT_EQ(co_await read_eof(socket), errors::eof); EXPECT_EQ(served, 1); @@ -213,9 +248,8 @@ TEST_F(ConnectionClose, WHEN_handler_says_close_THEN_stream_ends_and_the_rest_is auto socket = co_await connect(); check(co_await io::write(socket, boost::asio::buffer(raw_get() + raw_get()))); - Response response; - check(co_await h1::read_message(socket, buffer_, response)); - EXPECT_EQ(response.result_int(), 200); + auto response = co_await read_response(socket); + EXPECT_EQ(response.status_int(), 200); EXPECT_FALSE(response.keep_alive()); EXPECT_EQ(co_await read_eof(socket), errors::eof); EXPECT_EQ(served, 1); @@ -315,13 +349,14 @@ TEST_F(RejectedRequest, WHEN_request_is_rejected_THEN_the_response_arrives_anywa co_return ec; }; auto receive = [&]() -> Task { - auto [ec, n] = co_await h1::read_message(socket, buffer_, response); + auto [ec, received] = co_await try_read_response(socket); + response = std::move(received); co_return ec; }; auto [send_ec, receive_ec] = co_await when_both(send(), receive()); EXPECT_FALSE(receive_ec) << "the 431 was lost: " << receive_ec.message(); - EXPECT_EQ(response.result_int(), 431); + EXPECT_EQ(response.status_int(), 431); EXPECT_FALSE(response.keep_alive()); }()); } @@ -381,11 +416,11 @@ TEST_F(TlsConnectionClose, WHEN_request_asks_to_close_THEN_close_notify_comes_be run([&]() -> Task { auto stream = co_await connect_tls(); - Request request{http::verb::get, "/dump", 11}; + Request request(http::method::get, "/dump"); request.set(http::field::connection, "close"); auto response = co_await exchange(stream, std::move(request)); - EXPECT_EQ(response.result_int(), 200); + EXPECT_EQ(response.status_int(), 200); EXPECT_FALSE(response.keep_alive()); // a clean end of stream, not ssl::error::stream_truncated diff --git a/test/test_connection_end.cpp b/test/test_connection_end.cpp index 4154c2c..7a16c10 100644 --- a/test/test_connection_end.cpp +++ b/test/test_connection_end.cpp @@ -3,8 +3,9 @@ #include "anyhttp/h1/io.hpp" #include "anyhttp/net.hpp" -#include -#include +#include +#include +#include #if ANYHTTP_ASIO #include @@ -25,7 +26,7 @@ using namespace testing; -namespace http = boost::beast::http; +namespace http = boost::http; namespace h1 = anyhttp::h1; // ================================================================================================= @@ -228,15 +229,15 @@ class ConnectionEndBase : public Server } else { - http::request request{http::verb::get, "/dump", 11}; + http::request request(http::method::get, "/dump"); request.set(http::field::host, authority()); request.set(http::field::connection, connection); - check(co_await h1::write_message(stream, request)); + check(co_await h1::write_message(stream, serializer_, request)); - http::response response; - check(co_await h1::read_message(stream, buffer_, response)); - EXPECT_EQ(response.result_int(), 200); - EXPECT_EQ(response.keep_alive(), connection == "keep-alive"); + parser_.start(); + check(co_await h1::read_message(stream, buffer_, parser_)); + EXPECT_EQ(parser_.get().status_int(), 200); + EXPECT_EQ(parser_.get().keep_alive(), connection == "keep-alive"); } } @@ -269,7 +270,7 @@ class ConnectionEndBase : public Server */ Task read_end(Stream& stream) { - EXPECT_EQ(buffer_.size(), 0) << "unread data left over from the response"; + EXPECT_FALSE(parser_.has_buffered_data()) << "unread data left over from the response"; std::array data; for (;;) @@ -436,7 +437,13 @@ class ConnectionEndBase : public Server } H2Client h2_; - boost::beast::flat_buffer buffer_; + Buffer buffer_; // input to go to the parser before the stream: none, everything is read by it + http::serializer serializer_{h1::make_serializer_config()}; + http::response_parser parser_ = [] { + http::response_parser parser(h1::make_parser_config(false, default_max_header_size)); + parser.reset(); + return parser; + }(); }; // ================================================================================================= diff --git a/test/test_corosio.cpp b/test/test_corosio.cpp index 370bc40..72ed69e 100644 --- a/test/test_corosio.cpp +++ b/test/test_corosio.cpp @@ -7,17 +7,19 @@ #if ANYHTTP_COROSIO -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include #include #include +#include +#include #include #include #include @@ -28,6 +30,7 @@ #include #include +#include "anyhttp/h1/io.hpp" #include "anyhttp/runtime.hpp" #include @@ -40,150 +43,113 @@ using namespace testing; namespace capy = boost::capy; -namespace http = boost::beast::http; - -// ================================================================================================= - -namespace -{ - -// -// Reads from `socket` into `parser` until it has a complete message. put() consumes only part of -// what it is given -- the header first, and the body in a call of its own -- so everything it -// leaves has to be offered again. -// -template -capy::task read_message(boost::corosio::tcp_socket& socket, Parser& parser) -{ - std::array buffer; - while (!parser.is_done()) - { - auto [ec, n] = co_await socket.read_some(capy::mutable_buffer(buffer.data(), buffer.size())); - if (ec) - co_return ec; - - for (const char* data = buffer.data(); n > 0 && !parser.is_done();) - { - boost::system::error_code bec; - const auto used = parser.put(boost::asio::buffer(data, n), bec); - if (bec == http::error::need_more) - break; - if (bec) - co_return bec; - data += used, n -= used; - } - } - co_return std::error_code{}; -} - -// -// Writes what `serializer` produces to `socket`. The buffers next() hands out are valid until -// consume(), so their descriptors are copied out of the visitor and consumed after the write. -// -template -capy::task write_message(boost::corosio::tcp_socket& socket, - Serializer& serializer) -{ - while (!serializer.is_done()) - { - boost::system::error_code bec; - std::vector buffers; - serializer.next(bec, [&](boost::system::error_code&, const auto& sequence) { - for (auto buffer : boost::beast::buffers_range_ref(sequence)) - buffers.emplace_back(buffer.data(), buffer.size()); - }); - if (bec) - co_return bec; - - auto [ec, n] = co_await capy::write(socket, buffers); - if (ec) - co_return ec; - serializer.consume(n); - } - co_return std::error_code{}; -} - -} // namespace +namespace http = boost::http; // ================================================================================================= // -// HTTP/1.1 without Beast's asynchronous operations, which need an ASIO stream: its parser and -// serializer, driven by read and write loops on corosio sockets. A chunked request and a response -// with a Content-Length, so that both framings go through both halves. +// Boost.HTTP's own loops around its parser and serializer -- parser::read() and serializer::sink +// -- are written for capy's streams, so they run on corosio sockets as they are. anyhttp has loops +// of its own (h1/io.hpp), to run on either runtime, but these say what Boost.HTTP takes for +// granted of a stream. A chunked request and a response with a Content-Length, so that both +// framings go through both halves. // -TEST(Corosio, BeastParserAndSerializer) +TEST(Corosio, HttpParserAndSerializer) { boost::corosio::io_context context; auto [ec, server, client] = boost::corosio::test::make_socket_pair(context); ASSERT_FALSE(ec); std::error_code server_ec, client_ec; - http::request_parser request_parser; - http::response_parser response_parser; - - auto serve = [](boost::corosio::tcp_socket& socket, auto& parser, - std::error_code& result) -> capy::task<> { - if ((result = co_await read_message(socket, parser))) + http::request_parser request_parser(anyhttp::h1::make_parser_config(true, 64 * 1024)); + http::response_parser response_parser(anyhttp::h1::make_parser_config(false, 64 * 1024)); + http::serializer server_serializer(anyhttp::h1::make_serializer_config()); + http::serializer client_serializer(anyhttp::h1::make_serializer_config()); + std::string request_body, response_body; + bool chunked = false; + + auto serve = [&](boost::corosio::tcp_socket& socket) -> capy::task<> { + request_parser.reset(); + request_parser.start(); + if (auto [read_ec] = co_await request_parser.read(socket); read_ec) + { + server_ec = read_ec; co_return; - - http::response response{http::status::ok, 11, parser.get().body()}; - response.prepare_payload(); - http::response_serializer serializer{response}; - result = co_await write_message(socket, serializer); + } + chunked = request_parser.get().metadata().transfer_encoding.is_chunked; + request_body = request_parser.body(); + + http::response response(http::status::ok); + response.set_payload_size(request_body.size()); + server_serializer.set_message(response); + auto sink = server_serializer.sink_for(socket); + auto [write_ec, n] = co_await sink.write_eof(capy::make_buffer(request_body)); + server_ec = write_ec; }; - auto request = [](boost::corosio::tcp_socket& socket, auto& parser, - std::error_code& result) -> capy::task<> { - http::request request{http::verb::post, "/echo", 11, "hello, corosio"}; - request.chunked(true); - http::request_serializer serializer{request}; - if ((result = co_await write_message(socket, serializer))) + auto request = [&](boost::corosio::tcp_socket& socket) -> capy::task<> { + http::request request(http::method::post, "/echo"); + request.set_chunked(true); + client_serializer.set_message(request); + auto sink = client_serializer.sink_for(socket); + if (auto [write_ec, n] = + co_await sink.write_eof(capy::make_buffer(std::string_view("hello, corosio"))); + write_ec) + { + client_ec = write_ec; co_return; + } - result = co_await read_message(socket, parser); + response_parser.reset(); + response_parser.start(); + auto [read_ec] = co_await response_parser.read(socket); + client_ec = read_ec; + if (!read_ec) + response_body = response_parser.body(); }; - capy::run_async(context.get_executor())(serve(server, request_parser, server_ec)); - capy::run_async(context.get_executor())(request(client, response_parser, client_ec)); + capy::run_async(context.get_executor())(serve(server)); + capy::run_async(context.get_executor())(request(client)); context.run(); EXPECT_FALSE(server_ec) << server_ec.message(); EXPECT_FALSE(client_ec) << client_ec.message(); - EXPECT_TRUE(request_parser.chunked()); + EXPECT_TRUE(chunked); EXPECT_EQ(request_parser.get().target(), "/echo"); - EXPECT_EQ(response_parser.get().result_int(), 200); - EXPECT_EQ(response_parser.get().body(), "hello, corosio"); + EXPECT_EQ(request_body, "hello, corosio"); + EXPECT_EQ(response_parser.get().status_int(), 200); + EXPECT_EQ(response_body, "hello, corosio"); } // ------------------------------------------------------------------------------------------------- // -// Why each I/O runtime needs error constants of its own: a Boost.System code survives the -// conversion to std::error_code -- Beast's errors stay comparable across it -- but asio's eof does -// not become capy's. What docs/end-of-body.rst calls errors::eof at the end of a body is -// capy::error::eof in COROSIO. +// Why each I/O runtime needs error constants of its own: Boost.HTTP's errors are std::error_code, +// the same in both runtimes, but asio's eof does not become capy's. What docs/end-of-body.rst +// calls errors::eof at the end of a body is capy::error::eof in COROSIO. // TEST(Corosio, ErrorCodeInterop) { - const std::error_code partial = boost::system::error_code{http::error::partial_message}; - EXPECT_EQ(partial, std::error_code(boost::system::error_code{http::error::partial_message})); - EXPECT_EQ(partial.message(), "partial message"); + const std::error_code incomplete = http::error::incomplete; + EXPECT_EQ(incomplete, std::error_code(boost::system::error_code(incomplete))); const std::error_code eof = boost::system::error_code{boost::asio::error::eof}; EXPECT_NE(eof, capy::cond::eof); EXPECT_EQ(std::error_code{capy::error::eof}, capy::cond::eof); // - // What the parser itself reports: Beast attaches the source location of the error. + // What the parser itself reports, and what anyhttp makes of it. // - http::request_parser parser; - parser.header_limit(16); - boost::system::error_code limit; + http::request_parser parser(anyhttp::h1::make_parser_config(true, 32)); + parser.reset(); + parser.start(); std::string_view request = "GET / HTTP/1.1\r\nX-Field: a value well over sixteen bytes\r\n"; - parser.put(boost::asio::buffer(request), limit); - ASSERT_EQ(limit, http::error::header_limit) << limit.message(); - EXPECT_EQ(std::error_code(limit), anyhttp::errors::header_limit) << limit.what(); + auto buffers = parser.prepare(); + const auto n = capy::buffer_copy(buffers, capy::make_buffer(request)); + parser.commit(n); + std::error_code limit; + parser.parse(limit); + EXPECT_EQ(anyhttp::h1::to_error_code(limit), anyhttp::errors::header_limit) << limit.message(); } // ================================================================================================= diff --git a/test/test_coroutine_api.cpp b/test/test_coroutine_api.cpp index d5bb93d..18b0754 100644 --- a/test/test_coroutine_api.cpp +++ b/test/test_coroutine_api.cpp @@ -4,7 +4,7 @@ using namespace testing; -namespace http = boost::beast::http; +namespace http = boost::http; // ================================================================================================= @@ -144,7 +144,7 @@ TEST_P(CoroutineApi, WHEN_connecting_and_getting_THEN_message_arrives) auto [get_ec, message] = co_await session.get(url); EXPECT_FALSE(get_ec) << get_ec.message(); - EXPECT_EQ(message.result(), http::status::ok); + EXPECT_EQ(message.status(), http::status::ok); EXPECT_EQ(message.body(), "ok"); }; } diff --git a/test/test_docs.cpp b/test/test_docs.cpp index e3a4ec0..3d65e7b 100644 --- a/test/test_docs.cpp +++ b/test/test_docs.cpp @@ -38,7 +38,7 @@ TEST_P(Guide, Route) auto [missing_ec, missing] = co_await session.get(url.set_path("/missing")); EXPECT_FALSE(missing_ec) << missing_ec.message(); - EXPECT_EQ(missing.result_int(), 404); + EXPECT_EQ(missing.status_int(), 404); }; } diff --git a/test/test_external.cpp b/test/test_external.cpp index 0f0e197..5fecf76 100644 --- a/test/test_external.cpp +++ b/test/test_external.cpp @@ -463,7 +463,7 @@ TEST_F(ExternalCustom, nghttp2_trailers) requestHandler = [](server::Request request, server::Response response) -> Task { const auto received = co_await drain(request); auto body = std::format("{} bytes, x-trailer: {}", received, - std::string_view(request.fields()["x-trailer"])); + std::string_view(request.fields().value_or("x-trailer", ""))); check(co_await response.submit(200, fields({{"Content-Length", body.size()}}))); check(co_await response.write_eof(boost::asio::buffer(body))); }; diff --git a/test/test_file_handler.cpp b/test/test_file_handler.cpp index 5f900cd..b61a885 100644 --- a/test/test_file_handler.cpp +++ b/test/test_file_handler.cpp @@ -88,7 +88,7 @@ TEST_P(FileHandler, WHEN_file_exists_THEN_serves_content) { clientSession = [this](Session session) -> Task { auto message = co_await get(session, "/custom/hello.txt"); - EXPECT_EQ(message.result_int(), 200); + EXPECT_EQ(message.status_int(), 200); EXPECT_EQ(message.body(), "Hello, File!"); }; } @@ -97,7 +97,7 @@ TEST_P(FileHandler, WHEN_file_is_in_subdirectory_THEN_serves_content) { clientSession = [this](Session session) -> Task { auto message = co_await get(session, "/custom/sub/nested.txt"); - EXPECT_EQ(message.result_int(), 200); + EXPECT_EQ(message.status_int(), 200); EXPECT_EQ(message.body(), "Nested!"); }; } @@ -110,7 +110,7 @@ TEST_P(FileHandler, WHEN_file_is_empty_THEN_serves_empty_body) { clientSession = [this](Session session) -> Task { auto message = co_await get(session, "/custom/empty.txt"); - EXPECT_EQ(message.result_int(), 200); + EXPECT_EQ(message.status_int(), 200); EXPECT_THAT(message.body(), IsEmpty()); }; } @@ -123,7 +123,7 @@ TEST_P(FileHandler, WHEN_file_is_large_THEN_serves_all_of_it) { clientSession = [this](Session session) -> Task { auto message = co_await get(session, "/custom/large.bin"); - EXPECT_EQ(message.result_int(), 200); + EXPECT_EQ(message.status_int(), 200); EXPECT_EQ(message.body(), std::string(256_k, 'x')); }; } @@ -132,7 +132,7 @@ TEST_P(FileHandler, WHEN_file_does_not_exist_THEN_error_404) { clientSession = [this](Session session) -> Task { auto message = co_await get(session, "/custom/missing.txt"); - EXPECT_EQ(message.result_int(), 404); + EXPECT_EQ(message.status_int(), 404); EXPECT_THAT(message.body(), IsEmpty()); }; } @@ -143,17 +143,17 @@ TEST_P(FileHandler, WHEN_file_does_not_exist_THEN_error_404) TEST_P(FileHandler, WHEN_path_is_a_directory_THEN_error_404) { clientSession = [this](Session session) -> Task { - EXPECT_EQ((co_await get(session, "/custom/sub")).result_int(), 404); - EXPECT_EQ((co_await get(session, "/custom/")).result_int(), 404); + EXPECT_EQ((co_await get(session, "/custom/sub")).status_int(), 404); + EXPECT_EQ((co_await get(session, "/custom/")).status_int(), 404); }; } TEST_P(FileHandler, WHEN_path_escapes_the_root_THEN_error_404) { clientSession = [this](Session session) -> Task { - EXPECT_EQ((co_await get(session, "/custom/../outside.txt")).result_int(), 404); - EXPECT_EQ((co_await get(session, "/custom/sub/../../outside.txt")).result_int(), 404); - EXPECT_EQ((co_await get(session, encoded("/custom/%2e%2e/outside.txt"))).result_int(), 404); + EXPECT_EQ((co_await get(session, "/custom/../outside.txt")).status_int(), 404); + EXPECT_EQ((co_await get(session, "/custom/sub/../../outside.txt")).status_int(), 404); + EXPECT_EQ((co_await get(session, encoded("/custom/%2e%2e/outside.txt"))).status_int(), 404); }; } @@ -163,7 +163,7 @@ TEST_P(FileHandler, WHEN_path_escapes_the_root_THEN_error_404) TEST_P(FileHandler, WHEN_symlink_points_outside_the_root_THEN_error_404) { clientSession = [this](Session session) -> Task { - EXPECT_EQ((co_await get(session, "/custom/escape.txt")).result_int(), 404); + EXPECT_EQ((co_await get(session, "/custom/escape.txt")).status_int(), 404); }; } @@ -175,9 +175,9 @@ TEST_P(FileHandler, WHEN_prefix_matches_mid_segment_THEN_error_404) { clientSession = [this](Session session) -> Task { auto message = co_await get(session, "/customer.txt"); - EXPECT_EQ(message.result_int(), 404); + EXPECT_EQ(message.status_int(), 404); EXPECT_THAT(message.body(), IsEmpty()); - EXPECT_EQ((co_await get(session, "/customer/hello.txt")).result_int(), 404); + EXPECT_EQ((co_await get(session, "/customer/hello.txt")).status_int(), 404); }; } @@ -188,7 +188,7 @@ TEST_P(FileHandler, WHEN_file_is_not_readable_THEN_error_403) clientSession = [this](Session session) -> Task { auto message = co_await get(session, "/custom/secret.txt"); - EXPECT_EQ(message.result_int(), 403); + EXPECT_EQ(message.status_int(), 403); EXPECT_THAT(message.body(), IsEmpty()); }; } diff --git a/test/test_fixtures.hpp b/test/test_fixtures.hpp index 701fdb3..cba1954 100644 --- a/test/test_fixtures.hpp +++ b/test/test_fixtures.hpp @@ -11,7 +11,6 @@ #include "anyhttp/session.hpp" #include "anyhttp/utils.hpp" -#include #include #include diff --git a/test/test_fixtures_asio.hpp b/test/test_fixtures_asio.hpp index a97fb0a..f090a2e 100644 --- a/test/test_fixtures_asio.hpp +++ b/test/test_fixtures_asio.hpp @@ -27,9 +27,6 @@ #include #include -#include -#include - #include #include #include diff --git a/test/test_formatter.cpp b/test/test_formatter.cpp index e79ac57..93485d4 100644 --- a/test/test_formatter.cpp +++ b/test/test_formatter.cpp @@ -14,8 +14,8 @@ namespace h2 = anyhttp::h2v2; namespace h2 = anyhttp::h2; #endif -#include #include +#include #include #include @@ -115,26 +115,26 @@ TEST(FormatterTest, EndpointIPv6Full) } // ================================================================================================= -// Test boost::beast::http::field formatter +// Test boost::http::field formatter // ================================================================================================= TEST(FormatterTest, HttpField) { - auto field = boost::beast::http::field::content_type; + auto field = boost::http::field::content_type; auto formatted = std::format("{}", field); EXPECT_EQ(formatted, "Content-Type"); } TEST(FormatterTest, HttpFieldAccept) { - auto field = boost::beast::http::field::accept; + auto field = boost::http::field::accept; auto formatted = std::format("{}", field); EXPECT_EQ(formatted, "Accept"); } TEST(FormatterTest, HttpFieldUserAgent) { - auto field = boost::beast::http::field::user_agent; + auto field = boost::http::field::user_agent; auto formatted = std::format("{}", field); EXPECT_EQ(formatted, "User-Agent"); } diff --git a/test/test_get.cpp b/test/test_get.cpp index fe424ac..ef16630 100644 --- a/test/test_get.cpp +++ b/test/test_get.cpp @@ -15,13 +15,13 @@ using namespace testing; -namespace http = boost::beast::http; +namespace http = boost::http; // ================================================================================================= // // Session::async_get(): a whole GET request as a single operation, with the response handed back -// as a plain Beast message -- status, fields and the body as a string. +// as a client::Message -- status, fields and the body as a string. // class AsyncGet : public ClientAsync { @@ -59,9 +59,9 @@ TEST_P(AsyncGet, WHEN_get_THEN_message_has_status_fields_and_body) respond_with("Hello, World!"); clientSession = [this](Session session) -> Task { auto message = check(co_await session.get(url)); - EXPECT_EQ(message.result(), http::status::ok); - EXPECT_EQ(message.result_int(), 200); - EXPECT_EQ(message["x-answer"], "42"); + EXPECT_EQ(message.status(), http::status::ok); + EXPECT_EQ(message.status_int(), 200); + EXPECT_EQ(message.value_or("x-answer", ""), "42"); EXPECT_EQ(message.body(), "Hello, World!"); }; } @@ -101,7 +101,7 @@ TEST_P(AsyncGet, WHEN_response_has_no_body_THEN_body_is_empty) respond_with(""); clientSession = [this](Session session) -> Task { auto message = check(co_await session.get(url)); - EXPECT_EQ(message.result_int(), 200); + EXPECT_EQ(message.status_int(), 200); EXPECT_THAT(message.body(), IsEmpty()); }; } @@ -114,7 +114,7 @@ TEST_P(AsyncGet, WHEN_path_is_unknown_THEN_message_says_404) { clientSession = [this](Session session) -> Task { auto message = check(co_await session.get(url.set_path("unknown"))); - EXPECT_EQ(message.result(), http::status::not_found); + EXPECT_EQ(message.status(), http::status::not_found); }; } @@ -124,7 +124,7 @@ TEST_P(AsyncGet, WHEN_body_is_large_THEN_all_of_it_arrives) respond_with(body); clientSession = [this, body](Session session) -> Task { auto message = check(co_await session.get(url)); - EXPECT_EQ(message.result_int(), 200); + EXPECT_EQ(message.status_int(), 200); EXPECT_EQ(message.body().size(), body.size()); EXPECT_EQ(message.body(), body); }; @@ -133,15 +133,15 @@ TEST_P(AsyncGet, WHEN_body_is_large_THEN_all_of_it_arrives) TEST_P(AsyncGet, WHEN_headers_are_given_THEN_they_arrive_with_the_request) { requestHandler = [](server::Request request, server::Response response) -> Task { - EXPECT_EQ(request.fields()["x-question"], "what?"); - EXPECT_EQ(request.fields()["content-length"], "0"); + EXPECT_EQ(request.fields().value_or("x-question", ""), "what?"); + EXPECT_EQ(request.fields().value_or("content-length", ""), "0"); co_await drain(request); check(co_await response.submit(200, fields({{"Content-Length", 0}}))); check(co_await response.write_eof()); }; clientSession = [this](Session session) -> Task { auto message = check(co_await session.get(url, fields({{"X-Question", "what?"}}))); - EXPECT_EQ(message.result_int(), 200); + EXPECT_EQ(message.status_int(), 200); }; } @@ -156,7 +156,7 @@ TEST_P(AsyncGet, WHEN_two_requests_in_a_row_THEN_both_are_answered) for (size_t i = 0; i < 2; ++i) { auto message = check(co_await session.get(url)); - EXPECT_EQ(message.result_int(), 200); + EXPECT_EQ(message.status_int(), 200); EXPECT_EQ(message.body(), "Hello, World!"); } }; @@ -184,7 +184,7 @@ TEST_P(AsyncGet, WHEN_cancelled_THEN_completes_with_operation_canceled_and_empty auto [ec, message] = co_await session.async_get(url, {}, cancel_after(100ms, as_tuple)); #endif EXPECT_EQ(ec, errc::operation_canceled); - EXPECT_EQ(message.result_int(), 0); + EXPECT_EQ(message.status_int(), 0); EXPECT_THAT(message.body(), IsEmpty()); }; } @@ -228,7 +228,7 @@ TEST(AsyncGetRaw, WHEN_get_THEN_request_line_says_GET) auto get = [&]() -> Task { auto session = check(co_await client.connect()); auto message = check(co_await session.get(url.set_path("/index.html"))); - EXPECT_EQ(message.result_int(), 200); + EXPECT_EQ(message.status_int(), 200); EXPECT_EQ(message.body(), "hello"); }; launch(context.get_executor(), get(), @@ -318,7 +318,7 @@ TEST_P(SubmitMethod, WHEN_get_stops_reading_at_a_limit_THEN_status_and_fields_ar check(co_await request.write_eof()); auto response = check(co_await request.get_response()); EXPECT_EQ(response.status_code(), 200); - EXPECT_EQ(response.fields()["x-answer"], "42"); + EXPECT_EQ(response.fields().value_or("x-answer", ""), "42"); std::array buffer; size_t total = 0; @@ -385,7 +385,7 @@ TEST_P(SubmitMethod, WHEN_response_is_released_early_THEN_only_http11_loses_the_ else { EXPECT_EQ(ec, error_code{}) << ec.message(); - EXPECT_EQ(message.result_int(), 200); + EXPECT_EQ(message.status_int(), 200); for (int i = 0; i < 2000 && !stopped; ++i) co_await sleep(1ms); EXPECT_TRUE(stopped) << "the server is still sending on the stream that was released"; @@ -410,7 +410,8 @@ TEST_P(SubmitMethod, WHEN_method_is_head_THEN_response_ends_after_its_header) check(co_await request.write_eof()); auto response = check(co_await request.get_response()); EXPECT_EQ(response.status_code(), 200); - EXPECT_EQ(response.fields()["content-length"], "5"); + EXPECT_EQ(response.fields().value_or("content-length", ""), "5"); + EXPECT_EQ(response.content_length(), 5) << "announced, even if nothing follows"; EXPECT_EQ(co_await read_all(response), ""); }; } diff --git a/test/test_h1_parser.cpp b/test/test_h1_parser.cpp new file mode 100644 index 0000000..8a5b600 --- /dev/null +++ b/test/test_h1_parser.cpp @@ -0,0 +1,115 @@ +// +// Boost.HTTP's parser on its own, for what anyhttp needs of it that its tests do not cover, and for +// the patches cmake/http.cmake applies to it. +// +#include "anyhttp/h1/io.hpp" + +#include + +#include +#include + +#include +#include +#include + +using namespace testing; + +namespace http = boost::http; + +// ================================================================================================= + +class H1Parser : public Test +{ +protected: + H1Parser() : parser(anyhttp::h1::make_parser_config(true, 16 * 1024)) + { + parser.reset(); + parser.start(); + } + + /// Gives the parser \p input, as one read would, and parses it. + std::error_code put(std::string_view input) + { + auto buffers = parser.prepare(); + EXPECT_GE(buffers[0].size(), input.size()); + input.copy(static_cast(buffers[0].data()), input.size()); + parser.commit(input.size()); + std::error_code ec; + parser.parse(ec); + if (parser.got_header() && !parser.is_complete()) + parser.parse(ec); // the body, which the header stops short of + return ec; + } + + http::request_parser parser; +}; + +// ------------------------------------------------------------------------------------------------- + +// +// The h2c upgrade (h1/session.cpp) hands what follows the request to HTTP/2. +// parser::release_buffered_data() is a stub on Boost.HTTP's 'develop', see +// cmake/patches/http-release-buffered-data.patch. +// + +TEST_F(H1Parser, WHEN_data_follows_a_request_without_body_THEN_it_is_released) +{ + put("GET / HTTP/1.1\r\nHost: x\r\n\r\nPRI * HTTP/2.0\r\n"); + ASSERT_TRUE(parser.is_complete()); + ASSERT_TRUE(parser.has_buffered_data()); + + EXPECT_EQ(parser.release_buffered_data(), "PRI * HTTP/2.0\r\n"); + EXPECT_FALSE(parser.has_buffered_data()) << "released, so the next message does not see it"; + EXPECT_EQ(parser.release_buffered_data(), ""); +} + +TEST_F(H1Parser, WHEN_data_follows_a_body_THEN_only_that_is_released_and_body_stays) +{ + put("POST / HTTP/1.1\r\nHost: x\r\nContent-Length: 5\r\n\r\nhelloworld"); + ASSERT_TRUE(parser.is_complete()); + + EXPECT_EQ(parser.release_buffered_data(), "world"); + EXPECT_EQ(parser.body(), "hello"); +} + +TEST_F(H1Parser, WHEN_nothing_follows_THEN_nothing_is_released) +{ + put("GET / HTTP/1.1\r\nHost: x\r\n\r\n"); + ASSERT_TRUE(parser.is_complete()); + EXPECT_FALSE(parser.has_buffered_data()); + EXPECT_EQ(parser.release_buffered_data(), ""); +} + +TEST_F(H1Parser, WHEN_message_is_incomplete_THEN_nothing_is_released) +{ + put("POST / HTTP/1.1\r\nHost: x\r\nContent-Length: 5\r\n\r\nhel"); + ASSERT_FALSE(parser.is_complete()); + EXPECT_EQ(parser.release_buffered_data(), ""); +} + +// ------------------------------------------------------------------------------------------------- + +// +// The parser's limits on the header section all come out as errors::header_limit, see +// h1::make_parser_config() and h1::to_error_code(). +// + +TEST_F(H1Parser, WHEN_a_single_field_is_larger_than_the_old_limit_THEN_it_is_fine) +{ + // Boost.HTTP's default for a single field is 4 KiB; anyhttp's limit is on the whole section + std::string request = "GET / HTTP/1.1\r\nX-Large: " + std::string(8000, 'a') + "\r\n\r\n"; + EXPECT_FALSE(put(request)); + EXPECT_TRUE(parser.is_complete()); +} + +TEST_F(H1Parser, WHEN_there_are_many_small_fields_THEN_they_count_against_the_total) +{ + std::string request = "GET / HTTP/1.1\r\n"; + for (int i = 0; request.size() < 15 * 1024; ++i) + request += std::format("x-{}: 1\r\n", i); + auto ec = put(request + "\r\n"); + EXPECT_EQ(anyhttp::h1::to_error_code(ec), anyhttp::errors::header_limit) << ec.message(); +} + +// ================================================================================================= diff --git a/test/test_h2_push.cpp b/test/test_h2_push.cpp index 3f2f31a..950d79c 100644 --- a/test/test_h2_push.cpp +++ b/test/test_h2_push.cpp @@ -331,10 +331,12 @@ TEST_P(H2Push, WHEN_a_push_beats_the_settings_ack_THEN_it_is_logged_and_cancelle }, [](Session session, boost::urls::url target) -> Task { auto first = check(co_await session.get(target)); - EXPECT_EQ(first.result_int(), 200); + EXPECT_EQ(first.status_int(), 200); EXPECT_EQ(first.body(), "first"); - EXPECT_EQ(first["x-promised"], "") << "the promise is not part of this response"; - EXPECT_EQ(first["x-pushed"], "") << "the pushed response is not part of this one"; + EXPECT_EQ(first.value_or("x-promised", ""), "") + << "the promise is not part of this response"; + EXPECT_EQ(first.value_or("x-pushed", ""), "") + << "the pushed response is not part of this one"; auto second = check(co_await session.get(target)); EXPECT_EQ(second.body(), "second") << "the session has to stay usable"; diff --git a/test/test_h2c_upgrade.cpp b/test/test_h2c_upgrade.cpp index af3af7f..023d9ae 100644 --- a/test/test_h2c_upgrade.cpp +++ b/test/test_h2c_upgrade.cpp @@ -3,9 +3,9 @@ #include "anyhttp/h1/io.hpp" #include "anyhttp/net.hpp" -#include -#include -#include +#include +#include +#include #include #include @@ -39,21 +39,41 @@ class H2CUpgrade : public Server }; using Responses = std::map; - using Request = boost::beast::http::request; - using Http11Response = boost::beast::http::response; + using Http11Response = client::Message; + /// A request with a body, which a header alone has no place for. + struct Request : boost::http::request + { + using boost::http::request::request; + std::string body; + }; + + /// base64url without padding (RFC 4648, section 5), as 'HTTP2-Settings' has it. static std::string base64url(std::span data) { - namespace base64 = boost::beast::detail::base64; - std::string result(base64::encoded_size(data.size()), '\0'); - result.resize(base64::encode(result.data(), data.data(), data.size())); - std::ranges::replace(result, '+', '-'); - std::ranges::replace(result, '/', '_'); - while (result.ends_with('=')) - result.pop_back(); + static constexpr std::string_view alphabet = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + std::string result; + for (size_t i = 0; i < data.size(); i += 3) + { + const size_t n = std::min(3, data.size() - i); + uint32_t bits = 0; + for (size_t j = 0; j < 3; ++j) + bits = (bits << 8) | (j < n ? data[i + j] : 0); + for (size_t j = 0; j <= n; ++j) + result.push_back(alphabet[(bits >> (18 - 6 * j)) & 0x3f]); + } return result; } + static boost::http::response_parser make_parser() + { + boost::http::response_parser parser(h1::make_parser_config(false, default_max_header_size)); + parser.reset(); + parser.start(); + return parser; + } + static nghttp2_nv nv(std::string_view name, std::string_view value) { return {const_cast(reinterpret_cast(name.data())), @@ -63,10 +83,10 @@ class H2CUpgrade : public Server /// Upgrades a request for the first target, with \p method, and sends GETs for the others as /// HTTP/2 streams. \p fields go along with the upgrade request. - Task upgrade(std::vector targets, boost::beast::http::fields fields = {}, - boost::beast::http::verb method = boost::beast::http::verb::get) + Task upgrade(std::vector targets, Fields fields = {}, + boost::http::method method = boost::http::method::get) { - namespace http = boost::beast::http; + namespace http = boost::http; auto socket = io::make_socket(context.get_executor()); check(co_await io::connect(socket, {server->local_endpoint()})); @@ -117,20 +137,21 @@ class H2CUpgrade : public Server nghttp2_pack_settings_payload2(settings.data(), settings.size(), iv.data(), iv.size()); EXPECT_GT(len, 0); - http::request request{method, targets.front(), 11}; + http::request request(method, targets.front()); for (const auto& field : fields) - request.insert(field.name_string(), field.value()); + request.append(field.name, field.value); request.set(http::field::host, authority); request.set(http::field::connection, "Upgrade, HTTP2-Settings"); request.set(http::field::upgrade, "h2c"); - request.set("HTTP2-Settings", base64url({settings.data(), size_t(len)})); - check(co_await h1::write_message(socket, request)); + request.set(http::field::http2_settings, base64url({settings.data(), size_t(len)})); + http::serializer serializer(h1::make_serializer_config()); + check(co_await h1::write_message(socket, serializer, request)); - boost::beast::flat_buffer buffer; - http::response_parser parser; + Buffer buffer; + auto parser = make_parser(); check(co_await h1::read_header(socket, buffer, parser)); - EXPECT_EQ(parser.get().result(), http::status::switching_protocols); - if (parser.get().result() != http::status::switching_protocols) + EXPECT_EQ(parser.get().status(), http::status::switching_protocols); + if (parser.get().status() != http::status::switching_protocols) co_return responses; // @@ -177,7 +198,9 @@ class H2CUpgrade : public Server out.clear(); }; - recv(buffer.data()); // what came along with the 101 response + // what came along with the 101 response + const auto leftover = parser.release_buffered_data(); + recv(boost::asio::buffer(leftover.data(), leftover.size())); std::array data; for (co_await send(); !done(); co_await send()) { @@ -197,18 +220,22 @@ class H2CUpgrade : public Server /// Sends a single HTTP/1.1 request and reads the response. Task http11(Request request) { - namespace http = boost::beast::http; + namespace http = boost::http; auto socket = io::make_socket(context.get_executor()); check(co_await io::connect(socket, {server->local_endpoint()})); request.set(http::field::host, std::format("127.0.0.2:{}", server->local_endpoint().port())); - request.prepare_payload(); - check(co_await h1::write_message(socket, request)); - - boost::beast::flat_buffer buffer; - Http11Response response; - check(co_await h1::read_message(socket, buffer, response)); + if (!request.body.empty() || request.method() != http::method::get) + request.set_content_length(request.body.size()); + http::serializer serializer(h1::make_serializer_config()); + check(co_await h1::write_message(socket, serializer, request, request.body)); + + Buffer buffer; + auto parser = make_parser(); + auto body = check(co_await h1::read_message(socket, buffer, parser)); + Http11Response response(parser.get()); + response.body() = std::move(body); io::shutdown(socket, io::Shutdown::send); // the server may have closed the connection already co_return response; } @@ -229,13 +256,13 @@ class H2CUpgrade : public Server return result; } - static Request upgrade_request(boost::beast::http::verb method, std::string target) + static Request upgrade_request(boost::http::method method, std::string target) { - namespace http = boost::beast::http; - Request request{method, target, 11}; + namespace http = boost::http; + Request request(method, target); request.set(http::field::connection, "Upgrade, HTTP2-Settings"); request.set(http::field::upgrade, "h2c"); - request.set("HTTP2-Settings", "AAMAAABkAAQAAQAAAAIAAAAA"); // as sent by curl + request.set(http::field::http2_settings, "AAMAAABkAAQAAQAAAAIAAAAA"); // as sent by curl return request; } }; @@ -257,9 +284,9 @@ TEST_F(H2CUpgrade, WHEN_upgrade_is_requested_THEN_request_continues_as_stream_1) TEST_F(H2CUpgrade, WHEN_upgraded_THEN_request_headers_are_passed_on_to_stream_1) { - boost::beast::http::fields fields; + Fields fields; fields.set("x-custom", "value"); - fields.set(boost::beast::http::field::keep_alive, "timeout=5"); + fields.set(boost::http::field::keep_alive, "timeout=5"); auto responses = run(upgrade({"/dump?first"}, std::move(fields))); ASSERT_TRUE(responses.contains(1)); @@ -294,7 +321,7 @@ TEST_F(H2CUpgrade, WHEN_upgraded_THEN_connection_takes_more_streams) // TEST_F(H2CUpgrade, WHEN_options_is_upgraded_THEN_method_is_carried_over_to_stream_1) { - auto responses = run(upgrade({"/dump?options"}, {}, boost::beast::http::verb::options)); + auto responses = run(upgrade({"/dump?options"}, {}, boost::http::method::options)); ASSERT_EQ(responses.size(), 1); ASSERT_TRUE(responses.contains(1)); @@ -307,7 +334,7 @@ TEST_F(H2CUpgrade, WHEN_options_is_upgraded_THEN_method_is_carried_over_to_strea TEST_F(H2CUpgrade, WHEN_options_asterisk_is_upgraded_THEN_connection_takes_more_streams) { - auto responses = run(upgrade({"*", "/dump?second"}, {}, boost::beast::http::verb::options)); + auto responses = run(upgrade({"*", "/dump?second"}, {}, boost::http::method::options)); ASSERT_EQ(responses.size(), 2); EXPECT_EQ(responses[1].status, 404); // the test server has no handler for an asterisk target @@ -318,31 +345,31 @@ TEST_F(H2CUpgrade, WHEN_options_asterisk_is_upgraded_THEN_connection_takes_more_ TEST_F(H2CUpgrade, WHEN_request_has_body_THEN_is_served_as_http11) { - auto request = upgrade_request(boost::beast::http::verb::post, "/echo"); - request.body() = "Hello, World!"; + auto request = upgrade_request(boost::http::method::post, "/echo"); + request.body = "Hello, World!"; auto response = run(http11(std::move(request))); - EXPECT_EQ(response.result_int(), 200); + EXPECT_EQ(response.status_int(), 200); EXPECT_EQ(response.body(), "Hello, World!"); } TEST_F(H2CUpgrade, WHEN_http2_settings_are_missing_THEN_is_served_as_http11) { - auto request = upgrade_request(boost::beast::http::verb::get, "/dump?no-settings"); - request.erase("HTTP2-Settings"); + auto request = upgrade_request(boost::http::method::get, "/dump?no-settings"); + request.erase(boost::http::field::http2_settings); auto response = run(http11(std::move(request))); - EXPECT_EQ(response.result_int(), 200); + EXPECT_EQ(response.status_int(), 200); EXPECT_THAT(response.body(), HasSubstr("query: no-settings")); } TEST_F(H2CUpgrade, WHEN_http2_settings_are_invalid_THEN_is_served_as_http11) { - auto request = upgrade_request(boost::beast::http::verb::get, "/dump?invalid"); - request.set("HTTP2-Settings", "AAMAAABkAA"); // 7 bytes, not a multiple of 6 + auto request = upgrade_request(boost::http::method::get, "/dump?invalid"); + request.set(boost::http::field::http2_settings, "AAMAAABkAA"); // 7 bytes, not a multiple of 6 auto response = run(http11(std::move(request))); - EXPECT_EQ(response.result_int(), 200); + EXPECT_EQ(response.status_int(), 200); EXPECT_THAT(response.body(), HasSubstr("query: invalid")); } diff --git a/test/test_headers.cpp b/test/test_headers.cpp index f45e685..9a71cf7 100644 --- a/test/test_headers.cpp +++ b/test/test_headers.cpp @@ -12,7 +12,7 @@ using namespace testing; // // HTTP/2 splits a header block that does not fit into a single frame (16 KiB by default) into // HEADERS + CONTINUATION frames, HTTP/3 compresses the whole field section with QPACK, and -// HTTP/1.1 parses it with a Beast parser. All of them are subject to Config::max_header_size. +// HTTP/1.1 parses it with Boost.HTTP's parser. All of them are subject to Config::max_header_size. // class Headers : public ClientAsync { @@ -32,36 +32,42 @@ static Fields make_fields(size_t count, size_t size) { auto value = std::format("value-{}-", i); value.resize(std::max(size, value.size()), char('a' + i % 26)); - result.insert(std::format("x-header-{}", i), value); + result.append(std::format("x-header-{}", i), value); } return result; } +// +// The helpers take the fields of a message as well, a client::Message: both are Boost.HTTP's +// fields_base. +// +using AnyFields = boost::http::fields_base; + /// All values of the fields named \p name, in the order they appear in. -static std::vector values_of(const Fields& fields, std::string_view name) +static std::vector values_of(const AnyFields& fields, std::string_view name) { - auto [begin, end] = fields.equal_range(name); - return std::ranges::subrange(begin, end) | - rv::transform([](auto& field) { return std::string_view(field.value()); }) | - std::ranges::to(); + std::vector result; + for (auto value : fields.find_all(name)) + result.emplace_back(value); + return result; } /// All fields as (name, value) pairs, in the order they appear in. -static std::vector> pairs_of(const Fields& fields) +static std::vector> pairs_of(const AnyFields& fields) { - return fields | rv::transform([](auto& field) { - return std::pair(std::string_view(field.name_string()), - std::string_view(field.value())); - }) | - std::ranges::to(); + std::vector> result; + for (auto&& field : fields) + result.emplace_back(field.name, field.value); + return result; } /// Expects every field of \p expected to be found in \p actual. -static void expect_contains(const Fields& actual, const Fields& expected) +static void expect_contains(const AnyFields& actual, const Fields& expected) { auto fields = pairs_of(actual); for (auto&& field : expected) - EXPECT_THAT(fields, Contains(Pair(StrCaseEq(field.name_string()), field.value()))); + EXPECT_THAT(fields, Contains(Pair(StrCaseEq(std::string_view(field.name)), + std::string_view(field.value)))); } /// Number of bytes the fields take up on an HTTP/1.1 wire, roughly. @@ -69,7 +75,7 @@ static size_t wire_size(const Fields& fields) { size_t result = 0; for (auto&& field : fields) - result += field.name_string().size() + field.value().size() + 4; // ": " and CRLF + result += field.name.size() + field.value.size() + 4; // ": " and CRLF return result; } @@ -91,7 +97,7 @@ void Headers::round_trip(Fields sent) clientSession = [this, sent](Session session) -> Task { auto message = check(co_await session.get(url, sent)); - EXPECT_EQ(message.result_int(), 200); + EXPECT_EQ(message.status_int(), 200); expect_contains(message, sent); EXPECT_THAT(message.body(), IsEmpty()); }; @@ -112,7 +118,7 @@ TEST_P(Headers, WHEN_header_name_repeats_THEN_all_values_arrive_in_order) Fields sent; std::vector values; for (size_t i = 0; i < 50; ++i) - sent.insert("x-repeated", values.emplace_back(std::format("value-{}", i))); + sent.append("x-repeated", values.emplace_back(std::format("value-{}", i))); requestHandler = [sent, values](server::Request request, server::Response response) -> Task { @@ -159,7 +165,7 @@ TEST_P(Headers, WHEN_request_headers_exceed_default_limit_THEN_server_responds_4 clientSession = [this, sent](Session session) -> Task { auto message = check(co_await session.get(url, sent)); - EXPECT_EQ(message.result_int(), 431); + EXPECT_EQ(message.status_int(), 431); }; } @@ -185,7 +191,7 @@ TEST_P(Headers, WHEN_complex_url_is_submitted_THEN_is_provided_to_request_handle clientSession = [this](Session session) -> Task { std::println("URL: {}", url.buffer()); auto message = check(co_await session.get(url, {})); - EXPECT_EQ(message.result_int(), 200); + EXPECT_EQ(message.status_int(), 200); }; } @@ -256,7 +262,7 @@ class HeaderLimits : public ClientAsync auto [ec, message] = co_await session.get(target, sent); if (ec) co_return std::unexpected(ec); - co_return message.result_int(); + co_return message.status_int(); } size_t handled = 0; diff --git a/test/test_ipv6.cpp b/test/test_ipv6.cpp index 1678659..a5ae338 100644 --- a/test/test_ipv6.cpp +++ b/test/test_ipv6.cpp @@ -2,7 +2,7 @@ using namespace testing; -namespace http = boost::beast::http; +namespace http = boost::http; // ================================================================================================= @@ -39,7 +39,7 @@ TEST_P(IPv6, WHEN_server_listens_on_ipv6_loopback_THEN_get_succeeds) respond_with("Hello, IPv6!"); clientSession = [this](Session session) -> Task { auto message = check(co_await session.get(url)); - EXPECT_EQ(message.result(), http::status::ok); + EXPECT_EQ(message.status(), http::status::ok); EXPECT_EQ(message.body(), "Hello, IPv6!"); }; } @@ -66,7 +66,7 @@ TEST_P(DualStack, WHEN_ipv6_server_is_reached_over_ipv4_THEN_get_succeeds) respond_with("Hello, IPv4!"); clientSession = [this](Session session) -> Task { auto message = check(co_await session.get(url)); - EXPECT_EQ(message.result(), http::status::ok); + EXPECT_EQ(message.status(), http::status::ok); EXPECT_EQ(message.body(), "Hello, IPv4!"); }; } @@ -94,7 +94,7 @@ TEST_P(DualStackSecondaryAddress, WHEN_server_is_reached_on_a_secondary_address_ respond_with("Hello, 127.0.0.2!"); clientSession = [this](Session session) -> Task { auto message = check(co_await session.get(url)); - EXPECT_EQ(message.result(), http::status::ok); + EXPECT_EQ(message.status(), http::status::ok); EXPECT_EQ(message.body(), "Hello, 127.0.0.2!"); }; } diff --git a/test/test_quic.cpp b/test/test_quic.cpp index 9b74e48..97587ff 100644 --- a/test/test_quic.cpp +++ b/test/test_quic.cpp @@ -513,7 +513,7 @@ TEST_P(QuicStreams, WHEN_http3_is_on_the_same_port_THEN_both_work) EXPECT_FALSE(connect_ec) << connect_ec.message(); auto [get_ec, message] = co_await session.get(url, {}); EXPECT_FALSE(get_ec) << get_ec.message(); - EXPECT_EQ(message.result_int(), 404); + EXPECT_EQ(message.status_int(), 404); }()); }