From 09c8aaca05f5a4d2c30bc23b1cecc7794a6d0203 Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Mon, 2 Nov 2026 21:34:25 +0000 Subject: [PATCH 01/27] chore: crawl, connecting the way the HTTPS records of a host say The crawler looks up a host's HTTPS records with c-ares-async and follows RFC 9460 to get from them to an ordered list of routes: aliases, compatible records, priority, port prefixes, and treating "http" as "https" where there is a record, without going back to cleartext. routes.cpp does this without any I/O. Only what anyhttp can do becomes a route, which is HTTP/3 to the origin's own name, and HTTP/1.1 in the clear to addresses from c-ares where there is no record. Anything else is rejected with its reason, so most real sites can't be crawled until anyhttp has TLS over TCP and can connect to one name while verifying another. It refuses to run without an allowlist of exact host names, which every seed, link and redirect is checked against. It reads robots.txt first, stops on an origin whose robots.txt can't be read, and spaces its requests per host. The tests run on loopback only, against a real anyhttp server and the DNS server from c-ares-async's tests. Two of them stop the context early, as anyhttp does not close a session whose h3 connect was cancelled and it goes on sending for 30 seconds. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 17 +- src/crawler.cpp | 461 ++++++++++++++++++++++++++++ src/crawler.hpp | 142 +++++++++ src/links.cpp | 188 ++++++++++++ src/links.hpp | 40 +++ src/main.cpp | 135 ++++++++- src/robots.cpp | 189 ++++++++++++ src/robots.hpp | 53 ++++ src/routes.cpp | 220 ++++++++++++++ src/routes.hpp | 90 ++++++ test/CMakeLists.txt | 56 ++++ test/crawl_test.cpp | 698 +++++++++++++++++++++++++++++++++++++++++++ test/demo_site.cpp | 41 +++ test/links_test.cpp | 122 ++++++++ test/robots_test.cpp | 159 ++++++++++ test/routes_test.cpp | 273 +++++++++++++++++ test/site.cpp | 103 +++++++ test/site.hpp | 87 ++++++ 18 files changed, 3061 insertions(+), 13 deletions(-) create mode 100644 src/crawler.cpp create mode 100644 src/crawler.hpp create mode 100644 src/links.cpp create mode 100644 src/links.hpp create mode 100644 src/robots.cpp create mode 100644 src/robots.hpp create mode 100644 src/routes.cpp create mode 100644 src/routes.hpp create mode 100644 test/CMakeLists.txt create mode 100644 test/crawl_test.cpp create mode 100644 test/demo_site.cpp create mode 100644 test/links_test.cpp create mode 100644 test/robots_test.cpp create mode 100644 test/routes_test.cpp create mode 100644 test/site.cpp create mode 100644 test/site.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index d77f2b6..4f7ef46 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,9 +55,20 @@ FetchContent_MakeAvailable(anyhttp c_ares_async) # -------------------------------------------------------------------------------------------------- +# +# The crawler in a library of its own, for the tests to link. The warnings are not for the sources +# from c-ares-async that the tests borrow, see test/CMakeLists.txt. +# +set(crawler_warnings -Wall -Wextra -Werror -Wno-missing-designated-field-initializers) + +add_library(crawler_core STATIC src/crawler.cpp src/links.cpp src/robots.cpp src/routes.cpp) +target_include_directories(crawler_core PUBLIC src) +target_link_libraries(crawler_core PUBLIC anyhttp::anyhttp c_ares_async::c_ares_async) +target_compile_options(crawler_core PRIVATE ${crawler_warnings}) + add_executable(crawler src/main.cpp) -target_link_libraries(crawler PRIVATE anyhttp::anyhttp c_ares_async::c_ares_async) -target_compile_options(crawler PRIVATE -Wall -Wextra -Werror) +target_link_libraries(crawler PRIVATE crawler_core) +target_compile_options(crawler PRIVATE ${crawler_warnings}) enable_testing() -add_test(NAME crawler_starts COMMAND crawler) +add_subdirectory(test) diff --git a/src/crawler.cpp b/src/crawler.cpp new file mode 100644 index 0000000..e17744c --- /dev/null +++ b/src/crawler.cpp @@ -0,0 +1,461 @@ +#include "crawler.hpp" + +#include "links.hpp" + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace http = boost::beast::http; + +namespace crawler +{ + +namespace +{ + +// ================================================================================================= + +constexpr std::string_view agent_token = "crawler"; // what a robots.txt addresses us by +constexpr std::string_view user_agent = "crawler/0.0.1 (+https://github.com/pgit/crawler)"; + +constexpr int max_alias_chain = 8; +constexpr int max_robots_redirects = 5; // RFC 9309, section 2.3.1.2 +constexpr std::size_t max_frontier = 100'000; + +bool is_html(std::string_view content_type) +{ + std::string type(content_type.substr(0, content_type.find(';'))); + std::ranges::transform(type, type.begin(), [](unsigned char c) { return std::tolower(c); }); + return type.empty() || type.starts_with("text/html") || + type.starts_with("application/xhtml+xml"); +} + +bool is_redirect(unsigned status) +{ + return status == 301 || status == 302 || status == 303 || status == 307 || status == 308; +} + +// ================================================================================================= + +} // namespace + +Crawler::Crawler(anyhttp::Executor executor, Config config, Observer observer) + : executor_(executor), config_(std::move(config)), observer_(std::move(observer)), + resolver_(std::move(executor)) +{ + if (config_.allowed_hosts.empty()) + throw std::invalid_argument("refusing to crawl without an allowlist of hosts"); + + for (const auto& host : config_.allowed_hosts) + { + if (host.empty() || host.find('*') != host.npos) + throw std::invalid_argument( + std::format("'{}' is not a host: the allowlist takes exact names", host)); + allowed_hosts_.insert(normalize_host(host)); + } + + if (config_.seeds.empty()) + throw std::invalid_argument("nothing to crawl: no URLs given"); + + if (!config_.dns_server.empty()) + if (const auto ec = resolver_.set_servers(config_.dns_server)) + throw std::invalid_argument( + std::format("invalid DNS server '{}': {}", config_.dns_server, ec.message())); + + for (const auto& seed : config_.seeds) + { + const auto parsed = boost::urls::parse_uri(seed); + const auto url = parsed ? resolve_link(*parsed, seed) : std::nullopt; + if (!url) + throw std::invalid_argument(std::format("'{}' is not an http or https URL", seed)); + if (!is_allowed(url->host_address())) + throw std::invalid_argument( + std::format("'{}' is not on an allowed host: '{}'", seed, url->host_address())); + if (seen_.insert(std::string(url->buffer())).second) + frontier_.push_back(*url); + } +} + +// ------------------------------------------------------------------------------------------------- + +bool Crawler::is_allowed(std::string_view host) const +{ + return allowed_hosts_.contains(normalize_host(host)); +} + +void Crawler::emit(Event event) const +{ + if (observer_) + observer_(event); +} + +void Crawler::enqueue(boost::urls::url url) +{ + if (!seen_.insert(std::string(url.buffer())).second) + return; + + if (!is_allowed(url.host_address())) + emit({.kind = Event::Kind::outside_allowlist, .url = std::string(url.buffer())}); + else if (frontier_.size() < max_frontier) + frontier_.push_back(std::move(url)); +} + +// ------------------------------------------------------------------------------------------------- + +// +// SVCB resolution, section 3 of RFC 9460, up to the point where it would take addresses. A lookup +// that fails is not fatal: DNS is not protected here, and this is the choice section 3.1 leaves +// for that. A name without a record is the common case and not even worth a note. +// +anyhttp::Task Crawler::resolve_service(const Origin& origin) +{ + ServiceRecords found{.owner = https_query_name(origin)}; + std::set visited{found.owner}; + + for (int hop = 0; hop < max_alias_chain; ++hop) + { + auto [ec, records] = co_await resolver_.resolve_https(found.owner); + if (ec) + { + if (ec != c_ares_async::make_ares_error(ARES_ENODATA) && + ec != c_ares_async::make_ares_error(ARES_ENOTFOUND)) + found.notes.push_back( + std::format("HTTPS lookup of '{}': {}", found.owner, ec.message())); + co_return found; + } + + // + // A record in AliasMode makes the rest of the set irrelevant (section 2.4.1). + // + const auto alias = std::ranges::find(records, 0, &c_ares_async::https_record::priority); + if (alias == records.end()) + { + found.records = std::move(records); + co_return found; + } + + if (alias->target.empty() || alias->target == ".") + { + // + // "The service is not available" (section 2.5.1), which is advisory. Treated like no + // record at all, what comes of that is the connection attempt's to find out. + // + found.notes.push_back(std::format("'{}' says the service is not available", found.owner)); + found.aliased = false; + co_return found; + } + + found.aliased = true; + found.owner = normalize_host(alias->target); + if (!visited.insert(found.owner).second) + break; + } + + // + // Section 3.1: as if the name had no record at all. + // + co_return ServiceRecords{.owner = https_query_name(origin), + .notes = {std::format("alias chain of '{}' is too long or has a loop", + https_query_name(origin))}}; +} + +anyhttp::Task Crawler::plan_for(const Origin& origin) +{ + const auto key = origin.key(); + if (auto it = plans_.find(key); it != plans_.end()) + co_return &it->second; + + auto found = + origin.is_address ? ServiceRecords{.owner = origin.host} : co_await resolve_service(origin); + + auto plan = plan_routes(origin, found); + for (const auto& route : plan.routes) + emit({.kind = Event::Kind::note, + .url = key, + .route = route.to_string(), + .detail = route.source}); + for (const auto& rejected : plan.rejected) + emit({.kind = Event::Kind::note, .url = key, .detail = rejected}); + + co_return &plans_.emplace(key, std::move(plan)).first->second; +} + +// +// RFC 9309, section 2.3.1: anything in the 400s means there are no rules, anything else that is not +// a robots.txt means that we can not know them, and then nothing is allowed. That goes for a +// redirect to another origin as well, which the RFC would have us follow. It is not worth the +// complication to do that while we can not connect to most of them. +// +anyhttp::Task Crawler::robots_for(Plan& plan) +{ + const auto key = plan.origin.key(); + if (auto it = robots_.find(key); it != robots_.end()) + co_return &it->second; + + auto url = boost::urls::url(std::format("{}/robots.txt", key)); + Robots robots = Robots::disallow_all(); + std::string problem; + + for (int redirects = 0;; ++redirects) + { + const auto response = co_await fetch(plan, url); + if (response.ec) + { + problem = response.error; + break; + } + + const auto status = response.message.result_int(); + if (status >= 200 && status < 300) + robots = Robots::parse(response.message.body(), agent_token); + else if (status >= 400 && status < 500) + robots = Robots{}; + else if (is_redirect(status) && redirects < max_robots_redirects) + { + const auto target = resolve_link(url, response.message[http::field::location]); + if (target && origin_of(*target) && origin_of(*target)->key() == key) + { + url = *target; + continue; + } + problem = "redirected to another origin"; + } + else + problem = std::format("status {}", status); + break; + } + + if (!problem.empty()) + emit({.kind = Event::Kind::note, + .url = key, + .detail = + std::format("robots.txt could not be read, so nothing is allowed: {}", problem)}); + else if (const auto crawl_delay = robots.crawl_delay()) + { + auto& pacing = pacing_[plan.origin.host]; + pacing.delay = std::max(pacing.delay, *crawl_delay); + } + + co_return &robots_.emplace(key, std::move(robots)).first->second; +} + +// ------------------------------------------------------------------------------------------------- + +// +// One request, whichever route gets it through. This is the only place that sends requests, so it +// is the one to see to the spacing, and to the allowlist one last time. +// +anyhttp::Task Crawler::fetch(Plan& plan, boost::urls::url url) +{ + if (!is_allowed(plan.origin.host)) + throw std::logic_error(std::format("about to contact '{}'", plan.origin.host)); + + Response response; + response.ec = make_error_code(boost::system::errc::host_unreachable); + response.error = "no way to connect"; + + auto& pacing = pacing_[plan.origin.host]; + const auto wait_for = [&] { return pacing.next_request - std::chrono::steady_clock::now(); }; + if (wait_for().count() > 0) + co_await anyhttp::delay(wait_for()); + + for (auto route = plan.routes.begin(); route != plan.routes.end(); ++route) + { + response = co_await attempt(*route, url); + if (response.ec) + { + emit({.kind = Event::Kind::note, + .url = std::string(url.buffer()), + .route = route->to_string(), + .detail = response.error}); + continue; + } + + std::rotate(plan.routes.begin(), route, std::next(route)); // go straight there next time + break; + } + + pacing.next_request = std::chrono::steady_clock::now() + std::max(config_.delay, pacing.delay); + co_return response; +} + +anyhttp::Task Crawler::attempt(const Route& route, boost::urls::url url) +{ + anyhttp::client::Config client_config; + client_config.protocol = route.protocol; + client_config.tls_ca_file = config_.tls_ca_file; + + const auto with_route = [&](Response response) { + response.route = route.to_string(); + return response; + }; + + // + // HTTP/3 goes to the name of the route, as that is also what anyhttp verifies the certificate + // against, which leaves it to its own resolver to look up the address. + // + if (route.protocol == anyhttp::Protocol::h3) + { + client_config.url.set_scheme("https"); + client_config.url.set_host_address(route.host); + client_config.url.set_port_number(route.port); + co_return with_route(co_await exchange(std::move(client_config), std::move(url))); + } + + // + // In the clear, nothing has to be verified, so the addresses can be ours. The lookup stays + // outside of the timeout: the image has c-ares 1.34.5, and cancelling a lookup is what + // GHSA-6wfj-rwm7-3542 is about. c-ares has timeouts of its own. + // + auto [ec, endpoints] = co_await resolver_.resolve(route.host, std::to_string(route.port)); + if (ec) + co_return with_route( + Response{.ec = ec, .error = std::format("{}: {}", route.host, ec.message())}); + + Response response{.ec = make_error_code(boost::system::errc::host_unreachable), + .error = std::format("{}: no addresses", route.host)}; + for (const auto& endpoint : endpoints) + { + client_config.url.set_scheme("http"); + client_config.url.set_host_address(endpoint.address().to_string()); + client_config.url.set_port_number(endpoint.port()); + response = co_await exchange(client_config, url); + if (!response.ec) + break; + } + co_return with_route(std::move(response)); +} + +// +// Connects as told and asks for \p url, whose host is what goes into the "Host" field, whatever +// \p client_config connects to. Everything is reported rather than thrown: when_either() would +// otherwise sit out the timeout, as what throws does not count as having finished first. +// +anyhttp::Task Crawler::exchange(anyhttp::client::Config client_config, + boost::urls::url url) +{ + const auto run = [](anyhttp::Executor executor, anyhttp::client::Config client_config, + boost::urls::url url) -> anyhttp::Task { + try + { + anyhttp::client::Client client(executor, std::move(client_config)); + auto [connect_ec, session] = co_await client.connect(); + if (connect_ec) + co_return Response{.ec = connect_ec, + .error = std::format("connect: {}", connect_ec.message())}; + + anyhttp::Fields headers; + headers.set(http::field::user_agent, user_agent); + auto [get_ec, message] = co_await session.get(std::move(url), std::move(headers)); + co_return Response{.ec = get_ec, + .message = std::move(message), + .error = get_ec ? std::format("request: {}", get_ec.message()) : ""}; + } + catch (const std::exception& ex) + { + co_return Response{.ec = make_error_code(boost::system::errc::connection_aborted), + .error = ex.what()}; + } + }; + + auto result = co_await anyhttp::when_either( + run(executor_, std::move(client_config), std::move(url)), anyhttp::delay(config_.timeout)); + + if (auto* response = std::get_if<0>(&result)) + co_return std::move(*response); + + co_return Response{.ec = make_error_code(boost::system::errc::timed_out), + .error = std::format("timed out after {}", config_.timeout)}; +} + +// ------------------------------------------------------------------------------------------------- + +// +// What a response means for the rest of the crawl: more URLs, if it has any. +// +void Crawler::visit(const boost::urls::url& url, const Response& response) +{ + const auto status = response.message.result_int(); + std::size_t links = 0; + + if (is_redirect(status)) + { + if (const auto target = resolve_link(url, response.message[http::field::location])) + enqueue(*target); + } + else if (status >= 200 && status < 300 && is_html(response.message[http::field::content_type])) + { + for (const auto& href : extract_links(response.message.body())) + if (auto target = resolve_link(url, href)) + { + enqueue(std::move(*target)); + ++links; + } + } + + emit({.kind = Event::Kind::fetched, + .url = std::string(url.buffer()), + .status = static_cast(status), + .route = response.route, + .detail = std::format("{} bytes, {} links", response.message.body().size(), links)}); +} + +anyhttp::Task Crawler::run() +{ + while (!frontier_.empty() && requested_ < config_.max_pages) + { + auto url = std::move(frontier_.front()); + frontier_.pop_front(); + + auto* plan = co_await plan_for(*origin_of(url)); + + // + // The records can turn "http" into "https" (RFC 9460, section 9.5), and that may well be a + // page we have already been to, or are going to. + // + if (url.scheme() != plan->origin.scheme) + { + url.set_scheme(plan->origin.scheme); + if (url.has_port()) + url.set_port_number(plan->origin.port); + if (!seen_.insert(std::string(url.buffer())).second) + continue; + } + + const auto text = std::string(url.buffer()); + if (plan->routes.empty()) + { + emit({.kind = Event::Kind::failed, .url = text, .detail = "no way to connect"}); + continue; + } + + const auto* robots = co_await robots_for(*plan); + if (!robots->allowed(url.encoded_target())) + { + emit({.kind = Event::Kind::disallowed, .url = text}); + continue; + } + + ++requested_; + const auto response = co_await fetch(*plan, url); + if (response.ec) + emit({.kind = Event::Kind::failed, .url = text, .detail = response.error}); + else + visit(url, response); + } +} + +// ================================================================================================= + +} // namespace crawler diff --git a/src/crawler.hpp b/src/crawler.hpp new file mode 100644 index 0000000..9cbd94d --- /dev/null +++ b/src/crawler.hpp @@ -0,0 +1,142 @@ +// +// The crawl: one URL at a time, breadth first, from the seeds and within the allowlist. +// +#pragma once + +#include "robots.hpp" +#include "routes.hpp" + +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace crawler +{ + +// ================================================================================================= + +struct Config +{ + /// The hosts that may be connected to, exactly: no wildcards, and no subdomains implied. + std::vector allowed_hosts; + + /// Where to start. Each has to be on an allowed host. + std::vector seeds; + + /// How many pages to request at most, whatever comes of them. robots.txt does not count. + std::size_t max_pages = 100; + + /// The least time from the end of one request to the start of the next one, per host. A longer + /// "Crawl-delay" in the host's robots.txt takes precedence. + std::chrono::milliseconds delay{1000}; + + /// How long one request may take on one route, from connecting to the end of the response. + std::chrono::milliseconds timeout{10000}; + + /// The DNS servers to ask, as resolver::set_servers() takes them. Empty for those of the system. + std::string dns_server; + + /// The CA certificates to verify servers against, as a PEM file. Empty for the system's. + std::string tls_ca_file; +}; + +/// Something that happened in the course of a crawl, for whoever is watching. +struct Event +{ + enum class Kind + { + fetched, ///< a response arrived, whatever its status + failed, ///< no response could be had + disallowed, ///< robots.txt says no, or it could not be read + outside_allowlist, ///< a link or a redirect to a host that is not allowed + note, ///< something worth knowing about how a host is reached + }; + + Kind kind = Kind::note; + std::string url; + int status = 0; ///< with \c fetched + std::string route; ///< with \c fetched, and \c note where about a route + std::string detail; +}; + +/** + * Crawls, using the resolver of c-ares-async to find out how to connect (see plan_routes()). + * + * The only thing that can make this reach a host is that host being in Config::allowed_hosts: + * every seed, link and redirect is checked against it. Requests per host are spaced by + * Config::delay and the host's robots.txt is honoured -- a robots.txt that can not be read + * stops the crawl of its origin. + * + * There is one request at a time and a new connection for each of them. + */ +class Crawler +{ +public: + using Observer = std::function; + + /// Throws \c std::invalid_argument for a configuration that is not good to go, starting with + /// one that has no allowlist, and does nothing else until run(). + Crawler(anyhttp::Executor executor, Config config, Observer observer = {}); + + /// Crawls until there is nothing left to do or Config::max_pages is reached. + anyhttp::Task run(); + +private: + struct Response + { + anyhttp::error_code ec; + anyhttp::client::Message message; + std::string route; + std::string error; // what went wrong, with ec or from an exception, on the last route + }; + + struct Pacing + { + std::chrono::steady_clock::time_point next_request{}; + std::chrono::milliseconds delay{}; + }; + + bool is_allowed(std::string_view host) const; + void emit(Event event) const; + void enqueue(boost::urls::url url); + + anyhttp::Task resolve_service(const Origin& origin); + anyhttp::Task plan_for(const Origin& origin); + anyhttp::Task robots_for(Plan& plan); + + anyhttp::Task fetch(Plan& plan, boost::urls::url url); + anyhttp::Task attempt(const Route& route, boost::urls::url url); + anyhttp::Task exchange(anyhttp::client::Config client_config, boost::urls::url url); + + void visit(const boost::urls::url& url, const Response& response); + + anyhttp::Executor executor_; + Config config_; + Observer observer_; + std::set allowed_hosts_; + c_ares_async::resolver resolver_; + + std::deque frontier_; + std::set seen_; + std::size_t requested_ = 0; + + std::map plans_; // by Origin::key() + std::map robots_; // by Origin::key(), of the origin in the plan + std::map pacing_; // by host +}; + +// ================================================================================================= + +} // namespace crawler diff --git a/src/links.cpp b/src/links.cpp new file mode 100644 index 0000000..ea0bddb --- /dev/null +++ b/src/links.cpp @@ -0,0 +1,188 @@ +#include "links.hpp" + +#include + +#include +#include +#include + +namespace crawler +{ + +namespace +{ + +// ================================================================================================= + +bool equals_ignoring_case(std::string_view a, std::string_view b) +{ + return std::ranges::equal( + a, b, [](unsigned char x, unsigned char y) { return std::tolower(x) == std::tolower(y); }); +} + +// +// Where \p needle starts in \p text, from \p from on and in any case. +// +std::size_t find_ignoring_case(std::string_view text, std::string_view needle, std::size_t from) +{ + for (std::size_t i = from; i + needle.size() <= text.size(); ++i) + if (equals_ignoring_case(text.substr(i, needle.size()), needle)) + return i; + return std::string_view::npos; +} + +std::string decode_ampersands(std::string_view text) +{ + std::string result; + for (std::size_t i = 0; i < text.size();) + if (text.substr(i, 5) == "&") + { + result += '&'; + i += 5; + } + else + result += text[i++]; + return result; +} + +std::string_view trim(std::string_view text) +{ + const auto is_space = [](unsigned char c) { return std::isspace(c) != 0; }; + while (!text.empty() && is_space(text.front())) + text.remove_prefix(1); + while (!text.empty() && is_space(text.back())) + text.remove_suffix(1); + return text; +} + +// ================================================================================================= + +} // namespace + +std::vector extract_links(std::string_view html) +{ + std::vector links; + const auto is_space = [](unsigned char c) { return std::isspace(c) != 0; }; + + std::size_t pos = 0; + while ((pos = html.find('<', pos)) != html.npos) + { + if (html.substr(pos, 4) == "", pos + 4); + if (end == html.npos) + break; + pos = end + 3; + continue; + } + + auto cursor = pos + 1; + const auto name_start = cursor; + while (cursor < html.size() && std::isalnum(static_cast(html[cursor]))) + ++cursor; + const auto name = html.substr(name_start, cursor - name_start); + if (name.empty()) + { + ++pos; // a closing tag, a declaration, or just a '<' in the text + continue; + } + + // + // The attributes, up to the end of the tag. + // + std::string_view href, rel; + bool has_href = false; + while (cursor < html.size() && html[cursor] != '>') + { + if (is_space(html[cursor]) || html[cursor] == '/') + { + ++cursor; + continue; + } + + const auto attribute_start = cursor; + while (cursor < html.size() && !is_space(html[cursor]) && html[cursor] != '=' && + html[cursor] != '>') + ++cursor; + const auto attribute = html.substr(attribute_start, cursor - attribute_start); + + while (cursor < html.size() && is_space(html[cursor])) + ++cursor; + if (cursor >= html.size() || html[cursor] != '=') + continue; // an attribute without a value + + ++cursor; + while (cursor < html.size() && is_space(html[cursor])) + ++cursor; + + std::string_view value; + if (cursor < html.size() && (html[cursor] == '"' || html[cursor] == '\'')) + { + const auto end = html.find(html[cursor], cursor + 1); + if (end == html.npos) + return links; // the quote never ends, and neither does the tag + value = html.substr(cursor + 1, end - cursor - 1); + cursor = end + 1; + } + else + { + const auto value_start = cursor; + while (cursor < html.size() && !is_space(html[cursor]) && html[cursor] != '>') + ++cursor; + value = html.substr(value_start, cursor - value_start); + } + + if (equals_ignoring_case(attribute, "href")) + { + href = value; + has_href = true; + } + else if (equals_ignoring_case(attribute, "rel")) + rel = value; + } + + if (equals_ignoring_case(name, "a") && has_href && + find_ignoring_case(rel, "nofollow", 0) == rel.npos) + links.push_back(decode_ampersands(trim(href))); + + pos = cursor; + + // + // What is inside of these is not markup, whatever it looks like. + // + if (equals_ignoring_case(name, "script") || equals_ignoring_case(name, "style")) + { + const auto end = find_ignoring_case(html, std::string(" resolve_link(boost::urls::url_view base, std::string_view reference) +{ + const auto parsed = boost::urls::parse_uri_reference(reference); + if (!parsed) + return std::nullopt; + + boost::urls::url url; + if (!boost::urls::resolve(base, *parsed, url)) + return std::nullopt; + + const auto scheme = url.scheme_id(); + if ((scheme != boost::urls::scheme::http && scheme != boost::urls::scheme::https) || + url.encoded_host().empty() || url.has_userinfo()) + return std::nullopt; + + url.remove_fragment(); + url.normalize(); + if (url.encoded_path().empty()) + url.set_encoded_path("/"); + return url; +} + +// ================================================================================================= + +} // namespace crawler diff --git a/src/links.hpp b/src/links.hpp new file mode 100644 index 0000000..6fb9d71 --- /dev/null +++ b/src/links.hpp @@ -0,0 +1,40 @@ +// +// Finding the links in a page, and making absolute URLs out of them. +// +#pragma once + +#include +#include + +#include +#include +#include +#include + +namespace crawler +{ + +// ================================================================================================= + +/** + * The targets of the \c elements in \p html, in the order they appear and as they are written, + * without those marked rel="nofollow". + * + * This is a scanner, not an HTML parser. It knows about quotes, comments, and the contents of + * \c )" + R"()" + R"(y)"), + ElementsAre("/yes")); +} + +TEST(ExtractLinks, HonoursNofollow) +{ + EXPECT_THAT(extract_links(R"(xy)"), + ElementsAre("/yes")); +} + +TEST(ExtractLinks, DecodesAmpersandsAndTrimsSpace) +{ + EXPECT_THAT(extract_links(R"(a)"), ElementsAre("/a?x=1&y=2")); +} + +TEST(ExtractLinks, SurvivesWhatIsCutOff) +{ + EXPECT_THAT(extract_links(R"(never ends"), IsEmpty()); + EXPECT_THAT(extract_links("1 < 2

"), ElementsAre("/a")); +} + +// ------------------------------------------------------------------------------------------------- + +namespace +{ +std::string resolved(std::string_view reference) +{ + const auto url = resolve_link(boost::urls::url("http://example.com/dir/page?q=1"), reference); + return url ? std::string(url->buffer()) : "(none)"; +} +} // namespace + +TEST(ResolveLink, ResolvesAgainstTheBase) +{ + EXPECT_EQ(resolved("other"), "http://example.com/dir/other"); + EXPECT_EQ(resolved("/top"), "http://example.com/top"); + EXPECT_EQ(resolved("../up"), "http://example.com/up"); + EXPECT_EQ(resolved("?x=2"), "http://example.com/dir/page?x=2"); + EXPECT_EQ(resolved("//other.example/"), "http://other.example/"); + EXPECT_EQ(resolved("https://other.example/x"), "https://other.example/x"); +} + +TEST(ResolveLink, MakesTheSameOfTheSamePage) +{ + EXPECT_EQ(resolved("/a#section"), "http://example.com/a"); + EXPECT_EQ(resolved("#section"), "http://example.com/dir/page?q=1"); + EXPECT_EQ(resolved("HTTP://EXAMPLE.com"), "http://example.com/"); + EXPECT_EQ(resolved("/x/../a"), "http://example.com/a"); +} + +TEST(ResolveLink, TakesNothingButHttpAndHttps) +{ + EXPECT_EQ(resolved("mailto:someone@example.com"), "(none)"); + EXPECT_EQ(resolved("javascript:alert(1)"), "(none)"); + EXPECT_EQ(resolved("ftp://example.com/"), "(none)"); + EXPECT_EQ(resolved("data:text/html,x"), "(none)"); +} + +TEST(ResolveLink, TakesNoUserInformation) +{ + EXPECT_EQ(resolved("http://user:secret@example.com/"), "(none)"); +} + +TEST(ResolveLink, TakesNothingThatDoesNotParse) +{ + EXPECT_EQ(resolved("http://exa mple.com/"), "(none)"); +} diff --git a/test/robots_test.cpp b/test/robots_test.cpp new file mode 100644 index 0000000..4b01a36 --- /dev/null +++ b/test/robots_test.cpp @@ -0,0 +1,159 @@ +#include "robots.hpp" + +#include + +using crawler::Robots; +using namespace std::chrono_literals; + +TEST(Robots, AllowsEverythingByDefault) +{ + EXPECT_TRUE(Robots{}.allowed("/anything")); + EXPECT_TRUE(Robots::parse("", "crawler").allowed("/anything")); +} + +TEST(Robots, DisallowAllAllowsOnlyItself) +{ + EXPECT_FALSE(Robots::disallow_all().allowed("/")); + EXPECT_TRUE(Robots::disallow_all().allowed("/robots.txt")); +} + +TEST(Robots, AppliesTheGroupForEveryone) +{ + const auto robots = Robots::parse("User-agent: *\nDisallow: /private/\n", "crawler"); + EXPECT_FALSE(robots.allowed("/private/x")); + EXPECT_TRUE(robots.allowed("/public/x")); +} + +TEST(Robots, PrefersTheGroupThatNamesTheAgentInAnyCase) +{ + const auto robots = Robots::parse("User-agent: *\nDisallow: /\n\n" + "User-Agent: CRAWLER\nDisallow: /mine/\n", + "crawler"); + EXPECT_TRUE(robots.allowed("/x")); + EXPECT_FALSE(robots.allowed("/mine/x")); +} + +TEST(Robots, IgnoresGroupsForOthers) +{ + EXPECT_TRUE(Robots::parse("User-agent: other\nDisallow: /\n", "crawler").allowed("/x")); +} + +TEST(Robots, MergesTheGroupsOfAnAgent) +{ + const auto robots = Robots::parse("User-agent: crawler\nDisallow: /a\n\n" + "User-agent: other\nDisallow: /b\n\n" + "User-agent: crawler\nDisallow: /c\n", + "crawler"); + EXPECT_FALSE(robots.allowed("/a")); + EXPECT_TRUE(robots.allowed("/b")); + EXPECT_FALSE(robots.allowed("/c")); +} + +TEST(Robots, SharesAGroupBetweenAgents) +{ + const auto robots = + Robots::parse("User-agent: other\nUser-agent: crawler\nDisallow: /x\n", "crawler"); + EXPECT_FALSE(robots.allowed("/x")); +} + +TEST(Robots, LetsTheLongestRuleDecide) +{ + const auto robots = + Robots::parse("User-agent: *\nDisallow: /a/\nAllow: /a/b/\nDisallow: /a/b/c\n", "crawler"); + EXPECT_FALSE(robots.allowed("/a/x")); + EXPECT_TRUE(robots.allowed("/a/b/x")); + EXPECT_FALSE(robots.allowed("/a/b/c")); +} + +TEST(Robots, PrefersAllowBetweenRulesOfTheSameLength) +{ + EXPECT_TRUE(Robots::parse("User-agent: *\nDisallow: /a\nAllow: /a\n", "crawler").allowed("/a")); + EXPECT_TRUE(Robots::parse("User-agent: *\nAllow: /a\nDisallow: /a\n", "crawler").allowed("/a")); +} + +TEST(Robots, AnEmptyDisallowDisallowsNothing) +{ + EXPECT_TRUE(Robots::parse("User-agent: *\nDisallow:\n", "crawler").allowed("/x")); +} + +TEST(Robots, UnderstandsWildcards) +{ + const auto robots = Robots::parse("User-agent: *\nDisallow: /*.php\n", "crawler"); + EXPECT_FALSE(robots.allowed("/a.php")); + EXPECT_FALSE(robots.allowed("/a/b.php?x=1")); + EXPECT_TRUE(robots.allowed("/a.html")); +} + +TEST(Robots, UnderstandsTheEndAnchor) +{ + const auto robots = + Robots::parse("User-agent: *\nDisallow: /*.php$\nDisallow: /exact$\n", "crawler"); + EXPECT_FALSE(robots.allowed("/a.php")); + EXPECT_FALSE(robots.allowed("/a.php.php")); + EXPECT_TRUE(robots.allowed("/a.php.bak")); + EXPECT_FALSE(robots.allowed("/exact")); + EXPECT_TRUE(robots.allowed("/exactly")); +} + +TEST(Robots, ComparesPathsWithCase) +{ + const auto robots = Robots::parse("User-agent: *\nDisallow: /Private\n", "crawler"); + EXPECT_FALSE(robots.allowed("/Private")); + EXPECT_TRUE(robots.allowed("/private")); +} + +TEST(Robots, ChecksTheQueryToo) +{ + const auto robots = Robots::parse("User-agent: *\nDisallow: /*?session=\n", "crawler"); + EXPECT_FALSE(robots.allowed("/a?session=1")); + EXPECT_TRUE(robots.allowed("/a?page=1")); +} + +TEST(Robots, SkipsCommentsAndNoise) +{ + const auto robots = Robots::parse("# about us\r\n" + "User-agent: * # everyone\r\n" + "Sitemap: https://example.com/sitemap.xml\r\n" + "what is this\r\n" + "Disallow: /x # not there\r\n", + "crawler"); + EXPECT_FALSE(robots.allowed("/x")); + EXPECT_TRUE(robots.allowed("/y")); +} + +TEST(Robots, IgnoresRulesAheadOfAnyGroup) +{ + EXPECT_TRUE(Robots::parse("Disallow: /\nUser-agent: *\nAllow: /x\n", "crawler").allowed("/y")); +} + +TEST(Robots, AlwaysAllowsItself) +{ + EXPECT_TRUE(Robots::parse("User-agent: *\nDisallow: /\n", "crawler").allowed("/robots.txt")); +} + +TEST(Robots, ReadsTheCrawlDelay) +{ + EXPECT_EQ(Robots::parse("User-agent: *\nCrawl-delay: 2\n", "crawler").crawl_delay(), 2s); + EXPECT_EQ(Robots::parse("User-agent: *\nCrawl-delay: 0.5\n", "crawler").crawl_delay(), 500ms); + EXPECT_FALSE(Robots::parse("User-agent: *\nDisallow: /\n", "crawler").crawl_delay()); +} + +TEST(Robots, IgnoresACrawlDelayThatMakesNoSense) +{ + EXPECT_FALSE(Robots::parse("User-agent: *\nCrawl-delay: soon\n", "crawler").crawl_delay()); + EXPECT_FALSE(Robots::parse("User-agent: *\nCrawl-delay: -1\n", "crawler").crawl_delay()); + EXPECT_FALSE(Robots::parse("User-agent: *\nCrawl-delay: inf\n", "crawler").crawl_delay()); +} + +TEST(Robots, KeepsAHugeCrawlDelayToADay) +{ + EXPECT_EQ(Robots::parse("User-agent: *\nCrawl-delay: 1e30\n", "crawler").crawl_delay(), 24h); +} + +TEST(Robots, TakesTheCrawlDelayOfTheChosenGroupOnly) +{ + const auto robots = Robots::parse("User-agent: other\nCrawl-delay: 9\n\n" + "User-agent: crawler\nCrawl-delay: 1\n", + "crawler"); + EXPECT_EQ(robots.crawl_delay(), 1s); +} diff --git a/test/routes_test.cpp b/test/routes_test.cpp new file mode 100644 index 0000000..9aa28d9 --- /dev/null +++ b/test/routes_test.cpp @@ -0,0 +1,273 @@ +// +// What the records of a name mean for how to connect, without any I/O. +// +#include "routes.hpp" + +#include + +#include +#include + +using namespace crawler; +using testing::ElementsAre; +using testing::HasSubstr; +using testing::IsEmpty; + +namespace +{ + +Origin origin(std::string_view url) { return *origin_of(boost::urls::url(url)); } + +c_ares_async::https_record record(unsigned short priority, std::string target, + std::vector params = {}) +{ + return {.priority = priority, .target = std::move(target), .params = std::move(params)}; +} + +ServiceRecords found(std::string owner, std::vector records, + bool aliased = false) +{ + return {.owner = std::move(owner), .aliased = aliased, .records = std::move(records)}; +} + +std::vector routes(const Plan& plan) +{ + std::vector result; + for (const auto& route : plan.routes) + result.push_back(route.to_string()); + return result; +} + +} // namespace + +// ================================================================================================= + +TEST(Origin, TakesThePortOfTheScheme) +{ + EXPECT_EQ(origin("http://example.com/x").port, 80); + EXPECT_EQ(origin("https://example.com/x").port, 443); + EXPECT_EQ(origin("https://example.com:8443/x").port, 8443); +} + +TEST(Origin, ComparesHostsTheWayDnsDoes) +{ + EXPECT_EQ(origin("https://EXAMPLE.com./").host, "example.com"); +} + +TEST(Origin, KnowsAnAddressFromAName) +{ + EXPECT_FALSE(origin("http://example.com/").is_address); + EXPECT_TRUE(origin("http://127.0.0.1/").is_address); + EXPECT_TRUE(origin("http://[::1]/").is_address); +} + +TEST(Origin, HasNoneForOtherSchemes) +{ + EXPECT_FALSE(origin_of(boost::urls::url("ftp://example.com/"))); + EXPECT_FALSE(origin_of(boost::urls::url("mailto:someone@example.com"))); +} + +TEST(QueryName, IsTheHostOnTheDefaultPort) +{ + EXPECT_EQ(https_query_name(origin("https://example.com/")), "example.com"); + EXPECT_EQ(https_query_name(origin("https://example.com:443/")), "example.com"); +} + +TEST(QueryName, HasAPortPrefixOnAnyOther) +{ + EXPECT_EQ(https_query_name(origin("https://example.com:8443/")), "_8443._https.example.com"); +} + +TEST(QueryName, IsThatOfTheHttpsFormForHttp) +{ + EXPECT_EQ(https_query_name(origin("http://example.com/")), "example.com"); + EXPECT_EQ(https_query_name(origin("http://example.com:80/")), "example.com"); + EXPECT_EQ(https_query_name(origin("http://example.com:8080/")), "_8080._https.example.com"); +} + +// ------------------------------------------------------------------------------------------------- + +TEST(Plan, ConnectsOverHttp3WhereTheRecordOffersIt) +{ + const auto plan = plan_routes(origin("https://example.com/"), + found("example.com", {record(1, ".", {"alpn=h3,h2"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443")); + EXPECT_EQ(plan.origin.scheme, "https"); +} + +TEST(Plan, GoesToThePortOfTheRecord) +{ + const auto plan = plan_routes(origin("https://example.com/"), + found("example.com", {record(1, ".", {"alpn=h3", "port=8443"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:8443")); + EXPECT_EQ(plan.origin.port, 443) << "requests still go to the origin"; +} + +TEST(Plan, TriesTheLowestPriorityFirst) +{ + const auto plan = plan_routes(origin("https://example.com/"), + found("example.com", {record(3, ".", {"alpn=h3", "port=3"}), + record(1, ".", {"alpn=h3", "port=1"}), + record(2, ".", {"alpn=h3", "port=2"})})); + + EXPECT_THAT(routes(plan), + ElementsAre("h3 example.com:1", "h3 example.com:2", "h3 example.com:3")); +} + +TEST(Plan, RejectsARecordWithoutHttp3) +{ + const auto plan = plan_routes(origin("https://example.com/"), + found("example.com", {record(1, ".", {"alpn=h2"}), + record(2, ".", {"alpn=h3", "port=8443"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:8443")); + ASSERT_EQ(plan.rejected.size(), 1u); + EXPECT_THAT(plan.rejected[0], HasSubstr("alpn=h2")); + EXPECT_THAT(plan.rejected[0], HasSubstr("TLS over TCP")); +} + +TEST(Plan, HasNoRouteToAnHttpsOriginWithoutARecord) +{ + const auto plan = plan_routes(origin("https://example.com/"), found("example.com", {})); + + EXPECT_THAT(plan.routes, IsEmpty()); + ASSERT_EQ(plan.rejected.size(), 1u); + EXPECT_THAT(plan.rejected[0], HasSubstr("TLS over TCP")); +} + +TEST(Plan, RejectsATargetOtherThanTheOrigin) +{ + // + // The certificate has to be valid for the origin (section 9.1), and anyhttp would check it + // against what it connects to. + // + const auto plan = plan_routes(origin("https://example.com/"), + found("example.com", {record(1, "cdn.example.net", {"alpn=h3"})})); + + EXPECT_THAT(plan.routes, IsEmpty()); + ASSERT_EQ(plan.rejected.size(), 1u); + EXPECT_THAT(plan.rejected[0], HasSubstr("cdn.example.net")); +} + +TEST(Plan, TakesAnExplicitTargetThatIsTheOrigin) +{ + const auto plan = + plan_routes(origin("https://example.com:8443/"), + found("_8443._https.example.com", {record(1, "EXAMPLE.com.", {"alpn=h3"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:8443")); +} + +TEST(Plan, TakesTheOwnNameOfAPortPrefixedRecordLiterally) +{ + // + // "." is the owner name, section 2.5.2, and that includes the prefix. + // + const auto plan = plan_routes(origin("https://example.com:8443/"), + found("_8443._https.example.com", {record(1, ".", {"alpn=h3"})})); + + EXPECT_THAT(plan.routes, IsEmpty()); + ASSERT_EQ(plan.rejected.size(), 1u); + EXPECT_THAT(plan.rejected[0], HasSubstr("_8443._https.example.com")); +} + +TEST(Plan, HasNoDefaultSetToFallBackOnWithNoDefaultAlpn) +{ + const auto plan = plan_routes(origin("https://example.com/"), + found("example.com", {record(1, ".", {"no-default-alpn"})})); + + EXPECT_THAT(plan.routes, IsEmpty()); +} + +TEST(Plan, IgnoresARecordWithAMandatoryKeyItDoesNotKnow) +{ + const auto plan = + plan_routes(origin("https://example.com/"), + found("example.com", + {record(1, ".", {"mandatory=alpn,key65280", "alpn=h3", "key65280=01"})})); + + EXPECT_THAT(plan.routes, IsEmpty()); + EXPECT_THAT(plan.rejected.front(), HasSubstr("key65280")); +} + +TEST(Plan, IgnoresARecordThatRequiresEch) +{ + const auto plan = + plan_routes(origin("https://example.com/"), + found("example.com", {record(1, ".", {"mandatory=ech", "alpn=h3", "ech=dead"})})); + + EXPECT_THAT(plan.routes, IsEmpty()); +} + +TEST(Plan, AcceptsMandatoryKeysItCanLeaveAlone) +{ + const auto plan = plan_routes( + origin("https://example.com/"), + found("example.com", + {record(1, ".", {"mandatory=alpn,ipv4hint", "alpn=h3", "ipv4hint=192.0.2.1"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443")); +} + +TEST(Plan, IgnoresARecordWithAnInvalidPort) +{ + const auto plan = plan_routes(origin("https://example.com/"), + found("example.com", {record(1, ".", {"alpn=h3", "port=x"})})); + + EXPECT_THAT(plan.routes, IsEmpty()); +} + +// ------------------------------------------------------------------------------------------------- + +TEST(Plan, ConnectsInTheClearToAnHttpOriginWithoutARecord) +{ + const auto plan = + plan_routes(origin("http://example.com:8080/"), found("_8080._https.example.com", {})); + + EXPECT_THAT(routes(plan), ElementsAre("h1 example.com:8080")); + EXPECT_EQ(plan.origin.scheme, "http"); +} + +TEST(Plan, UpgradesAnHttpOriginWithARecord) +{ + const auto plan = plan_routes(origin("http://example.com/"), + found("example.com", {record(1, ".", {"alpn=h3"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443")); + EXPECT_EQ(plan.origin.scheme, "https"); + EXPECT_EQ(plan.origin.port, 443); +} + +TEST(Plan, DoesNotFallBackToCleartextAfterAnUpgrade) +{ + const auto plan = plan_routes(origin("http://example.com/"), + found("example.com", {record(1, ".", {"alpn=h2"})})); + + EXPECT_THAT(plan.routes, IsEmpty()); + EXPECT_EQ(plan.origin.scheme, "https"); +} + +TEST(Plan, DoesNotUpgradeForAnIncompatibleRecord) +{ + // + // Section 9.5: only a compatible record, or an alias. + // + const auto plan = plan_routes( + origin("http://example.com/"), + found("example.com", {record(1, ".", {"mandatory=key65280", "alpn=h3", "key65280=01"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h1 example.com:80")); + EXPECT_EQ(plan.origin.scheme, "http"); +} + +TEST(Plan, UpgradesAnHttpOriginWithAnAlias) +{ + const auto plan = plan_routes(origin("http://example.com/"), + found("alias.example.net", {}, /* aliased */ true)); + + EXPECT_THAT(plan.routes, IsEmpty()); + EXPECT_EQ(plan.origin.scheme, "https"); + EXPECT_THAT(plan.rejected.front(), HasSubstr("alias.example.net")); +} diff --git a/test/site.cpp b/test/site.cpp new file mode 100644 index 0000000..2df36ac --- /dev/null +++ b/test/site.cpp @@ -0,0 +1,103 @@ +#include "site.hpp" + +#include + +#include "routes.hpp" + +#include +#include +#include +#include +#include + +namespace http = boost::beast::http; + +namespace crawler::test +{ + +// ================================================================================================= + +Site::Site(const anyhttp::Executor& executor, const std::string& listen_address) + : requests_(std::make_shared>()) +{ + auto directory = (std::filesystem::temp_directory_path() / "crawler-test-XXXXXX").string(); + if (!mkdtemp(directory.data())) + throw std::runtime_error("mkdtemp"); + directory_ = directory; + certificate_file_ = directory_ / "certificate.pem"; + key_file_ = directory_ / "key.pem"; + + const auto certificate = make_test_certificate("localhost"); + std::ofstream(certificate_file_) << certificate.certificate_pem; + std::ofstream(key_file_) << certificate.key_pem; + + server_.emplace(executor, anyhttp::server::Config{.listen_address = listen_address, + .port = 0, + .tls_certificate_chain = certificate_file_, + .tls_private_key = key_file_}); + port_ = server_->local_endpoint().port(); + + server_->on_request([this, requests = requests_]( + anyhttp::server::Request request, + anyhttp::server::Response response) -> anyhttp::Task { + requests->push_back(Request{ + .target = std::string(request.url().encoded_target()), + .authority = std::string(request.url().encoded_host_and_port()), + .user_agent = std::string(request.fields()[http::field::user_agent]), + .time = std::chrono::steady_clock::now(), + }); + + std::array buffer; + for (;;) + if (auto [ec, n] = co_await request.read_some(boost::asio::buffer(buffer)); ec || n == 0) + break; + + const auto it = pages_.find(std::string(request.url().encoded_path())); + const auto page = it != pages_.end() ? it->second : Page{.status = 404}; + + anyhttp::Fields fields; + fields.set(http::field::content_type, page.content_type); + if (!page.location.empty()) + fields.set(http::field::location, page.location); + + if (auto [ec] = co_await response.submit(page.status, std::move(fields)); ec) + co_return; + co_await response.write_eof(boost::asio::buffer(page.body)); + }); + + dns_.emplace(executor); +} + +Site::~Site() +{ + std::error_code ignored; + std::filesystem::remove_all(directory_, ignored); +} + +std::vector Site::targets() const +{ + std::vector targets; + for (const auto& request : *requests_) + targets.push_back(request.target); + return targets; +} + +std::size_t Site::dns_queries(std::string_view name) const +{ + const auto& queries = dns_ ? dns_->queries() : queries_; + return std::ranges::count_if(queries, [&](const auto& query) { + return normalize_host(query.name) == normalize_host(name); + }); +} + +void Site::stop() +{ + server_.reset(); + if (dns_) + queries_ = dns_->queries(); + dns_.reset(); +} + +// ================================================================================================= + +} // namespace crawler::test diff --git a/test/site.hpp b/test/site.hpp new file mode 100644 index 0000000..ff68a80 --- /dev/null +++ b/test/site.hpp @@ -0,0 +1,87 @@ +// +// A web site with DNS of its own, all on loopback, for a crawler to be pointed at: an anyhttp +// server on one port, for every protocol, and a DNS server with whatever zone the test wants. The +// server has a certificate for "localhost", the one name that also resolves without our DNS server, +// which anyhttp's own resolver needs for HTTP/3. +// +#pragma once + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace crawler::test +{ + +// ================================================================================================= + +class Site +{ +public: + struct Page + { + unsigned status = 200; + std::string body; + std::string content_type = "text/html"; + std::string location; + }; + + /// One request that arrived. + struct Request + { + std::string target; + std::string authority; + std::string user_agent; + std::chrono::steady_clock::time_point time; + }; + + /// Listens on \p listen_address: all of them by default, "::" taking IPv4 as well, as + /// "localhost" may be either. + explicit Site(const anyhttp::Executor& executor, const std::string& listen_address = "::"); + ~Site(); + + /// A path that has no page gets a 404. + void page(std::string path, Page page) { pages_[std::move(path)] = std::move(page); } + + std::uint16_t port() const { return port_; } + const std::filesystem::path& ca_file() const { return certificate_file_; } + + /// The zone. Good until stop(), as opposed to what was asked of it, see dns_queries(). + fake_dns_server& dns() { return *dns_; } + std::string dns_address() const { return dns_->address(); } + + /// How often the DNS server was asked about \p name, also after stop(). + std::size_t dns_queries(std::string_view name) const; + + /// Everything the server was asked so far, in order, and all of the targets alone. + const std::vector& requests() const { return *requests_; } + std::vector targets() const; + + /// Ends the servers, which is what lets the context run dry. + void stop(); + +private: + std::filesystem::path directory_; + std::filesystem::path certificate_file_; + std::filesystem::path key_file_; + std::map pages_; + std::shared_ptr> requests_; + std::optional server_; + std::optional dns_; + std::vector queries_; // what the DNS server knew when it was stopped + std::uint16_t port_ = 0; +}; + +// ================================================================================================= + +} // namespace crawler::test From 6ca7fc992fc5ed4859cbecc36de9340a83a24d6b Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Mon, 2 Nov 2026 21:34:25 +0000 Subject: [PATCH 02/27] docs: describe what the crawler does, and how its code is laid out The README had it not crawling anything yet. It now says what happens for each kind of host, and why, given what anyhttp's client can do so far. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 6 ++++++ README.md | 44 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index d800e22..81bfa4c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,6 +16,12 @@ These hold for all code in this repository, including tests and experiments. - **Run tests and programs under `timeout -s KILL`**, 5 seconds per test. Never let a suite sit on a hang. +## The code + +[routes.cpp](src/routes.cpp) says what the HTTPS records of a name mean for how to connect, without +any I/O, and [crawler.cpp](src/crawler.cpp) does the rest. The tests in [test/](test) run against +[Site](test/site.hpp), a real anyhttp server and a DNS server with a zone of its own, on loopback. + ## The libraries - Changes to a library belong in that library's repository and session, with a test in its suite. diff --git a/README.md b/README.md index c35cf33..2e60d42 100644 --- a/README.md +++ b/README.md @@ -3,14 +3,52 @@ A simple web crawler on [anyhttp](https://github.com/pgit/anyhttp) (HTTP/1.1, HTTP/2 and HTTP/3) and [c-ares-async](https://github.com/pgit/c-ares-async) (DNS). -**It does not crawl anything yet.** The `crawler` program sets up both libraries on one event loop -and exits, which shows that they build and link into one process. - > [!WARNING] > This repository is purely experimental. It is not meant for production use, and anything in it > can change or break at any time, without notice. +## What it does + +```sh +crawler --allow example.com https://example.com/ +``` + +It starts from the URLs given and follows links, one request at a time, breadth first. It refuses to +run without `--allow`, and only ever connects to the hosts named there, by their exact names. Each +host gets its `robots.txt` read first, and requests to it are at least `--delay-ms` apart, or further +if its `Crawl-delay` says so. `crawler` without arguments lists the options. + +How to connect is up to the host's HTTPS records in DNS ([RFC 9460](https://www.rfc-editor.org/rfc/rfc9460)): +aliases are followed, a record has to be compatible, and the lowest priority goes first. The port +comes from the record, and an `http://` URL with a record is treated as `https://`, without going +back to cleartext. + +What can be used of that is limited by what anyhttp's client does so far: + +| The host has | What happens | +|-------------------------------------------|-----------------------------------------------| +| a record with `h3` in its ALPN set | HTTP/3, to the port of the record | +| a record without `h3` | rejected: needs TLS over TCP | +| a record with a target that is not itself | rejected: can't verify one name and connect to another | +| no record, on an `https://` URL | rejected: needs TLS over TCP | +| no record, on an `http://` URL | HTTP/1.1 in the clear, to the addresses from c-ares | + +Every rejection is printed with its reason. Over HTTP/3, the address comes from anyhttp's own resolver +and not from c-ares, so `--dns-server` and the address hints in the record do not apply to it. + +The resolver in the devcontainer may answer with no data for HTTPS records, even where they exist +(see c-ares-async's [example](https://github.com/pgit/c-ares-async/blob/master/example/resolve/README.md)). +`--dns-server` is what gets around that. + +To see it work without going anywhere, `demo_site` runs a site and a DNS server on loopback, and +prints the command to crawl them with: + +```sh +./build/test/demo_site +``` + + ## Build C++23 and CMake 3.30 or later. The devcontainer in [.devcontainer](.devcontainer) has the compilers From 08807b5574b2133932946bb1735e111cfdd2d1c9 Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Mon, 2 Nov 2026 22:19:52 +0000 Subject: [PATCH 03/27] refactor: stop crawling, and request each host once instead What is wanted is DNS and a single request: https:/// for each host, with the status and the rest of what comes back. So the links, robots.txt, the pacing and the allowlist go, and so do http:// and port prefixes, as the port is always 443. The HTTPS records still decide how to connect. Redirects are reported and not followed, so the only names ever connected to are the ones given. Those can now come from a file, with --file, and c-ares-async's lists of hosts are fine for that, which CLAUDE.md now says. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 20 +- CMakeLists.txt | 6 +- README.md | 24 +- src/crawler.cpp | 461 ---------------------------- src/crawler.hpp | 142 --------- src/links.cpp | 188 ------------ src/links.hpp | 40 --- src/main.cpp | 106 ++++--- src/prober.cpp | 254 ++++++++++++++++ src/prober.hpp | 105 +++++++ src/robots.cpp | 189 ------------ src/robots.hpp | 53 ---- src/routes.cpp | 110 ++----- src/routes.hpp | 57 +--- test/CMakeLists.txt | 21 +- test/crawl_test.cpp | 698 ------------------------------------------- test/demo_site.cpp | 15 +- test/links_test.cpp | 122 -------- test/prober_test.cpp | 418 ++++++++++++++++++++++++++ test/robots_test.cpp | 159 ---------- test/routes_test.cpp | 175 +++-------- test/site.cpp | 8 +- test/site.hpp | 4 +- 23 files changed, 964 insertions(+), 2411 deletions(-) delete mode 100644 src/crawler.cpp delete mode 100644 src/crawler.hpp delete mode 100644 src/links.cpp delete mode 100644 src/links.hpp create mode 100644 src/prober.cpp create mode 100644 src/prober.hpp delete mode 100644 src/robots.cpp delete mode 100644 src/robots.hpp delete mode 100644 test/crawl_test.cpp delete mode 100644 test/links_test.cpp create mode 100644 test/prober_test.cpp delete mode 100644 test/robots_test.cpp diff --git a/CLAUDE.md b/CLAUDE.md index 81bfa4c..6068665 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,25 +1,27 @@ # Working on the crawler -A web crawler on two libraries of the same author, each in a repository of its own: anyhttp for HTTP -and c-ares-async for DNS. [README.md](README.md) has the presets and the build commands. +Despite the name it does not crawl: it requests `https:///` once per host and reports what +came back, connecting the way the host's HTTPS records say. It is built on two libraries of the same +author, each in a repository of its own: anyhttp for HTTP and c-ares-async for DNS. +[README.md](README.md) has the presets and the build commands. ## Safety rules These hold for all code in this repository, including tests and experiments. -- **Never open, fetch or connect to anything in c-ares-async's `url_list.txt`, and never use it as - input.** It is for DNS lookups only: 1000 mixed sites, some adult or spam-looking. The crawler is - the first program here that could turn those names into connections. -- **Tests crawl local servers on loopback only.** -- **Once the program crawls, it refuses to run without an explicit host allowlist.** robots.txt and - per-host rate limits are built in from the start, not added later. +- **One request per host, and nothing more.** c-ares-async's lists of hosts are fine as input here, + which is what makes this rule matter: they have 1000 mixed sites, some adult or spam-looking. So + nothing is followed: no links and no redirects. +- **Tests connect to local servers on loopback only.** +- **If the program ever crawls, it refuses to run without an explicit host allowlist,** with + robots.txt and per-host rate limits built in from the start, not added later. - **Run tests and programs under `timeout -s KILL`**, 5 seconds per test. Never let a suite sit on a hang. ## The code [routes.cpp](src/routes.cpp) says what the HTTPS records of a name mean for how to connect, without -any I/O, and [crawler.cpp](src/crawler.cpp) does the rest. The tests in [test/](test) run against +any I/O, and [prober.cpp](src/prober.cpp) does the rest. The tests in [test/](test) run against [Site](test/site.hpp), a real anyhttp server and a DNS server with a zone of its own, on loopback. ## The libraries diff --git a/CMakeLists.txt b/CMakeLists.txt index 4f7ef46..522e265 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.30) -project(crawler VERSION 0.0.1 DESCRIPTION "A web crawler on anyhttp and c-ares-async" LANGUAGES C CXX) +project(crawler VERSION 0.0.1 DESCRIPTION "Requests hosts the way their HTTPS records say, on anyhttp and c-ares-async" LANGUAGES C CXX) set(CMAKE_CXX_STANDARD 23) set(CMAKE_CXX_STANDARD_REQUIRED On) @@ -56,12 +56,12 @@ FetchContent_MakeAvailable(anyhttp c_ares_async) # -------------------------------------------------------------------------------------------------- # -# The crawler in a library of its own, for the tests to link. The warnings are not for the sources +# The program in a library of its own, for the tests to link. The warnings are not for the sources # from c-ares-async that the tests borrow, see test/CMakeLists.txt. # set(crawler_warnings -Wall -Wextra -Werror -Wno-missing-designated-field-initializers) -add_library(crawler_core STATIC src/crawler.cpp src/links.cpp src/robots.cpp src/routes.cpp) +add_library(crawler_core STATIC src/prober.cpp src/routes.cpp) target_include_directories(crawler_core PUBLIC src) target_link_libraries(crawler_core PUBLIC anyhttp::anyhttp c_ares_async::c_ares_async) target_compile_options(crawler_core PRIVATE ${crawler_warnings}) diff --git a/README.md b/README.md index 2e60d42..d6454cd 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # crawler -A simple web crawler on [anyhttp](https://github.com/pgit/anyhttp) (HTTP/1.1, HTTP/2 and HTTP/3) and -[c-ares-async](https://github.com/pgit/c-ares-async) (DNS). +Requests hosts the way their HTTPS records say, on [anyhttp](https://github.com/pgit/anyhttp) (HTTP) and +[c-ares-async](https://github.com/pgit/c-ares-async) (DNS). Despite the name, it does not crawl. > [!WARNING] > This repository is purely experimental. It is not meant for production use, and anything in it @@ -11,18 +11,19 @@ A simple web crawler on [anyhttp](https://github.com/pgit/anyhttp) (HTTP/1.1, HT ## What it does ```sh -crawler --allow example.com https://example.com/ +crawler example.com www.example.org +crawler --file hostnames.txt ``` -It starts from the URLs given and follows links, one request at a time, breadth first. It refuses to -run without `--allow`, and only ever connects to the hosts named there, by their exact names. Each -host gets its `robots.txt` read first, and requests to it are at least `--delay-ms` apart, or further -if its `Crawl-delay` says so. `crawler` without arguments lists the options. +It requests `https:///` once for each host, in turn, and prints the status, the route, the time +it took and the header fields of the response, along with the host's HTTPS records. A redirect is +reported and not followed, and nothing else on the host is requested, `robots.txt` included. +`crawler` without arguments lists the options. The exit code is 0 if every host answered, whatever +the status. How to connect is up to the host's HTTPS records in DNS ([RFC 9460](https://www.rfc-editor.org/rfc/rfc9460)): aliases are followed, a record has to be compatible, and the lowest priority goes first. The port -comes from the record, and an `http://` URL with a record is treated as `https://`, without going -back to cleartext. +comes from the record. What can be used of that is limited by what anyhttp's client does so far: @@ -31,8 +32,7 @@ What can be used of that is limited by what anyhttp's client does so far: | a record with `h3` in its ALPN set | HTTP/3, to the port of the record | | a record without `h3` | rejected: needs TLS over TCP | | a record with a target that is not itself | rejected: can't verify one name and connect to another | -| no record, on an `https://` URL | rejected: needs TLS over TCP | -| no record, on an `http://` URL | HTTP/1.1 in the clear, to the addresses from c-ares | +| no record | rejected: needs TLS over TCP | Every rejection is printed with its reason. Over HTTP/3, the address comes from anyhttp's own resolver and not from c-ares, so `--dns-server` and the address hints in the record do not apply to it. @@ -42,7 +42,7 @@ The resolver in the devcontainer may answer with no data for HTTPS records, even `--dns-server` is what gets around that. To see it work without going anywhere, `demo_site` runs a site and a DNS server on loopback, and -prints the command to crawl them with: +prints the command to request it with: ```sh ./build/test/demo_site diff --git a/src/crawler.cpp b/src/crawler.cpp deleted file mode 100644 index e17744c..0000000 --- a/src/crawler.cpp +++ /dev/null @@ -1,461 +0,0 @@ -#include "crawler.hpp" - -#include "links.hpp" - -#include - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace http = boost::beast::http; - -namespace crawler -{ - -namespace -{ - -// ================================================================================================= - -constexpr std::string_view agent_token = "crawler"; // what a robots.txt addresses us by -constexpr std::string_view user_agent = "crawler/0.0.1 (+https://github.com/pgit/crawler)"; - -constexpr int max_alias_chain = 8; -constexpr int max_robots_redirects = 5; // RFC 9309, section 2.3.1.2 -constexpr std::size_t max_frontier = 100'000; - -bool is_html(std::string_view content_type) -{ - std::string type(content_type.substr(0, content_type.find(';'))); - std::ranges::transform(type, type.begin(), [](unsigned char c) { return std::tolower(c); }); - return type.empty() || type.starts_with("text/html") || - type.starts_with("application/xhtml+xml"); -} - -bool is_redirect(unsigned status) -{ - return status == 301 || status == 302 || status == 303 || status == 307 || status == 308; -} - -// ================================================================================================= - -} // namespace - -Crawler::Crawler(anyhttp::Executor executor, Config config, Observer observer) - : executor_(executor), config_(std::move(config)), observer_(std::move(observer)), - resolver_(std::move(executor)) -{ - if (config_.allowed_hosts.empty()) - throw std::invalid_argument("refusing to crawl without an allowlist of hosts"); - - for (const auto& host : config_.allowed_hosts) - { - if (host.empty() || host.find('*') != host.npos) - throw std::invalid_argument( - std::format("'{}' is not a host: the allowlist takes exact names", host)); - allowed_hosts_.insert(normalize_host(host)); - } - - if (config_.seeds.empty()) - throw std::invalid_argument("nothing to crawl: no URLs given"); - - if (!config_.dns_server.empty()) - if (const auto ec = resolver_.set_servers(config_.dns_server)) - throw std::invalid_argument( - std::format("invalid DNS server '{}': {}", config_.dns_server, ec.message())); - - for (const auto& seed : config_.seeds) - { - const auto parsed = boost::urls::parse_uri(seed); - const auto url = parsed ? resolve_link(*parsed, seed) : std::nullopt; - if (!url) - throw std::invalid_argument(std::format("'{}' is not an http or https URL", seed)); - if (!is_allowed(url->host_address())) - throw std::invalid_argument( - std::format("'{}' is not on an allowed host: '{}'", seed, url->host_address())); - if (seen_.insert(std::string(url->buffer())).second) - frontier_.push_back(*url); - } -} - -// ------------------------------------------------------------------------------------------------- - -bool Crawler::is_allowed(std::string_view host) const -{ - return allowed_hosts_.contains(normalize_host(host)); -} - -void Crawler::emit(Event event) const -{ - if (observer_) - observer_(event); -} - -void Crawler::enqueue(boost::urls::url url) -{ - if (!seen_.insert(std::string(url.buffer())).second) - return; - - if (!is_allowed(url.host_address())) - emit({.kind = Event::Kind::outside_allowlist, .url = std::string(url.buffer())}); - else if (frontier_.size() < max_frontier) - frontier_.push_back(std::move(url)); -} - -// ------------------------------------------------------------------------------------------------- - -// -// SVCB resolution, section 3 of RFC 9460, up to the point where it would take addresses. A lookup -// that fails is not fatal: DNS is not protected here, and this is the choice section 3.1 leaves -// for that. A name without a record is the common case and not even worth a note. -// -anyhttp::Task Crawler::resolve_service(const Origin& origin) -{ - ServiceRecords found{.owner = https_query_name(origin)}; - std::set visited{found.owner}; - - for (int hop = 0; hop < max_alias_chain; ++hop) - { - auto [ec, records] = co_await resolver_.resolve_https(found.owner); - if (ec) - { - if (ec != c_ares_async::make_ares_error(ARES_ENODATA) && - ec != c_ares_async::make_ares_error(ARES_ENOTFOUND)) - found.notes.push_back( - std::format("HTTPS lookup of '{}': {}", found.owner, ec.message())); - co_return found; - } - - // - // A record in AliasMode makes the rest of the set irrelevant (section 2.4.1). - // - const auto alias = std::ranges::find(records, 0, &c_ares_async::https_record::priority); - if (alias == records.end()) - { - found.records = std::move(records); - co_return found; - } - - if (alias->target.empty() || alias->target == ".") - { - // - // "The service is not available" (section 2.5.1), which is advisory. Treated like no - // record at all, what comes of that is the connection attempt's to find out. - // - found.notes.push_back(std::format("'{}' says the service is not available", found.owner)); - found.aliased = false; - co_return found; - } - - found.aliased = true; - found.owner = normalize_host(alias->target); - if (!visited.insert(found.owner).second) - break; - } - - // - // Section 3.1: as if the name had no record at all. - // - co_return ServiceRecords{.owner = https_query_name(origin), - .notes = {std::format("alias chain of '{}' is too long or has a loop", - https_query_name(origin))}}; -} - -anyhttp::Task Crawler::plan_for(const Origin& origin) -{ - const auto key = origin.key(); - if (auto it = plans_.find(key); it != plans_.end()) - co_return &it->second; - - auto found = - origin.is_address ? ServiceRecords{.owner = origin.host} : co_await resolve_service(origin); - - auto plan = plan_routes(origin, found); - for (const auto& route : plan.routes) - emit({.kind = Event::Kind::note, - .url = key, - .route = route.to_string(), - .detail = route.source}); - for (const auto& rejected : plan.rejected) - emit({.kind = Event::Kind::note, .url = key, .detail = rejected}); - - co_return &plans_.emplace(key, std::move(plan)).first->second; -} - -// -// RFC 9309, section 2.3.1: anything in the 400s means there are no rules, anything else that is not -// a robots.txt means that we can not know them, and then nothing is allowed. That goes for a -// redirect to another origin as well, which the RFC would have us follow. It is not worth the -// complication to do that while we can not connect to most of them. -// -anyhttp::Task Crawler::robots_for(Plan& plan) -{ - const auto key = plan.origin.key(); - if (auto it = robots_.find(key); it != robots_.end()) - co_return &it->second; - - auto url = boost::urls::url(std::format("{}/robots.txt", key)); - Robots robots = Robots::disallow_all(); - std::string problem; - - for (int redirects = 0;; ++redirects) - { - const auto response = co_await fetch(plan, url); - if (response.ec) - { - problem = response.error; - break; - } - - const auto status = response.message.result_int(); - if (status >= 200 && status < 300) - robots = Robots::parse(response.message.body(), agent_token); - else if (status >= 400 && status < 500) - robots = Robots{}; - else if (is_redirect(status) && redirects < max_robots_redirects) - { - const auto target = resolve_link(url, response.message[http::field::location]); - if (target && origin_of(*target) && origin_of(*target)->key() == key) - { - url = *target; - continue; - } - problem = "redirected to another origin"; - } - else - problem = std::format("status {}", status); - break; - } - - if (!problem.empty()) - emit({.kind = Event::Kind::note, - .url = key, - .detail = - std::format("robots.txt could not be read, so nothing is allowed: {}", problem)}); - else if (const auto crawl_delay = robots.crawl_delay()) - { - auto& pacing = pacing_[plan.origin.host]; - pacing.delay = std::max(pacing.delay, *crawl_delay); - } - - co_return &robots_.emplace(key, std::move(robots)).first->second; -} - -// ------------------------------------------------------------------------------------------------- - -// -// One request, whichever route gets it through. This is the only place that sends requests, so it -// is the one to see to the spacing, and to the allowlist one last time. -// -anyhttp::Task Crawler::fetch(Plan& plan, boost::urls::url url) -{ - if (!is_allowed(plan.origin.host)) - throw std::logic_error(std::format("about to contact '{}'", plan.origin.host)); - - Response response; - response.ec = make_error_code(boost::system::errc::host_unreachable); - response.error = "no way to connect"; - - auto& pacing = pacing_[plan.origin.host]; - const auto wait_for = [&] { return pacing.next_request - std::chrono::steady_clock::now(); }; - if (wait_for().count() > 0) - co_await anyhttp::delay(wait_for()); - - for (auto route = plan.routes.begin(); route != plan.routes.end(); ++route) - { - response = co_await attempt(*route, url); - if (response.ec) - { - emit({.kind = Event::Kind::note, - .url = std::string(url.buffer()), - .route = route->to_string(), - .detail = response.error}); - continue; - } - - std::rotate(plan.routes.begin(), route, std::next(route)); // go straight there next time - break; - } - - pacing.next_request = std::chrono::steady_clock::now() + std::max(config_.delay, pacing.delay); - co_return response; -} - -anyhttp::Task Crawler::attempt(const Route& route, boost::urls::url url) -{ - anyhttp::client::Config client_config; - client_config.protocol = route.protocol; - client_config.tls_ca_file = config_.tls_ca_file; - - const auto with_route = [&](Response response) { - response.route = route.to_string(); - return response; - }; - - // - // HTTP/3 goes to the name of the route, as that is also what anyhttp verifies the certificate - // against, which leaves it to its own resolver to look up the address. - // - if (route.protocol == anyhttp::Protocol::h3) - { - client_config.url.set_scheme("https"); - client_config.url.set_host_address(route.host); - client_config.url.set_port_number(route.port); - co_return with_route(co_await exchange(std::move(client_config), std::move(url))); - } - - // - // In the clear, nothing has to be verified, so the addresses can be ours. The lookup stays - // outside of the timeout: the image has c-ares 1.34.5, and cancelling a lookup is what - // GHSA-6wfj-rwm7-3542 is about. c-ares has timeouts of its own. - // - auto [ec, endpoints] = co_await resolver_.resolve(route.host, std::to_string(route.port)); - if (ec) - co_return with_route( - Response{.ec = ec, .error = std::format("{}: {}", route.host, ec.message())}); - - Response response{.ec = make_error_code(boost::system::errc::host_unreachable), - .error = std::format("{}: no addresses", route.host)}; - for (const auto& endpoint : endpoints) - { - client_config.url.set_scheme("http"); - client_config.url.set_host_address(endpoint.address().to_string()); - client_config.url.set_port_number(endpoint.port()); - response = co_await exchange(client_config, url); - if (!response.ec) - break; - } - co_return with_route(std::move(response)); -} - -// -// Connects as told and asks for \p url, whose host is what goes into the "Host" field, whatever -// \p client_config connects to. Everything is reported rather than thrown: when_either() would -// otherwise sit out the timeout, as what throws does not count as having finished first. -// -anyhttp::Task Crawler::exchange(anyhttp::client::Config client_config, - boost::urls::url url) -{ - const auto run = [](anyhttp::Executor executor, anyhttp::client::Config client_config, - boost::urls::url url) -> anyhttp::Task { - try - { - anyhttp::client::Client client(executor, std::move(client_config)); - auto [connect_ec, session] = co_await client.connect(); - if (connect_ec) - co_return Response{.ec = connect_ec, - .error = std::format("connect: {}", connect_ec.message())}; - - anyhttp::Fields headers; - headers.set(http::field::user_agent, user_agent); - auto [get_ec, message] = co_await session.get(std::move(url), std::move(headers)); - co_return Response{.ec = get_ec, - .message = std::move(message), - .error = get_ec ? std::format("request: {}", get_ec.message()) : ""}; - } - catch (const std::exception& ex) - { - co_return Response{.ec = make_error_code(boost::system::errc::connection_aborted), - .error = ex.what()}; - } - }; - - auto result = co_await anyhttp::when_either( - run(executor_, std::move(client_config), std::move(url)), anyhttp::delay(config_.timeout)); - - if (auto* response = std::get_if<0>(&result)) - co_return std::move(*response); - - co_return Response{.ec = make_error_code(boost::system::errc::timed_out), - .error = std::format("timed out after {}", config_.timeout)}; -} - -// ------------------------------------------------------------------------------------------------- - -// -// What a response means for the rest of the crawl: more URLs, if it has any. -// -void Crawler::visit(const boost::urls::url& url, const Response& response) -{ - const auto status = response.message.result_int(); - std::size_t links = 0; - - if (is_redirect(status)) - { - if (const auto target = resolve_link(url, response.message[http::field::location])) - enqueue(*target); - } - else if (status >= 200 && status < 300 && is_html(response.message[http::field::content_type])) - { - for (const auto& href : extract_links(response.message.body())) - if (auto target = resolve_link(url, href)) - { - enqueue(std::move(*target)); - ++links; - } - } - - emit({.kind = Event::Kind::fetched, - .url = std::string(url.buffer()), - .status = static_cast(status), - .route = response.route, - .detail = std::format("{} bytes, {} links", response.message.body().size(), links)}); -} - -anyhttp::Task Crawler::run() -{ - while (!frontier_.empty() && requested_ < config_.max_pages) - { - auto url = std::move(frontier_.front()); - frontier_.pop_front(); - - auto* plan = co_await plan_for(*origin_of(url)); - - // - // The records can turn "http" into "https" (RFC 9460, section 9.5), and that may well be a - // page we have already been to, or are going to. - // - if (url.scheme() != plan->origin.scheme) - { - url.set_scheme(plan->origin.scheme); - if (url.has_port()) - url.set_port_number(plan->origin.port); - if (!seen_.insert(std::string(url.buffer())).second) - continue; - } - - const auto text = std::string(url.buffer()); - if (plan->routes.empty()) - { - emit({.kind = Event::Kind::failed, .url = text, .detail = "no way to connect"}); - continue; - } - - const auto* robots = co_await robots_for(*plan); - if (!robots->allowed(url.encoded_target())) - { - emit({.kind = Event::Kind::disallowed, .url = text}); - continue; - } - - ++requested_; - const auto response = co_await fetch(*plan, url); - if (response.ec) - emit({.kind = Event::Kind::failed, .url = text, .detail = response.error}); - else - visit(url, response); - } -} - -// ================================================================================================= - -} // namespace crawler diff --git a/src/crawler.hpp b/src/crawler.hpp deleted file mode 100644 index 9cbd94d..0000000 --- a/src/crawler.hpp +++ /dev/null @@ -1,142 +0,0 @@ -// -// The crawl: one URL at a time, breadth first, from the seeds and within the allowlist. -// -#pragma once - -#include "robots.hpp" -#include "routes.hpp" - -#include - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace crawler -{ - -// ================================================================================================= - -struct Config -{ - /// The hosts that may be connected to, exactly: no wildcards, and no subdomains implied. - std::vector allowed_hosts; - - /// Where to start. Each has to be on an allowed host. - std::vector seeds; - - /// How many pages to request at most, whatever comes of them. robots.txt does not count. - std::size_t max_pages = 100; - - /// The least time from the end of one request to the start of the next one, per host. A longer - /// "Crawl-delay" in the host's robots.txt takes precedence. - std::chrono::milliseconds delay{1000}; - - /// How long one request may take on one route, from connecting to the end of the response. - std::chrono::milliseconds timeout{10000}; - - /// The DNS servers to ask, as resolver::set_servers() takes them. Empty for those of the system. - std::string dns_server; - - /// The CA certificates to verify servers against, as a PEM file. Empty for the system's. - std::string tls_ca_file; -}; - -/// Something that happened in the course of a crawl, for whoever is watching. -struct Event -{ - enum class Kind - { - fetched, ///< a response arrived, whatever its status - failed, ///< no response could be had - disallowed, ///< robots.txt says no, or it could not be read - outside_allowlist, ///< a link or a redirect to a host that is not allowed - note, ///< something worth knowing about how a host is reached - }; - - Kind kind = Kind::note; - std::string url; - int status = 0; ///< with \c fetched - std::string route; ///< with \c fetched, and \c note where about a route - std::string detail; -}; - -/** - * Crawls, using the resolver of c-ares-async to find out how to connect (see plan_routes()). - * - * The only thing that can make this reach a host is that host being in Config::allowed_hosts: - * every seed, link and redirect is checked against it. Requests per host are spaced by - * Config::delay and the host's robots.txt is honoured -- a robots.txt that can not be read - * stops the crawl of its origin. - * - * There is one request at a time and a new connection for each of them. - */ -class Crawler -{ -public: - using Observer = std::function; - - /// Throws \c std::invalid_argument for a configuration that is not good to go, starting with - /// one that has no allowlist, and does nothing else until run(). - Crawler(anyhttp::Executor executor, Config config, Observer observer = {}); - - /// Crawls until there is nothing left to do or Config::max_pages is reached. - anyhttp::Task run(); - -private: - struct Response - { - anyhttp::error_code ec; - anyhttp::client::Message message; - std::string route; - std::string error; // what went wrong, with ec or from an exception, on the last route - }; - - struct Pacing - { - std::chrono::steady_clock::time_point next_request{}; - std::chrono::milliseconds delay{}; - }; - - bool is_allowed(std::string_view host) const; - void emit(Event event) const; - void enqueue(boost::urls::url url); - - anyhttp::Task resolve_service(const Origin& origin); - anyhttp::Task plan_for(const Origin& origin); - anyhttp::Task robots_for(Plan& plan); - - anyhttp::Task fetch(Plan& plan, boost::urls::url url); - anyhttp::Task attempt(const Route& route, boost::urls::url url); - anyhttp::Task exchange(anyhttp::client::Config client_config, boost::urls::url url); - - void visit(const boost::urls::url& url, const Response& response); - - anyhttp::Executor executor_; - Config config_; - Observer observer_; - std::set allowed_hosts_; - c_ares_async::resolver resolver_; - - std::deque frontier_; - std::set seen_; - std::size_t requested_ = 0; - - std::map plans_; // by Origin::key() - std::map robots_; // by Origin::key(), of the origin in the plan - std::map pacing_; // by host -}; - -// ================================================================================================= - -} // namespace crawler diff --git a/src/links.cpp b/src/links.cpp deleted file mode 100644 index ea0bddb..0000000 --- a/src/links.cpp +++ /dev/null @@ -1,188 +0,0 @@ -#include "links.hpp" - -#include - -#include -#include -#include - -namespace crawler -{ - -namespace -{ - -// ================================================================================================= - -bool equals_ignoring_case(std::string_view a, std::string_view b) -{ - return std::ranges::equal( - a, b, [](unsigned char x, unsigned char y) { return std::tolower(x) == std::tolower(y); }); -} - -// -// Where \p needle starts in \p text, from \p from on and in any case. -// -std::size_t find_ignoring_case(std::string_view text, std::string_view needle, std::size_t from) -{ - for (std::size_t i = from; i + needle.size() <= text.size(); ++i) - if (equals_ignoring_case(text.substr(i, needle.size()), needle)) - return i; - return std::string_view::npos; -} - -std::string decode_ampersands(std::string_view text) -{ - std::string result; - for (std::size_t i = 0; i < text.size();) - if (text.substr(i, 5) == "&") - { - result += '&'; - i += 5; - } - else - result += text[i++]; - return result; -} - -std::string_view trim(std::string_view text) -{ - const auto is_space = [](unsigned char c) { return std::isspace(c) != 0; }; - while (!text.empty() && is_space(text.front())) - text.remove_prefix(1); - while (!text.empty() && is_space(text.back())) - text.remove_suffix(1); - return text; -} - -// ================================================================================================= - -} // namespace - -std::vector extract_links(std::string_view html) -{ - std::vector links; - const auto is_space = [](unsigned char c) { return std::isspace(c) != 0; }; - - std::size_t pos = 0; - while ((pos = html.find('<', pos)) != html.npos) - { - if (html.substr(pos, 4) == "", pos + 4); - if (end == html.npos) - break; - pos = end + 3; - continue; - } - - auto cursor = pos + 1; - const auto name_start = cursor; - while (cursor < html.size() && std::isalnum(static_cast(html[cursor]))) - ++cursor; - const auto name = html.substr(name_start, cursor - name_start); - if (name.empty()) - { - ++pos; // a closing tag, a declaration, or just a '<' in the text - continue; - } - - // - // The attributes, up to the end of the tag. - // - std::string_view href, rel; - bool has_href = false; - while (cursor < html.size() && html[cursor] != '>') - { - if (is_space(html[cursor]) || html[cursor] == '/') - { - ++cursor; - continue; - } - - const auto attribute_start = cursor; - while (cursor < html.size() && !is_space(html[cursor]) && html[cursor] != '=' && - html[cursor] != '>') - ++cursor; - const auto attribute = html.substr(attribute_start, cursor - attribute_start); - - while (cursor < html.size() && is_space(html[cursor])) - ++cursor; - if (cursor >= html.size() || html[cursor] != '=') - continue; // an attribute without a value - - ++cursor; - while (cursor < html.size() && is_space(html[cursor])) - ++cursor; - - std::string_view value; - if (cursor < html.size() && (html[cursor] == '"' || html[cursor] == '\'')) - { - const auto end = html.find(html[cursor], cursor + 1); - if (end == html.npos) - return links; // the quote never ends, and neither does the tag - value = html.substr(cursor + 1, end - cursor - 1); - cursor = end + 1; - } - else - { - const auto value_start = cursor; - while (cursor < html.size() && !is_space(html[cursor]) && html[cursor] != '>') - ++cursor; - value = html.substr(value_start, cursor - value_start); - } - - if (equals_ignoring_case(attribute, "href")) - { - href = value; - has_href = true; - } - else if (equals_ignoring_case(attribute, "rel")) - rel = value; - } - - if (equals_ignoring_case(name, "a") && has_href && - find_ignoring_case(rel, "nofollow", 0) == rel.npos) - links.push_back(decode_ampersands(trim(href))); - - pos = cursor; - - // - // What is inside of these is not markup, whatever it looks like. - // - if (equals_ignoring_case(name, "script") || equals_ignoring_case(name, "style")) - { - const auto end = find_ignoring_case(html, std::string(" resolve_link(boost::urls::url_view base, std::string_view reference) -{ - const auto parsed = boost::urls::parse_uri_reference(reference); - if (!parsed) - return std::nullopt; - - boost::urls::url url; - if (!boost::urls::resolve(base, *parsed, url)) - return std::nullopt; - - const auto scheme = url.scheme_id(); - if ((scheme != boost::urls::scheme::http && scheme != boost::urls::scheme::https) || - url.encoded_host().empty() || url.has_userinfo()) - return std::nullopt; - - url.remove_fragment(); - url.normalize(); - if (url.encoded_path().empty()) - url.set_encoded_path("/"); - return url; -} - -// ================================================================================================= - -} // namespace crawler diff --git a/src/links.hpp b/src/links.hpp deleted file mode 100644 index 6fb9d71..0000000 --- a/src/links.hpp +++ /dev/null @@ -1,40 +0,0 @@ -// -// Finding the links in a page, and making absolute URLs out of them. -// -#pragma once - -#include -#include - -#include -#include -#include -#include - -namespace crawler -{ - -// ================================================================================================= - -/** - * The targets of the \c elements in \p html, in the order they appear and as they are written, - * without those marked rel="nofollow". - * - * This is a scanner, not an HTML parser. It knows about quotes, comments, and the contents of - * \c )" - R"()" - R"(y)"), - ElementsAre("/yes")); -} - -TEST(ExtractLinks, HonoursNofollow) -{ - EXPECT_THAT(extract_links(R"(xy)"), - ElementsAre("/yes")); -} - -TEST(ExtractLinks, DecodesAmpersandsAndTrimsSpace) -{ - EXPECT_THAT(extract_links(R"(a)"), ElementsAre("/a?x=1&y=2")); -} - -TEST(ExtractLinks, SurvivesWhatIsCutOff) -{ - EXPECT_THAT(extract_links(R"(never ends"), IsEmpty()); - EXPECT_THAT(extract_links("1 < 2

"), ElementsAre("/a")); -} - -// ------------------------------------------------------------------------------------------------- - -namespace -{ -std::string resolved(std::string_view reference) -{ - const auto url = resolve_link(boost::urls::url("http://example.com/dir/page?q=1"), reference); - return url ? std::string(url->buffer()) : "(none)"; -} -} // namespace - -TEST(ResolveLink, ResolvesAgainstTheBase) -{ - EXPECT_EQ(resolved("other"), "http://example.com/dir/other"); - EXPECT_EQ(resolved("/top"), "http://example.com/top"); - EXPECT_EQ(resolved("../up"), "http://example.com/up"); - EXPECT_EQ(resolved("?x=2"), "http://example.com/dir/page?x=2"); - EXPECT_EQ(resolved("//other.example/"), "http://other.example/"); - EXPECT_EQ(resolved("https://other.example/x"), "https://other.example/x"); -} - -TEST(ResolveLink, MakesTheSameOfTheSamePage) -{ - EXPECT_EQ(resolved("/a#section"), "http://example.com/a"); - EXPECT_EQ(resolved("#section"), "http://example.com/dir/page?q=1"); - EXPECT_EQ(resolved("HTTP://EXAMPLE.com"), "http://example.com/"); - EXPECT_EQ(resolved("/x/../a"), "http://example.com/a"); -} - -TEST(ResolveLink, TakesNothingButHttpAndHttps) -{ - EXPECT_EQ(resolved("mailto:someone@example.com"), "(none)"); - EXPECT_EQ(resolved("javascript:alert(1)"), "(none)"); - EXPECT_EQ(resolved("ftp://example.com/"), "(none)"); - EXPECT_EQ(resolved("data:text/html,x"), "(none)"); -} - -TEST(ResolveLink, TakesNoUserInformation) -{ - EXPECT_EQ(resolved("http://user:secret@example.com/"), "(none)"); -} - -TEST(ResolveLink, TakesNothingThatDoesNotParse) -{ - EXPECT_EQ(resolved("http://exa mple.com/"), "(none)"); -} diff --git a/test/prober_test.cpp b/test/prober_test.cpp new file mode 100644 index 0000000..a7b3896 --- /dev/null +++ b/test/prober_test.cpp @@ -0,0 +1,418 @@ +// +// The whole program but its command line, against a server and a DNS server of its own on loopback. +// Nothing in here may ever reach for anything else. +// +#include "prober.hpp" +#include "site.hpp" + +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace crawler; +using namespace std::chrono_literals; +using crawler::test::Site; +using testing::Contains; +using testing::ElementsAre; +using testing::HasSubstr; +using testing::IsEmpty; +using testing::Pair; + +namespace +{ + +std::vector h3_at(std::uint16_t port) +{ + return {{ARES_SVCB_PARAM_ALPN, {2, 'h', '3'}}, + {ARES_SVCB_PARAM_PORT, + {static_cast(port >> 8), static_cast(port & 0xff)}}}; +} + +const fake_dns_server::svc_param alpn_h2{ARES_SVCB_PARAM_ALPN, {2, 'h', '2'}}; + +class Probe : public testing::Test +{ +protected: + Config config(std::vector hosts = {"localhost"}) + { + return {.hosts = std::move(hosts), + .timeout = 1000ms, + .dns_server = site.dns_address(), + .tls_ca_file = site.ca_file().string()}; + } + + void probe(Config config) + { + Prober prober(context.get_executor(), std::move(config), + [this](const Result& result) { results.push_back(result); }); + + boost::asio::co_spawn( + context, + [&]() -> anyhttp::Task { + co_await prober.run(); + site.stop(); + // + // For the tests with a QUIC endpoint that never answers. anyhttp does not close a + // session whose connect() was cancelled: it goes on sending until its idle timeout of + // 30 seconds, and the context does not run dry before that. To go once that is fixed. + // + if (abandon_quic_sessions) + context.stop(); + }, + [](const std::exception_ptr& ex) { + if (ex) + std::rethrow_exception(ex); + }); + context.run(); + } + + // + // What the tests that do not get as far as run() still have to do. + // + void finish() + { + site.stop(); + context.run(); + } + + static int quiet() + { + spdlog::set_level(spdlog::level::off); // anyhttp logs through the default logger + return 0; + } + + static std::string notes(const Result& result) + { + std::string text; + for (const auto& note : result.notes) + text += note + "\n"; + return text; + } + + int quiet_ = quiet(); // ahead of the server, which already has something to say when it starts + boost::asio::io_context context; + Site site{context.get_executor()}; + std::vector results; + bool abandon_quic_sessions = false; +}; + +} // namespace + +// ================================================================================================= +// What it refuses +// ================================================================================================= + +TEST_F(Probe, RefusesToRunWithoutHosts) +{ + EXPECT_THROW(probe(config({})), std::invalid_argument); + finish(); +} + +TEST_F(Probe, RefusesAnythingButAHostName) +{ + for (const auto* host : {"https://localhost/", "localhost/x", "localhost:443", "*.localhost", + "user@localhost", "", "127.0.0.1", "::1"}) + EXPECT_THROW(probe(config({host})), std::invalid_argument) << host; + finish(); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST(ReadHosts, TakesNamesAndTheHostsOfUrls) +{ + std::istringstream list("example.com\n" + " www.example.com \r\n" + "\n" + "# a comment\n" + "https://www.example.net/some/path?query\n" + "http://example.org:8080/\n"); + std::vector skipped; + + EXPECT_THAT(read_hosts(list, skipped), + ElementsAre("example.com", "www.example.com", "www.example.net", "example.org")); + EXPECT_THAT(skipped, IsEmpty()); +} + +TEST(ReadHosts, SkipsWhatIsNeitherAndSaysSo) +{ + std::istringstream list("example.com\nnot a host\nhttps://127.0.0.1/\nexample.net\n"); + std::vector skipped; + + EXPECT_THAT(read_hosts(list, skipped), ElementsAre("example.com", "example.net")); + EXPECT_THAT(skipped, ElementsAre("not a host", "https://127.0.0.1/")); +} + +// ================================================================================================= +// One request, and what is reported of it +// ================================================================================================= + +TEST_F(Probe, RequestsTheHostAtThePortOfItsRecord) +{ + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + const auto& result = results[0]; + EXPECT_EQ(result.error, "") << notes(result); + EXPECT_EQ(result.host, "localhost"); + EXPECT_EQ(result.status, 200); + EXPECT_EQ(result.route, std::format("h3 localhost:{}", site.port())); + EXPECT_EQ(result.body_size, 5u); + EXPECT_THAT(result.fields, Contains(Pair("content-type", "text/html"))); + ASSERT_EQ(result.records.size(), 1u); + EXPECT_THAT(to_text(result.records[0]), HasSubstr("alpn=h3")); + + ASSERT_EQ(site.requests().size(), 1u); + EXPECT_EQ(site.requests()[0].authority, "localhost") << "the host, not where it went"; + EXPECT_THAT(site.requests()[0].user_agent, HasSubstr("crawler/")); +} + +TEST_F(Probe, RequestsNothingButTheRoot) +{ + // + // No robots.txt either: that is about crawling, and this does not. + // + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.page("/", {.body = R"(a b)"}); + site.page("/a", {.body = "not asked for"}); + site.page("/b", {.body = "not asked for"}); + + probe(config()); + + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, ReportsARedirectAndDoesNotFollowIt) +{ + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.page("/", {.status = 302, .location = "/moved"}); + site.page("/moved", {.body = "not asked for"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 302); + EXPECT_THAT(results[0].fields, Contains(Pair("location", "/moved"))); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, ReportsAnErrorStatusAsAResponse) +{ + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.page("/", {.status = 503}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 503); + EXPECT_EQ(results[0].error, ""); +} + +TEST_F(Probe, RequestsEachHostOnce) +{ + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.page("/", {.body = "hello"}); + + probe(config({"localhost", "LOCALHOST", "localhost."})); + + EXPECT_EQ(results.size(), 1u); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, GoesOnAfterAHostThatFails) +{ + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.page("/", {.body = "hello"}); + + probe(config({"nowhere.example.test", "localhost"})); + + ASSERT_EQ(results.size(), 2u); + EXPECT_EQ(results[0].host, "nowhere.example.test"); + EXPECT_NE(results[0].error, ""); + EXPECT_EQ(results[1].host, "localhost"); + EXPECT_EQ(results[1].status, 200); +} + +// ================================================================================================= +// How the HTTPS records decide about the connection +// ================================================================================================= + +TEST_F(Probe, SkipsARecordItCanNotUse) +{ + site.dns().add_https("localhost", 1, ".", {alpn_h2}); + site.dns().add_https("localhost", 2, ".", h3_at(site.port())); + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 200) << notes(results[0]); + EXPECT_THAT(notes(results[0]), HasSubstr("alpn=h2")); + EXPECT_THAT(notes(results[0]), HasSubstr("TLS over TCP")); +} + +TEST_F(Probe, FallsBackToTheNextRecordWhenAConnectionFails) +{ + // + // Nothing answers on this port. The socket is ours for the length of the test, so that nothing + // else will either. + // + boost::asio::ip::udp::socket silent(context, {boost::asio::ip::udp::v6(), 0}); + abandon_quic_sessions = true; + + site.dns().add_https("localhost", 1, ".", h3_at(silent.local_endpoint().port())); + site.dns().add_https("localhost", 2, ".", h3_at(site.port())); + site.page("/", {.body = "hello"}); + + auto configuration = config(); + configuration.timeout = 200ms; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 200) << notes(results[0]); + EXPECT_EQ(results[0].route, std::format("h3 localhost:{}", site.port())); + EXPECT_THAT(notes(results[0]), HasSubstr("timed out")); +} + +TEST_F(Probe, FailsWithoutARecord) +{ + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 0); + EXPECT_EQ(results[0].error, "no way to connect"); + EXPECT_THAT(notes(results[0]), HasSubstr("TLS over TCP")); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(Probe, DoesNotConnectToATargetOtherThanTheHost) +{ + // + // The server would answer: it is the certificate that could not be right for this name. + // + site.dns().add_https("localhost", 1, "elsewhere.example.test", h3_at(site.port())); + site.dns().add_address("elsewhere.example.test", "127.0.0.1"); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].error, "no way to connect"); + EXPECT_THAT(notes(results[0]), HasSubstr("elsewhere.example.test")); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(Probe, RefusesACertificateItDoesNotTrust) +{ + Site other(context.get_executor()); // has a certificate of its own + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.page("/", {.body = "hello"}); + + auto configuration = config(); + configuration.tls_ca_file = other.ca_file().string(); + other.stop(); + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 0); + EXPECT_THAT(results[0].error, HasSubstr("connect")); + EXPECT_THAT(site.requests(), IsEmpty()) << "no request survives the handshake"; +} + +TEST_F(Probe, FollowsAnAliasToItsRecords) +{ + // + // At the end of the chain it is the name of the alias's target that counts. So this is a route + // only for an alias that leads back to the host, by way of another name. + // + site.dns().add_https("localhost", 0, "alias.example.test"); + site.dns().add_https("alias.example.test", 1, "localhost", h3_at(site.port())); + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 200) << notes(results[0]); + EXPECT_EQ(site.dns_queries("alias.example.test"), 1u); +} + +TEST_F(Probe, RecognizesAnAliasLoop) +{ + site.dns().add_https("localhost", 0, "alias.example.test"); + site.dns().add_https("alias.example.test", 0, "localhost"); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].error, "no way to connect"); + EXPECT_THAT(notes(results[0]), HasSubstr("too long or has a loop")); +} + +TEST_F(Probe, IgnoresServiceRecordsNextToAnAlias) +{ + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.dns().add_https("localhost", 0, "alias.example.test"); + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 0) << "the alias has no records of its own"; + EXPECT_THAT(site.requests(), IsEmpty()); +} + +// ================================================================================================= +// What can go wrong +// ================================================================================================= + +TEST_F(Probe, GivesUpOnAServerThatDoesNotAnswer) +{ + boost::asio::ip::udp::socket silent(context, {boost::asio::ip::udp::v6(), 0}); + abandon_quic_sessions = true; + site.dns().add_https("localhost", 1, ".", h3_at(silent.local_endpoint().port())); + + auto configuration = config(); + configuration.timeout = 100ms; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 0); + EXPECT_THAT(results[0].error, HasSubstr("timed out")); +} + +TEST_F(Probe, ReportsANameThatDoesNotExist) +{ + probe(config({"nowhere.example.test"})); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].error, "no way to connect"); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(Probe, ReportsAFailedLookup) +{ + site.dns().set_rcode(ARES_RCODE_SERVFAIL); + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].error, "no way to connect"); + EXPECT_THAT(notes(results[0]), HasSubstr("HTTPS lookup")); + EXPECT_THAT(site.requests(), IsEmpty()); +} diff --git a/test/robots_test.cpp b/test/robots_test.cpp deleted file mode 100644 index 4b01a36..0000000 --- a/test/robots_test.cpp +++ /dev/null @@ -1,159 +0,0 @@ -#include "robots.hpp" - -#include - -using crawler::Robots; -using namespace std::chrono_literals; - -TEST(Robots, AllowsEverythingByDefault) -{ - EXPECT_TRUE(Robots{}.allowed("/anything")); - EXPECT_TRUE(Robots::parse("", "crawler").allowed("/anything")); -} - -TEST(Robots, DisallowAllAllowsOnlyItself) -{ - EXPECT_FALSE(Robots::disallow_all().allowed("/")); - EXPECT_TRUE(Robots::disallow_all().allowed("/robots.txt")); -} - -TEST(Robots, AppliesTheGroupForEveryone) -{ - const auto robots = Robots::parse("User-agent: *\nDisallow: /private/\n", "crawler"); - EXPECT_FALSE(robots.allowed("/private/x")); - EXPECT_TRUE(robots.allowed("/public/x")); -} - -TEST(Robots, PrefersTheGroupThatNamesTheAgentInAnyCase) -{ - const auto robots = Robots::parse("User-agent: *\nDisallow: /\n\n" - "User-Agent: CRAWLER\nDisallow: /mine/\n", - "crawler"); - EXPECT_TRUE(robots.allowed("/x")); - EXPECT_FALSE(robots.allowed("/mine/x")); -} - -TEST(Robots, IgnoresGroupsForOthers) -{ - EXPECT_TRUE(Robots::parse("User-agent: other\nDisallow: /\n", "crawler").allowed("/x")); -} - -TEST(Robots, MergesTheGroupsOfAnAgent) -{ - const auto robots = Robots::parse("User-agent: crawler\nDisallow: /a\n\n" - "User-agent: other\nDisallow: /b\n\n" - "User-agent: crawler\nDisallow: /c\n", - "crawler"); - EXPECT_FALSE(robots.allowed("/a")); - EXPECT_TRUE(robots.allowed("/b")); - EXPECT_FALSE(robots.allowed("/c")); -} - -TEST(Robots, SharesAGroupBetweenAgents) -{ - const auto robots = - Robots::parse("User-agent: other\nUser-agent: crawler\nDisallow: /x\n", "crawler"); - EXPECT_FALSE(robots.allowed("/x")); -} - -TEST(Robots, LetsTheLongestRuleDecide) -{ - const auto robots = - Robots::parse("User-agent: *\nDisallow: /a/\nAllow: /a/b/\nDisallow: /a/b/c\n", "crawler"); - EXPECT_FALSE(robots.allowed("/a/x")); - EXPECT_TRUE(robots.allowed("/a/b/x")); - EXPECT_FALSE(robots.allowed("/a/b/c")); -} - -TEST(Robots, PrefersAllowBetweenRulesOfTheSameLength) -{ - EXPECT_TRUE(Robots::parse("User-agent: *\nDisallow: /a\nAllow: /a\n", "crawler").allowed("/a")); - EXPECT_TRUE(Robots::parse("User-agent: *\nAllow: /a\nDisallow: /a\n", "crawler").allowed("/a")); -} - -TEST(Robots, AnEmptyDisallowDisallowsNothing) -{ - EXPECT_TRUE(Robots::parse("User-agent: *\nDisallow:\n", "crawler").allowed("/x")); -} - -TEST(Robots, UnderstandsWildcards) -{ - const auto robots = Robots::parse("User-agent: *\nDisallow: /*.php\n", "crawler"); - EXPECT_FALSE(robots.allowed("/a.php")); - EXPECT_FALSE(robots.allowed("/a/b.php?x=1")); - EXPECT_TRUE(robots.allowed("/a.html")); -} - -TEST(Robots, UnderstandsTheEndAnchor) -{ - const auto robots = - Robots::parse("User-agent: *\nDisallow: /*.php$\nDisallow: /exact$\n", "crawler"); - EXPECT_FALSE(robots.allowed("/a.php")); - EXPECT_FALSE(robots.allowed("/a.php.php")); - EXPECT_TRUE(robots.allowed("/a.php.bak")); - EXPECT_FALSE(robots.allowed("/exact")); - EXPECT_TRUE(robots.allowed("/exactly")); -} - -TEST(Robots, ComparesPathsWithCase) -{ - const auto robots = Robots::parse("User-agent: *\nDisallow: /Private\n", "crawler"); - EXPECT_FALSE(robots.allowed("/Private")); - EXPECT_TRUE(robots.allowed("/private")); -} - -TEST(Robots, ChecksTheQueryToo) -{ - const auto robots = Robots::parse("User-agent: *\nDisallow: /*?session=\n", "crawler"); - EXPECT_FALSE(robots.allowed("/a?session=1")); - EXPECT_TRUE(robots.allowed("/a?page=1")); -} - -TEST(Robots, SkipsCommentsAndNoise) -{ - const auto robots = Robots::parse("# about us\r\n" - "User-agent: * # everyone\r\n" - "Sitemap: https://example.com/sitemap.xml\r\n" - "what is this\r\n" - "Disallow: /x # not there\r\n", - "crawler"); - EXPECT_FALSE(robots.allowed("/x")); - EXPECT_TRUE(robots.allowed("/y")); -} - -TEST(Robots, IgnoresRulesAheadOfAnyGroup) -{ - EXPECT_TRUE(Robots::parse("Disallow: /\nUser-agent: *\nAllow: /x\n", "crawler").allowed("/y")); -} - -TEST(Robots, AlwaysAllowsItself) -{ - EXPECT_TRUE(Robots::parse("User-agent: *\nDisallow: /\n", "crawler").allowed("/robots.txt")); -} - -TEST(Robots, ReadsTheCrawlDelay) -{ - EXPECT_EQ(Robots::parse("User-agent: *\nCrawl-delay: 2\n", "crawler").crawl_delay(), 2s); - EXPECT_EQ(Robots::parse("User-agent: *\nCrawl-delay: 0.5\n", "crawler").crawl_delay(), 500ms); - EXPECT_FALSE(Robots::parse("User-agent: *\nDisallow: /\n", "crawler").crawl_delay()); -} - -TEST(Robots, IgnoresACrawlDelayThatMakesNoSense) -{ - EXPECT_FALSE(Robots::parse("User-agent: *\nCrawl-delay: soon\n", "crawler").crawl_delay()); - EXPECT_FALSE(Robots::parse("User-agent: *\nCrawl-delay: -1\n", "crawler").crawl_delay()); - EXPECT_FALSE(Robots::parse("User-agent: *\nCrawl-delay: inf\n", "crawler").crawl_delay()); -} - -TEST(Robots, KeepsAHugeCrawlDelayToADay) -{ - EXPECT_EQ(Robots::parse("User-agent: *\nCrawl-delay: 1e30\n", "crawler").crawl_delay(), 24h); -} - -TEST(Robots, TakesTheCrawlDelayOfTheChosenGroupOnly) -{ - const auto robots = Robots::parse("User-agent: other\nCrawl-delay: 9\n\n" - "User-agent: crawler\nCrawl-delay: 1\n", - "crawler"); - EXPECT_EQ(robots.crawl_delay(), 1s); -} diff --git a/test/routes_test.cpp b/test/routes_test.cpp index 9aa28d9..a8e30d1 100644 --- a/test/routes_test.cpp +++ b/test/routes_test.cpp @@ -3,8 +3,6 @@ // #include "routes.hpp" -#include - #include #include @@ -16,16 +14,14 @@ using testing::IsEmpty; namespace { -Origin origin(std::string_view url) { return *origin_of(boost::urls::url(url)); } - c_ares_async::https_record record(unsigned short priority, std::string target, std::vector params = {}) { return {.priority = priority, .target = std::move(target), .params = std::move(params)}; } -ServiceRecords found(std::string owner, std::vector records, - bool aliased = false) +ServiceRecords found(std::vector records, + std::string owner = "example.com", bool aliased = false) { return {.owner = std::move(owner), .aliased = aliased, .records = std::move(records)}; } @@ -42,73 +38,37 @@ std::vector routes(const Plan& plan) // ================================================================================================= -TEST(Origin, TakesThePortOfTheScheme) -{ - EXPECT_EQ(origin("http://example.com/x").port, 80); - EXPECT_EQ(origin("https://example.com/x").port, 443); - EXPECT_EQ(origin("https://example.com:8443/x").port, 8443); -} - -TEST(Origin, ComparesHostsTheWayDnsDoes) -{ - EXPECT_EQ(origin("https://EXAMPLE.com./").host, "example.com"); -} - -TEST(Origin, KnowsAnAddressFromAName) -{ - EXPECT_FALSE(origin("http://example.com/").is_address); - EXPECT_TRUE(origin("http://127.0.0.1/").is_address); - EXPECT_TRUE(origin("http://[::1]/").is_address); -} - -TEST(Origin, HasNoneForOtherSchemes) -{ - EXPECT_FALSE(origin_of(boost::urls::url("ftp://example.com/"))); - EXPECT_FALSE(origin_of(boost::urls::url("mailto:someone@example.com"))); -} - -TEST(QueryName, IsTheHostOnTheDefaultPort) -{ - EXPECT_EQ(https_query_name(origin("https://example.com/")), "example.com"); - EXPECT_EQ(https_query_name(origin("https://example.com:443/")), "example.com"); -} - -TEST(QueryName, HasAPortPrefixOnAnyOther) +TEST(NormalizeHost, ComparesHostsTheWayDnsDoes) { - EXPECT_EQ(https_query_name(origin("https://example.com:8443/")), "_8443._https.example.com"); + EXPECT_EQ(normalize_host("EXAMPLE.com."), "example.com"); } -TEST(QueryName, IsThatOfTheHttpsFormForHttp) +TEST(ToText, ReadsLikeAZoneFile) { - EXPECT_EQ(https_query_name(origin("http://example.com/")), "example.com"); - EXPECT_EQ(https_query_name(origin("http://example.com:80/")), "example.com"); - EXPECT_EQ(https_query_name(origin("http://example.com:8080/")), "_8080._https.example.com"); + EXPECT_EQ(to_text(record(1, ".", {"alpn=h3", "port=8443"})), "HTTPS 1 . alpn=h3 port=8443"); + EXPECT_EQ(to_text(record(0, "other.example")), "HTTPS 0 other.example"); + EXPECT_EQ(to_text(record(1, "")), "HTTPS 1 ."); } // ------------------------------------------------------------------------------------------------- TEST(Plan, ConnectsOverHttp3WhereTheRecordOffersIt) { - const auto plan = plan_routes(origin("https://example.com/"), - found("example.com", {record(1, ".", {"alpn=h3,h2"})})); + const auto plan = plan_routes("example.com", found({record(1, ".", {"alpn=h3,h2"})})); EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443")); - EXPECT_EQ(plan.origin.scheme, "https"); } TEST(Plan, GoesToThePortOfTheRecord) { - const auto plan = plan_routes(origin("https://example.com/"), - found("example.com", {record(1, ".", {"alpn=h3", "port=8443"})})); + const auto plan = plan_routes("example.com", found({record(1, ".", {"alpn=h3", "port=8443"})})); EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:8443")); - EXPECT_EQ(plan.origin.port, 443) << "requests still go to the origin"; } TEST(Plan, TriesTheLowestPriorityFirst) { - const auto plan = plan_routes(origin("https://example.com/"), - found("example.com", {record(3, ".", {"alpn=h3", "port=3"}), + const auto plan = plan_routes("example.com", found({record(3, ".", {"alpn=h3", "port=3"}), record(1, ".", {"alpn=h3", "port=1"}), record(2, ".", {"alpn=h3", "port=2"})})); @@ -118,9 +78,9 @@ TEST(Plan, TriesTheLowestPriorityFirst) TEST(Plan, RejectsARecordWithoutHttp3) { - const auto plan = plan_routes(origin("https://example.com/"), - found("example.com", {record(1, ".", {"alpn=h2"}), - record(2, ".", {"alpn=h3", "port=8443"})})); + const auto plan = + plan_routes("example.com", + found({record(1, ".", {"alpn=h2"}), record(2, ".", {"alpn=h3", "port=8443"})})); EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:8443")); ASSERT_EQ(plan.rejected.size(), 1u); @@ -128,55 +88,59 @@ TEST(Plan, RejectsARecordWithoutHttp3) EXPECT_THAT(plan.rejected[0], HasSubstr("TLS over TCP")); } -TEST(Plan, HasNoRouteToAnHttpsOriginWithoutARecord) +TEST(Plan, HasNoRouteWithoutARecord) { - const auto plan = plan_routes(origin("https://example.com/"), found("example.com", {})); + const auto plan = plan_routes("example.com", found({})); EXPECT_THAT(plan.routes, IsEmpty()); ASSERT_EQ(plan.rejected.size(), 1u); EXPECT_THAT(plan.rejected[0], HasSubstr("TLS over TCP")); } -TEST(Plan, RejectsATargetOtherThanTheOrigin) +TEST(Plan, RejectsATargetOtherThanTheHost) { // // The certificate has to be valid for the origin (section 9.1), and anyhttp would check it // against what it connects to. // - const auto plan = plan_routes(origin("https://example.com/"), - found("example.com", {record(1, "cdn.example.net", {"alpn=h3"})})); + const auto plan = plan_routes("example.com", found({record(1, "cdn.example.net", {"alpn=h3"})})); EXPECT_THAT(plan.routes, IsEmpty()); ASSERT_EQ(plan.rejected.size(), 1u); EXPECT_THAT(plan.rejected[0], HasSubstr("cdn.example.net")); } -TEST(Plan, TakesAnExplicitTargetThatIsTheOrigin) +TEST(Plan, TakesAnExplicitTargetThatIsTheHost) { - const auto plan = - plan_routes(origin("https://example.com:8443/"), - found("_8443._https.example.com", {record(1, "EXAMPLE.com.", {"alpn=h3"})})); + const auto plan = plan_routes("example.com", found({record(1, "EXAMPLE.com.", {"alpn=h3"})})); - EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:8443")); + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443")); } -TEST(Plan, TakesTheOwnNameOfAPortPrefixedRecordLiterally) +TEST(Plan, TakesTheOwnNameOfARecordAtTheEndOfAnAliasAsThatName) { // - // "." is the owner name, section 2.5.2, and that includes the prefix. + // "." is the owner name, section 2.5.2, and that is not the host any more. // - const auto plan = plan_routes(origin("https://example.com:8443/"), - found("_8443._https.example.com", {record(1, ".", {"alpn=h3"})})); + const auto plan = + plan_routes("example.com", found({record(1, ".", {"alpn=h3"})}, "other.example", true)); + + EXPECT_THAT(plan.routes, IsEmpty()); + EXPECT_THAT(plan.rejected.front(), HasSubstr("other.example")); +} + +TEST(Plan, SaysSoAboutTheTargetOfAnAlias) +{ + const auto plan = plan_routes("example.com", found({}, "other.example", true)); EXPECT_THAT(plan.routes, IsEmpty()); ASSERT_EQ(plan.rejected.size(), 1u); - EXPECT_THAT(plan.rejected[0], HasSubstr("_8443._https.example.com")); + EXPECT_THAT(plan.rejected[0], HasSubstr("other.example")); } TEST(Plan, HasNoDefaultSetToFallBackOnWithNoDefaultAlpn) { - const auto plan = plan_routes(origin("https://example.com/"), - found("example.com", {record(1, ".", {"no-default-alpn"})})); + const auto plan = plan_routes("example.com", found({record(1, ".", {"no-default-alpn"})})); EXPECT_THAT(plan.routes, IsEmpty()); } @@ -184,9 +148,8 @@ TEST(Plan, HasNoDefaultSetToFallBackOnWithNoDefaultAlpn) TEST(Plan, IgnoresARecordWithAMandatoryKeyItDoesNotKnow) { const auto plan = - plan_routes(origin("https://example.com/"), - found("example.com", - {record(1, ".", {"mandatory=alpn,key65280", "alpn=h3", "key65280=01"})})); + plan_routes("example.com", + found({record(1, ".", {"mandatory=alpn,key65280", "alpn=h3", "key65280=01"})})); EXPECT_THAT(plan.routes, IsEmpty()); EXPECT_THAT(plan.rejected.front(), HasSubstr("key65280")); @@ -195,8 +158,7 @@ TEST(Plan, IgnoresARecordWithAMandatoryKeyItDoesNotKnow) TEST(Plan, IgnoresARecordThatRequiresEch) { const auto plan = - plan_routes(origin("https://example.com/"), - found("example.com", {record(1, ".", {"mandatory=ech", "alpn=h3", "ech=dead"})})); + plan_routes("example.com", found({record(1, ".", {"mandatory=ech", "alpn=h3", "ech=dead"})})); EXPECT_THAT(plan.routes, IsEmpty()); } @@ -204,70 +166,23 @@ TEST(Plan, IgnoresARecordThatRequiresEch) TEST(Plan, AcceptsMandatoryKeysItCanLeaveAlone) { const auto plan = plan_routes( - origin("https://example.com/"), - found("example.com", - {record(1, ".", {"mandatory=alpn,ipv4hint", "alpn=h3", "ipv4hint=192.0.2.1"})})); + "example.com", + found({record(1, ".", {"mandatory=alpn,ipv4hint", "alpn=h3", "ipv4hint=192.0.2.1"})})); EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443")); } TEST(Plan, IgnoresARecordWithAnInvalidPort) { - const auto plan = plan_routes(origin("https://example.com/"), - found("example.com", {record(1, ".", {"alpn=h3", "port=x"})})); - - EXPECT_THAT(plan.routes, IsEmpty()); -} - -// ------------------------------------------------------------------------------------------------- - -TEST(Plan, ConnectsInTheClearToAnHttpOriginWithoutARecord) -{ - const auto plan = - plan_routes(origin("http://example.com:8080/"), found("_8080._https.example.com", {})); - - EXPECT_THAT(routes(plan), ElementsAre("h1 example.com:8080")); - EXPECT_EQ(plan.origin.scheme, "http"); -} - -TEST(Plan, UpgradesAnHttpOriginWithARecord) -{ - const auto plan = plan_routes(origin("http://example.com/"), - found("example.com", {record(1, ".", {"alpn=h3"})})); - - EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443")); - EXPECT_EQ(plan.origin.scheme, "https"); - EXPECT_EQ(plan.origin.port, 443); -} - -TEST(Plan, DoesNotFallBackToCleartextAfterAnUpgrade) -{ - const auto plan = plan_routes(origin("http://example.com/"), - found("example.com", {record(1, ".", {"alpn=h2"})})); + const auto plan = plan_routes("example.com", found({record(1, ".", {"alpn=h3", "port=x"})})); EXPECT_THAT(plan.routes, IsEmpty()); - EXPECT_EQ(plan.origin.scheme, "https"); } -TEST(Plan, DoesNotUpgradeForAnIncompatibleRecord) +TEST(Plan, PassesOnWhatWentWrongInTheLookup) { - // - // Section 9.5: only a compatible record, or an alias. - // - const auto plan = plan_routes( - origin("http://example.com/"), - found("example.com", {record(1, ".", {"mandatory=key65280", "alpn=h3", "key65280=01"})})); - - EXPECT_THAT(routes(plan), ElementsAre("h1 example.com:80")); - EXPECT_EQ(plan.origin.scheme, "http"); -} + auto records = found({}); + records.notes.push_back("HTTPS lookup failed"); -TEST(Plan, UpgradesAnHttpOriginWithAnAlias) -{ - const auto plan = plan_routes(origin("http://example.com/"), - found("alias.example.net", {}, /* aliased */ true)); - - EXPECT_THAT(plan.routes, IsEmpty()); - EXPECT_EQ(plan.origin.scheme, "https"); - EXPECT_THAT(plan.rejected.front(), HasSubstr("alias.example.net")); + EXPECT_THAT(plan_routes("example.com", records).rejected.front(), "HTTPS lookup failed"); } diff --git a/test/site.cpp b/test/site.cpp index 2df36ac..7906bf8 100644 --- a/test/site.cpp +++ b/test/site.cpp @@ -17,8 +17,7 @@ namespace crawler::test // ================================================================================================= -Site::Site(const anyhttp::Executor& executor, const std::string& listen_address) - : requests_(std::make_shared>()) +Site::Site(const anyhttp::Executor& executor) : requests_(std::make_shared>()) { auto directory = (std::filesystem::temp_directory_path() / "crawler-test-XXXXXX").string(); if (!mkdtemp(directory.data())) @@ -31,7 +30,10 @@ Site::Site(const anyhttp::Executor& executor, const std::string& listen_address) std::ofstream(certificate_file_) << certificate.certificate_pem; std::ofstream(key_file_) << certificate.key_pem; - server_.emplace(executor, anyhttp::server::Config{.listen_address = listen_address, + // + // "::" takes IPv4 as well, and "localhost" may be either. + // + server_.emplace(executor, anyhttp::server::Config{.listen_address = "::", .port = 0, .tls_certificate_chain = certificate_file_, .tls_private_key = key_file_}); diff --git a/test/site.hpp b/test/site.hpp index ff68a80..88818d2 100644 --- a/test/site.hpp +++ b/test/site.hpp @@ -45,9 +45,7 @@ class Site std::chrono::steady_clock::time_point time; }; - /// Listens on \p listen_address: all of them by default, "::" taking IPv4 as well, as - /// "localhost" may be either. - explicit Site(const anyhttp::Executor& executor, const std::string& listen_address = "::"); + explicit Site(const anyhttp::Executor& executor); ~Site(); /// A path that has no page gets a 404. From abd9d878fc76ecfc3393e7030aa9080c4c0d9548 Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Mon, 2 Nov 2026 22:21:14 +0000 Subject: [PATCH 04/27] refactor: rename the project to http-probe It does not crawl any more, so the name says the opposite of what it does. http-probe for the program and the repository, like c-ares-async, http_probe in code. The User-Agent and the README now point at github.com/pgit/http-probe, which takes renaming the repository on GitHub to be right. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 12 ++++++------ CMakeLists.txt | 18 +++++++++--------- CMakePresets.json | 2 +- README.md | 10 +++++----- src/main.cpp | 26 +++++++++++++------------- src/prober.cpp | 6 +++--- src/prober.hpp | 4 ++-- src/routes.cpp | 4 ++-- src/routes.hpp | 4 ++-- test/CMakeLists.txt | 26 +++++++++++++------------- test/demo_site.cpp | 4 ++-- test/prober_test.cpp | 6 +++--- test/routes_test.cpp | 2 +- test/site.cpp | 6 +++--- test/site.hpp | 6 +++--- 15 files changed, 68 insertions(+), 68 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6068665..8b68134 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,8 +1,8 @@ -# Working on the crawler +# Working on http-probe -Despite the name it does not crawl: it requests `https:///` once per host and reports what -came back, connecting the way the host's HTTPS records say. It is built on two libraries of the same -author, each in a repository of its own: anyhttp for HTTP and c-ares-async for DNS. +It requests `https:///` once per host and reports what came back, connecting the way the +host's HTTPS records say. It is built on two libraries of the same author, each in a repository of +its own: anyhttp for HTTP and c-ares-async for DNS. [README.md](README.md) has the presets and the build commands. ## Safety rules @@ -27,9 +27,9 @@ any I/O, and [prober.cpp](src/prober.cpp) does the rest. The tests in [test/](te ## The libraries - Changes to a library belong in that library's repository and session, with a test in its suite. - Do not work around a library bug in crawler code: get it fixed there, then move the pin. + Do not work around a library bug in this code: get it fixed there, then move the pin. - Moving a pin in [CMakeLists.txt](CMakeLists.txt) is a commit of its own. CI builds from the pins - only, so a crawler change that needs a library change has to wait for that change to be pushed. + only, so a change here that needs a library change has to wait for that change to be pushed. - The `dev` preset builds the sibling checkouts as they are, on whatever branch they are on, and can differ from the pins. Check with the `pinned` preset before committing. - Everything in one process has to agree on the standard library, the TLS library and the I/O diff --git a/CMakeLists.txt b/CMakeLists.txt index 522e265..a1ad48a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.30) -project(crawler VERSION 0.0.1 DESCRIPTION "Requests hosts the way their HTTPS records say, on anyhttp and c-ares-async" LANGUAGES C CXX) +project(http_probe VERSION 0.0.1 DESCRIPTION "Requests hosts the way their HTTPS records say, on anyhttp and c-ares-async" LANGUAGES C CXX) set(CMAKE_CXX_STANDARD 23) set(CMAKE_CXX_STANDARD_REQUIRED On) @@ -59,16 +59,16 @@ FetchContent_MakeAvailable(anyhttp c_ares_async) # The program in a library of its own, for the tests to link. The warnings are not for the sources # from c-ares-async that the tests borrow, see test/CMakeLists.txt. # -set(crawler_warnings -Wall -Wextra -Werror -Wno-missing-designated-field-initializers) +set(http_probe_warnings -Wall -Wextra -Werror -Wno-missing-designated-field-initializers) -add_library(crawler_core STATIC src/prober.cpp src/routes.cpp) -target_include_directories(crawler_core PUBLIC src) -target_link_libraries(crawler_core PUBLIC anyhttp::anyhttp c_ares_async::c_ares_async) -target_compile_options(crawler_core PRIVATE ${crawler_warnings}) +add_library(http_probe_core STATIC src/prober.cpp src/routes.cpp) +target_include_directories(http_probe_core PUBLIC src) +target_link_libraries(http_probe_core PUBLIC anyhttp::anyhttp c_ares_async::c_ares_async) +target_compile_options(http_probe_core PRIVATE ${http_probe_warnings}) -add_executable(crawler src/main.cpp) -target_link_libraries(crawler PRIVATE crawler_core) -target_compile_options(crawler PRIVATE ${crawler_warnings}) +add_executable(http-probe src/main.cpp) +target_link_libraries(http-probe PRIVATE http_probe_core) +target_compile_options(http-probe PRIVATE ${http_probe_warnings}) enable_testing() add_subdirectory(test) diff --git a/CMakePresets.json b/CMakePresets.json index d859f73..667b0f3 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -21,7 +21,7 @@ { "name": "dev", "displayName": "Debug, against sibling checkouts", - "description": "What the crawler is developed with: the two libraries as they are in their checkouts next to this one, whatever the pins say. Keep this tree in build/, clangd reads its compile_commands.json", + "description": "What http-probe is developed with: the two libraries as they are in their checkouts next to this one, whatever the pins say. Keep this tree in build/, clangd reads its compile_commands.json", "inherits": "base", "binaryDir": "${sourceDir}/build", "cacheVariables": { diff --git a/README.md b/README.md index d6454cd..f355793 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ -# crawler +# http-probe Requests hosts the way their HTTPS records say, on [anyhttp](https://github.com/pgit/anyhttp) (HTTP) and -[c-ares-async](https://github.com/pgit/c-ares-async) (DNS). Despite the name, it does not crawl. +[c-ares-async](https://github.com/pgit/c-ares-async) (DNS). > [!WARNING] > This repository is purely experimental. It is not meant for production use, and anything in it @@ -11,14 +11,14 @@ Requests hosts the way their HTTPS records say, on [anyhttp](https://github.com/ ## What it does ```sh -crawler example.com www.example.org -crawler --file hostnames.txt +http-probe example.com www.example.org +http-probe --file hostnames.txt ``` It requests `https:///` once for each host, in turn, and prints the status, the route, the time it took and the header fields of the response, along with the host's HTTPS records. A redirect is reported and not followed, and nothing else on the host is requested, `robots.txt` included. -`crawler` without arguments lists the options. The exit code is 0 if every host answered, whatever +`http-probe` without arguments lists the options. The exit code is 0 if every host answered, whatever the status. How to connect is up to the host's HTTPS records in DNS ([RFC 9460](https://www.rfc-editor.org/rfc/rfc9460)): diff --git a/src/main.cpp b/src/main.cpp index f82b176..f6c0cc3 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,5 +1,5 @@ // -// crawler [options] HOST... +// http-probe [options] HOST... // #include "prober.hpp" @@ -22,8 +22,8 @@ namespace { -constexpr std::string_view usage = R"(usage: crawler [options] HOST... - crawler [options] --file LIST +constexpr std::string_view usage = R"(usage: http-probe [options] HOST... + http-probe [options] --file LIST Requests https://HOST/ once for each host name and prints the status and header fields of the response. How to connect is up to the host's HTTPS records in DNS (RFC 9460). Redirects are reported @@ -35,9 +35,9 @@ and not followed, and nothing else on the host is requested. --timeout-ms N how long one request may take; default: 10000 )"; -crawler::Config parse_arguments(int argc, char* argv[]) +http_probe::Config parse_arguments(int argc, char* argv[]) { - crawler::Config config; + http_probe::Config config; for (int i = 1; i < argc; ++i) { const std::string_view argument = argv[i]; @@ -57,9 +57,9 @@ crawler::Config parse_arguments(int argc, char* argv[]) if (!list) throw std::invalid_argument(std::format("can't read '{}'", value)); std::vector skipped; - config.hosts.append_range(crawler::read_hosts(list, skipped)); + config.hosts.append_range(http_probe::read_hosts(list, skipped)); for (const auto& line : skipped) - std::println(stderr, "crawler: skipping '{}': not a host name", line); + std::println(stderr, "http-probe: skipping '{}': not a host name", line); } else if (argument == "--dns-server") config.dns_server = value; @@ -80,7 +80,7 @@ crawler::Config parse_arguments(int argc, char* argv[]) return config; } -void print(const crawler::Result& result) +void print(const http_probe::Result& result) { if (result.error.empty()) std::println("{}: {} [{}] {} bytes in {}", result.host, result.status, result.route, @@ -89,7 +89,7 @@ void print(const crawler::Result& result) std::println("{}: failed: {}", result.host, result.error); for (const auto& record : result.records) - std::println(" record: {}", crawler::to_text(record)); + std::println(" record: {}", http_probe::to_text(record)); for (const auto& note : result.notes) std::println(" note: {}", note); for (const auto& [name, value] : result.fields) @@ -103,7 +103,7 @@ int main(int argc, char* argv[]) spdlog::set_level(spdlog::level::err); // anyhttp logs through the default logger bool all_answered = true; - const auto report = [&](const crawler::Result& result) { + const auto report = [&](const http_probe::Result& result) { all_answered = all_answered && result.error.empty(); print(result); }; @@ -111,7 +111,7 @@ int main(int argc, char* argv[]) try { boost::asio::io_context context; - crawler::Prober prober(context.get_executor(), parse_arguments(argc, argv), report); + http_probe::Prober prober(context.get_executor(), parse_arguments(argc, argv), report); boost::asio::co_spawn(context, prober.run(), [](const std::exception_ptr& ex) { if (ex) std::rethrow_exception(ex); @@ -120,12 +120,12 @@ int main(int argc, char* argv[]) } catch (const std::invalid_argument& ex) { - std::println(stderr, "crawler: {}\n\n{}", ex.what(), usage); + std::println(stderr, "http-probe: {}\n\n{}", ex.what(), usage); return 2; } catch (const std::exception& ex) { - std::println(stderr, "crawler: {}", ex.what()); + std::println(stderr, "http-probe: {}", ex.what()); return 1; } diff --git a/src/prober.cpp b/src/prober.cpp index 2d0b44b..a678896 100644 --- a/src/prober.cpp +++ b/src/prober.cpp @@ -17,7 +17,7 @@ namespace http = boost::beast::http; -namespace crawler +namespace http_probe { namespace @@ -25,7 +25,7 @@ namespace // ================================================================================================= -constexpr std::string_view user_agent = "crawler/0.0.1 (+https://github.com/pgit/crawler)"; +constexpr std::string_view user_agent = "http-probe/0.0.1 (+https://github.com/pgit/http-probe)"; constexpr int max_alias_chain = 8; @@ -251,4 +251,4 @@ anyhttp::Task Prober::attempt(const Route& route, const std::s // ================================================================================================= -} // namespace crawler +} // namespace http_probe diff --git a/src/prober.hpp b/src/prober.hpp index 649759a..8eeb423 100644 --- a/src/prober.hpp +++ b/src/prober.hpp @@ -17,7 +17,7 @@ #include #include -namespace crawler +namespace http_probe { // ================================================================================================= @@ -102,4 +102,4 @@ class Prober // ================================================================================================= -} // namespace crawler +} // namespace http_probe diff --git a/src/routes.cpp b/src/routes.cpp index 12970fc..75bea1e 100644 --- a/src/routes.cpp +++ b/src/routes.cpp @@ -7,7 +7,7 @@ #include #include -namespace crawler +namespace http_probe { namespace @@ -157,4 +157,4 @@ Plan plan_routes(std::string_view host, const ServiceRecords& found) // ================================================================================================= -} // namespace crawler +} // namespace http_probe diff --git a/src/routes.hpp b/src/routes.hpp index 5c6a788..d702b2d 100644 --- a/src/routes.hpp +++ b/src/routes.hpp @@ -11,7 +11,7 @@ #include #include -namespace crawler +namespace http_probe { // ================================================================================================= @@ -62,4 +62,4 @@ Plan plan_routes(std::string_view host, const ServiceRecords& found); // ================================================================================================= -} // namespace crawler +} // namespace http_probe diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index cc04a88..991e284 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -26,34 +26,34 @@ target_link_libraries(fake_dns PUBLIC c_ares_async::c_ares_async anyhttp::anyhtt add_library(test_site STATIC site.cpp) target_include_directories(test_site PUBLIC .) -target_link_libraries(test_site PUBLIC crawler_core fake_dns) -target_compile_options(test_site PRIVATE ${crawler_warnings}) +target_link_libraries(test_site PUBLIC http_probe_core fake_dns) +target_compile_options(test_site PRIVATE ${http_probe_warnings}) -add_executable(crawler_tests prober_test.cpp routes_test.cpp) -target_link_libraries(crawler_tests PRIVATE test_site GTest::gmock GTest::gtest_main) -target_compile_options(crawler_tests PRIVATE ${crawler_warnings}) +add_executable(http_probe_tests prober_test.cpp routes_test.cpp) +target_link_libraries(http_probe_tests PRIVATE test_site GTest::gmock GTest::gtest_main) +target_compile_options(http_probe_tests PRIVATE ${http_probe_warnings}) include(GoogleTest) -gtest_discover_tests(crawler_tests DISCOVERY_TIMEOUT 60 PROPERTIES TIMEOUT 5) +gtest_discover_tests(http_probe_tests DISCOVERY_TIMEOUT 60 PROPERTIES TIMEOUT 5) # # A site on loopback to point the program at by hand, as the README shows. # add_executable(demo_site demo_site.cpp) target_link_libraries(demo_site PRIVATE test_site) -target_compile_options(demo_site PRIVATE ${crawler_warnings}) +target_compile_options(demo_site PRIVATE ${http_probe_warnings}) # # What the program itself does ahead of any connection. Everything else is in the tests above. # -add_test(NAME crawler_refuses_no_hosts COMMAND crawler) -set_tests_properties(crawler_refuses_no_hosts PROPERTIES +add_test(NAME http_probe_refuses_no_hosts COMMAND http-probe) +set_tests_properties(http_probe_refuses_no_hosts PROPERTIES PASS_REGULAR_EXPRESSION "nothing to do: no host names") -add_test(NAME crawler_refuses_a_url COMMAND crawler https://localhost/) -set_tests_properties(crawler_refuses_a_url PROPERTIES +add_test(NAME http_probe_refuses_a_url COMMAND http-probe https://localhost/) +set_tests_properties(http_probe_refuses_a_url PROPERTIES PASS_REGULAR_EXPRESSION "is not a host name") -add_test(NAME crawler_refuses_a_file_it_can_not_read COMMAND crawler --file /nonexistent/list) -set_tests_properties(crawler_refuses_a_file_it_can_not_read PROPERTIES +add_test(NAME http_probe_refuses_a_file_it_can_not_read COMMAND http-probe --file /nonexistent/list) +set_tests_properties(http_probe_refuses_a_file_it_can_not_read PROPERTIES PASS_REGULAR_EXPRESSION "can't read") diff --git a/test/demo_site.cpp b/test/demo_site.cpp index d27b928..de7533c 100644 --- a/test/demo_site.cpp +++ b/test/demo_site.cpp @@ -16,7 +16,7 @@ int main() spdlog::set_level(spdlog::level::err); boost::asio::io_context context; - crawler::test::Site site(context.get_executor()); + http_probe::test::Site site(context.get_executor()); site.dns().add_https("localhost", 1, ".", {{ARES_SVCB_PARAM_ALPN, {2, 'h', '3'}}, @@ -26,7 +26,7 @@ int main() site.page("/", {.body = "

hello

"}); - std::println("crawler --dns-server {} --ca-file {} localhost", site.dns_address(), + std::println("http-probe --dns-server {} --ca-file {} localhost", site.dns_address(), site.ca_file().string()); std::fflush(stdout); diff --git a/test/prober_test.cpp b/test/prober_test.cpp index a7b3896..9d701c7 100644 --- a/test/prober_test.cpp +++ b/test/prober_test.cpp @@ -22,9 +22,9 @@ #include #include -using namespace crawler; +using namespace http_probe; using namespace std::chrono_literals; -using crawler::test::Site; +using http_probe::test::Site; using testing::Contains; using testing::ElementsAre; using testing::HasSubstr; @@ -178,7 +178,7 @@ TEST_F(Probe, RequestsTheHostAtThePortOfItsRecord) ASSERT_EQ(site.requests().size(), 1u); EXPECT_EQ(site.requests()[0].authority, "localhost") << "the host, not where it went"; - EXPECT_THAT(site.requests()[0].user_agent, HasSubstr("crawler/")); + EXPECT_THAT(site.requests()[0].user_agent, HasSubstr("http-probe/")); } TEST_F(Probe, RequestsNothingButTheRoot) diff --git a/test/routes_test.cpp b/test/routes_test.cpp index a8e30d1..2db8e71 100644 --- a/test/routes_test.cpp +++ b/test/routes_test.cpp @@ -6,7 +6,7 @@ #include #include -using namespace crawler; +using namespace http_probe; using testing::ElementsAre; using testing::HasSubstr; using testing::IsEmpty; diff --git a/test/site.cpp b/test/site.cpp index 7906bf8..85daecf 100644 --- a/test/site.cpp +++ b/test/site.cpp @@ -12,14 +12,14 @@ namespace http = boost::beast::http; -namespace crawler::test +namespace http_probe::test { // ================================================================================================= Site::Site(const anyhttp::Executor& executor) : requests_(std::make_shared>()) { - auto directory = (std::filesystem::temp_directory_path() / "crawler-test-XXXXXX").string(); + auto directory = (std::filesystem::temp_directory_path() / "http-probe-test-XXXXXX").string(); if (!mkdtemp(directory.data())) throw std::runtime_error("mkdtemp"); directory_ = directory; @@ -102,4 +102,4 @@ void Site::stop() // ================================================================================================= -} // namespace crawler::test +} // namespace http_probe::test diff --git a/test/site.hpp b/test/site.hpp index 88818d2..8e0b5d4 100644 --- a/test/site.hpp +++ b/test/site.hpp @@ -1,5 +1,5 @@ // -// A web site with DNS of its own, all on loopback, for a crawler to be pointed at: an anyhttp +// A web site with DNS of its own, all on loopback, for the program to be pointed at: an anyhttp // server on one port, for every protocol, and a DNS server with whatever zone the test wants. The // server has a certificate for "localhost", the one name that also resolves without our DNS server, // which anyhttp's own resolver needs for HTTP/3. @@ -20,7 +20,7 @@ #include #include -namespace crawler::test +namespace http_probe::test { // ================================================================================================= @@ -82,4 +82,4 @@ class Site // ================================================================================================= -} // namespace crawler::test +} // namespace http_probe::test From 365f651a81307db37ba5b07b911d416b3230eacb Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Mon, 2 Nov 2026 22:25:08 +0000 Subject: [PATCH 05/27] chore: connect to the hosts of a list that offer HTTP/3 anyhttp's client has TLS on HTTP/3 only, so that is what there is to connect to. For a list like c-ares-async's 999 names, each host now has an outcome: answered, failed, HTTP/3 out of reach on a target that can't be used, without HTTP/3, or lookup failed, which is told apart as it says nothing about the host. Hosts without HTTP/3 are only printed with --verbose, and the counts come at the end. Hosts are worked on 16 at a time by default, with --concurrency, so that one that does not answer does not hold up the others. The resolver in the devcontainer answers with no data for HTTPS records, which looks just like no host having any, so a run that finds none says so. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- README.md | 49 ++++++++++-------- src/main.cpp | 116 ++++++++++++++++++++++++++++++++++--------- src/prober.cpp | 42 +++++++++++++++- src/prober.hpp | 40 +++++++++++---- src/routes.cpp | 3 ++ src/routes.hpp | 2 + test/CMakeLists.txt | 5 ++ test/prober_test.cpp | 115 +++++++++++++++++++++++++++++++++++++----- test/routes_test.cpp | 40 +++++++++++++++ 10 files changed, 343 insertions(+), 71 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8b68134..71a3adb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,7 +11,7 @@ These hold for all code in this repository, including tests and experiments. - **One request per host, and nothing more.** c-ares-async's lists of hosts are fine as input here, which is what makes this rule matter: they have 1000 mixed sites, some adult or spam-looking. So - nothing is followed: no links and no redirects. + nothing is followed: no links and no redirects. Only hosts that offer HTTP/3 are connected to. - **Tests connect to local servers on loopback only.** - **If the program ever crawls, it refuses to run without an explicit host allowlist,** with robots.txt and per-host rate limits built in from the start, not added later. diff --git a/README.md b/README.md index f355793..2a597a5 100644 --- a/README.md +++ b/README.md @@ -11,35 +11,40 @@ Requests hosts the way their HTTPS records say, on [anyhttp](https://github.com/ ## What it does ```sh -http-probe example.com www.example.org -http-probe --file hostnames.txt +http-probe --dns-server 1.1.1.1 cloudflare.com +http-probe --dns-server 1.1.1.1 --file hostnames.txt ``` -It requests `https:///` once for each host, in turn, and prints the status, the route, the time -it took and the header fields of the response, along with the host's HTTPS records. A redirect is -reported and not followed, and nothing else on the host is requested, `robots.txt` included. -`http-probe` without arguments lists the options. The exit code is 0 if every host answered, whatever -the status. +It looks up the HTTPS records of each host in DNS ([RFC 9460](https://www.rfc-editor.org/rfc/rfc9460)), +and connects to those that offer HTTP/3. From each of them it requests `https:///` once, and +prints the status, the route, the time it took and the header fields of the response, along with the +host's records. A redirect is reported and not followed, and nothing else on the host is requested. -How to connect is up to the host's HTTPS records in DNS ([RFC 9460](https://www.rfc-editor.org/rfc/rfc9460)): -aliases are followed, a record has to be compatible, and the lowest priority goes first. The port -comes from the record. +HTTP/3 is what it takes because anyhttp's client has TLS on HTTP/3 only, so far. There is a line of +output for every host that offers it and every lookup that failed, and `--verbose` adds the others. +The counts at the end look like this, on stderr: -What can be used of that is limited by what anyhttp's client does so far: +``` +12 answered, 1 failed, 0 with HTTP/3 out of reach, 180 without HTTP/3, 2 lookups failed +``` + +| The host | Outcome | +|-----------------------------------------------|---------------------------------------------| +| has a record with `h3` in its ALPN set | answered or failed, to the port of the record | +| has that only on a target that is not itself | out of reach: anyhttp can't connect to one name and verify another | +| has records without `h3`, or none at all | without HTTP/3, not connected to | +| does not exist | without HTTP/3, not connected to | +| gets no answer from DNS | lookup failed, so there is no telling | -| The host has | What happens | -|-------------------------------------------|-----------------------------------------------| -| a record with `h3` in its ALPN set | HTTP/3, to the port of the record | -| a record without `h3` | rejected: needs TLS over TCP | -| a record with a target that is not itself | rejected: can't verify one name and connect to another | -| no record | rejected: needs TLS over TCP | +Aliases are followed, a record has to be compatible, and the lowest priority goes first. Over HTTP/3, +the address comes from anyhttp's own resolver and not from c-ares, so `--dns-server` and the address +hints in the record do not apply to it. -Every rejection is printed with its reason. Over HTTP/3, the address comes from anyhttp's own resolver -and not from c-ares, so `--dns-server` and the address hints in the record do not apply to it. +**Use `--dns-server`.** The resolver in the devcontainer answers with no data for HTTPS records, even +where they exist, which looks just like no host having any. -The resolver in the devcontainer may answer with no data for HTTPS records, even where they exist -(see c-ares-async's [example](https://github.com/pgit/c-ares-async/blob/master/example/resolve/README.md)). -`--dns-server` is what gets around that. +**The program can take a while to end.** If a host with HTTP/3 did not answer, anyhttp goes on sending +to it for 30 seconds after the timeout. The counts are printed when the work is done, ahead of that. To see it work without going anywhere, `demo_site` runs a site and a DNS server on loopback, and prints the command to request it with: diff --git a/src/main.cpp b/src/main.cpp index f6c0cc3..7d2c1a4 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -10,9 +10,12 @@ #include #include +#include +#include #include #include #include +#include #include #include #include @@ -25,19 +28,42 @@ namespace constexpr std::string_view usage = R"(usage: http-probe [options] HOST... http-probe [options] --file LIST -Requests https://HOST/ once for each host name and prints the status and header fields of the -response. How to connect is up to the host's HTTPS records in DNS (RFC 9460). Redirects are reported -and not followed, and nothing else on the host is requested. +Looks up the HTTPS records of each host (RFC 9460), and requests https://HOST/ once from those that +offer HTTP/3, printing the status and header fields of the response. Redirects are reported and not +followed, and nothing else on the host is requested. --file LIST read the hosts from a file, one per line, as names or as URLs - --dns-server ADDR the DNS server to ask, like 192.0.2.1 or 192.0.2.1:5353; default: the system's + --dns-server ADDR the DNS server to ask, like 192.0.2.1 or 192.0.2.1:5353; default: the system's, + which may well not answer for HTTPS records --ca-file PEM the CA certificates to verify servers with; default: the system's + --concurrency N how many hosts to work on at the same time; default: 16 --timeout-ms N how long one request may take; default: 10000 + --verbose also print the hosts without HTTP/3, and why + +The exit code is 0 if every host with HTTP/3 answered, whatever the status, and every lookup got an +answer. )"; -http_probe::Config parse_arguments(int argc, char* argv[]) +struct Arguments { http_probe::Config config; + bool verbose = false; +}; + +template +T number(std::string_view option, std::string_view text) +{ + T value{}; + const auto [end, ec] = std::from_chars(text.data(), text.data() + text.size(), value); + if (ec != std::errc{} || end != text.data() + text.size()) + throw std::invalid_argument(std::format("{}: '{}' is not a number", option, text)); + return value; +} + +Arguments parse_arguments(int argc, char* argv[]) +{ + Arguments arguments; + auto& config = arguments.config; for (int i = 1; i < argc; ++i) { const std::string_view argument = argv[i]; @@ -47,6 +73,12 @@ http_probe::Config parse_arguments(int argc, char* argv[]) continue; } + if (argument == "--verbose") + { + arguments.verbose = true; + continue; + } + if (i + 1 == argc) throw std::invalid_argument(std::format("{} needs a value, or is unknown", argument)); const std::string_view value = argv[++i]; @@ -65,28 +97,39 @@ http_probe::Config parse_arguments(int argc, char* argv[]) config.dns_server = value; else if (argument == "--ca-file") config.tls_ca_file = value; + else if (argument == "--concurrency") + config.concurrency = number(argument, value); else if (argument == "--timeout-ms") - { - unsigned milliseconds = 0; - const auto [end, ec] = - std::from_chars(value.data(), value.data() + value.size(), milliseconds); - if (ec != std::errc{} || end != value.data() + value.size()) - throw std::invalid_argument(std::format("{}: '{}' is not a number", argument, value)); - config.timeout = std::chrono::milliseconds(milliseconds); - } + config.timeout = std::chrono::milliseconds(number(argument, value)); else throw std::invalid_argument(std::format("unknown option '{}'", argument)); } - return config; + return arguments; } +using Outcome = http_probe::Result::Outcome; + void print(const http_probe::Result& result) { - if (result.error.empty()) + switch (result.outcome) + { + case Outcome::answered: std::println("{}: {} [{}] {} bytes in {}", result.host, result.status, result.route, result.body_size, result.duration); - else + break; + case Outcome::failed: std::println("{}: failed: {}", result.host, result.error); + break; + case Outcome::out_of_reach: + std::println("{}: HTTP/3 out of reach", result.host); + break; + case Outcome::no_http3: + std::println("{}: no HTTP/3", result.host); + break; + case Outcome::lookup_failed: + std::println("{}: lookup failed", result.host); + break; + } for (const auto& record : result.records) std::println(" record: {}", http_probe::to_text(record)); @@ -94,6 +137,8 @@ void print(const http_probe::Result& result) std::println(" note: {}", note); for (const auto& [name, value] : result.fields) std::println(" {}: {}", name, value); + + std::fflush(stdout); // a run can take a while, and it is worth watching } } // namespace @@ -102,19 +147,42 @@ int main(int argc, char* argv[]) { spdlog::set_level(spdlog::level::err); // anyhttp logs through the default logger - bool all_answered = true; - const auto report = [&](const http_probe::Result& result) { - all_answered = all_answered && result.error.empty(); - print(result); - }; + std::map counts; + std::size_t with_records = 0; try { + auto [config, verbose] = parse_arguments(argc, argv); + const bool own_dns_server = !config.dns_server.empty(); + + const auto report = [&, verbose](const http_probe::Result& result) { + ++counts[result.outcome]; + if (!result.records.empty() || result.aliased) + ++with_records; + if (verbose || result.outcome != Outcome::no_http3) + print(result); + }; + boost::asio::io_context context; - http_probe::Prober prober(context.get_executor(), parse_arguments(argc, argv), report); - boost::asio::co_spawn(context, prober.run(), [](const std::exception_ptr& ex) { + http_probe::Prober prober(context.get_executor(), std::move(config), report); + boost::asio::co_spawn(context, prober.run(), [&](const std::exception_ptr& ex) { if (ex) std::rethrow_exception(ex); + + // + // Here and not after run(): a host that did not answer can keep the context from running + // dry for a while, see the README, and this is what the wait would be for. + // + const auto count = [&](Outcome outcome) { return counts[outcome]; }; + std::println(stderr, + "{} answered, {} failed, {} with HTTP/3 out of reach, {} without HTTP/3, " + "{} lookups failed", + count(Outcome::answered), count(Outcome::failed), + count(Outcome::out_of_reach), count(Outcome::no_http3), + count(Outcome::lookup_failed)); + if (with_records == 0 && !own_dns_server) + std::println(stderr, "http-probe: none of the hosts has an HTTPS record. Not every " + "resolver answers for them, so try --dns-server."); }); context.run(); } @@ -129,5 +197,5 @@ int main(int argc, char* argv[]) return 1; } - return all_answered ? 0 : 1; + return counts[Outcome::failed] == 0 && counts[Outcome::lookup_failed] == 0 ? 0 : 1; } diff --git a/src/prober.cpp b/src/prober.cpp index a678896..52b32cf 100644 --- a/src/prober.cpp +++ b/src/prober.cpp @@ -2,6 +2,9 @@ #include +#include +#include +#include #include #include #include @@ -76,6 +79,8 @@ Prober::Prober(anyhttp::Executor executor, Config config, Observer observer) { if (config_.hosts.empty()) throw std::invalid_argument("nothing to do: no host names given"); + if (config_.concurrency == 0) + throw std::invalid_argument("a concurrency of 0 would never get anywhere"); std::set seen; for (const auto& host : config_.hosts) @@ -94,8 +99,28 @@ Prober::Prober(anyhttp::Executor executor, Config config, Observer observer) anyhttp::Task Prober::run() { - for (const auto& host : hosts_) + namespace asio = boost::asio; + using Worker = decltype(asio::co_spawn(executor_, work(), asio::deferred)); + + std::vector workers; + for (std::size_t i = 0; i < std::min(config_.concurrency, hosts_.size()); ++i) + workers.push_back(asio::co_spawn(executor_, work(), asio::deferred)); + + auto [order, exceptions] = co_await asio::experimental::make_parallel_group(std::move(workers)) + .async_wait(asio::experimental::wait_for_all(), asio::deferred); + + for (const auto& exception : exceptions) + if (exception) + std::rethrow_exception(exception); +} + +anyhttp::Task Prober::work() +{ + while (next_host_ < hosts_.size()) + { + const auto& host = hosts_[next_host_++]; observer_(co_await probe(host)); + } } // ------------------------------------------------------------------------------------------------- @@ -121,8 +146,11 @@ anyhttp::Task Prober::resolve_service(const std::string& host) { if (ec != c_ares_async::make_ares_error(ARES_ENODATA) && ec != c_ares_async::make_ares_error(ARES_ENOTFOUND)) + { + found.lookup_failed = true; found.notes.push_back( std::format("HTTPS lookup of '{}': {}", found.owner, ec.message())); + } co_return found; } @@ -165,12 +193,21 @@ anyhttp::Task Prober::probe(const std::string& host) Result result{.host = host}; const auto found = co_await resolve_service(host); + result.aliased = found.aliased; result.records = found.records; const auto plan = plan_routes(host, found); result.notes = plan.rejected; - result.error = "no way to connect"; + if (plan.routes.empty()) + { + result.outcome = plan.h3_out_of_reach ? Result::Outcome::out_of_reach + : found.lookup_failed ? Result::Outcome::lookup_failed + : Result::Outcome::no_http3; + co_return result; + } + + result.outcome = Result::Outcome::failed; for (const auto& route : plan.routes) { const auto start = std::chrono::steady_clock::now(); @@ -182,6 +219,7 @@ anyhttp::Task Prober::probe(const std::string& host) continue; } + result.outcome = Result::Outcome::answered; result.error.clear(); result.status = static_cast(response.message.result_int()); result.route = route.to_string(); diff --git a/src/prober.hpp b/src/prober.hpp index 8eeb423..026f2c3 100644 --- a/src/prober.hpp +++ b/src/prober.hpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -27,6 +28,9 @@ struct Config /// The host names to request, exactly as given: these are the only names that are connected to. std::vector hosts; + /// How many hosts to work on at the same time. + std::size_t concurrency = 16; + /// How long the request may take on one route, from connecting to the end of the response. std::chrono::milliseconds timeout{10000}; @@ -51,24 +55,38 @@ std::vector read_hosts(std::istream& list, std::vector /// What came of one host. struct Result { + enum class Outcome + { + answered, ///< it has HTTP/3, and a response arrived, whatever its status + failed, ///< it has HTTP/3, and no response could be had + out_of_reach, ///< it has HTTP/3, but on a target that can't be used, see plan_routes() + no_http3, ///< its records offer no HTTP/3, or it has none, or the name does not exist + lookup_failed, ///< DNS gave no answer, so there is no telling + }; + std::string host; + Outcome outcome = Outcome::no_http3; + bool aliased = false; ///< its records are those of another name std::vector records; ///< the records that were considered - int status = 0; ///< the status of the response, 0 if there was none - std::string route; ///< what the response came over - std::vector> fields; ///< the header fields of the response - std::size_t body_size = 0; - std::chrono::milliseconds duration{}; ///< of the attempt that gave the response + int status = 0; ///< with \c answered + std::string route; ///< with \c answered: what the response came over + std::vector> fields; ///< with \c answered + std::size_t body_size = 0; ///< with \c answered + std::chrono::milliseconds duration{}; ///< with \c answered: of the attempt that got through - std::string error; ///< why there is no response + std::string error; ///< with \c failed: what went wrong on the last route std::vector notes; ///< records that could not be used, routes that did not work }; /** - * Requests each of Config::hosts once and in turn, using the resolver of c-ares-async to find out - * how to connect to it (see plan_routes()). A response is what comes back, whatever its status: a - * redirect is reported and not followed, nothing in the body is looked at, and nothing else on the - * host is requested. + * Looks up the HTTPS records of each of Config::hosts with the resolver of c-ares-async, and for + * those that offer HTTP/3, sends one request (see plan_routes()). A response is what comes back, + * whatever its status: a redirect is reported and not followed, nothing in the body is looked at, + * and nothing else on the host is requested. + * + * Hosts are worked on Config::concurrency at a time, so results arrive as they are ready. All of + * this is on the one executor, which is also what calls the observer. */ class Prober { @@ -89,6 +107,7 @@ class Prober std::string error; }; + anyhttp::Task work(); anyhttp::Task resolve_service(const std::string& host); anyhttp::Task probe(const std::string& host); anyhttp::Task attempt(const Route& route, const std::string& host); @@ -97,6 +116,7 @@ class Prober Config config_; Observer observer_; std::vector hosts_; + std::size_t next_host_ = 0; c_ares_async::resolver resolver_; }; diff --git a/src/routes.cpp b/src/routes.cpp index 75bea1e..be5116c 100644 --- a/src/routes.cpp +++ b/src/routes.cpp @@ -133,9 +133,12 @@ Plan plan_routes(std::string_view host, const ServiceRecords& found) if (!service.offers("h3")) plan.rejected.push_back(std::format("{}: {}", text, no_tls_over_tcp)); else if (target != host) + { + plan.h3_out_of_reach = true; plan.rejected.push_back(std::format( "{}: target '{}' is not the host, and anyhttp can only verify the name it connects to", text, target)); + } else plan.routes.push_back( Route{.host = target, .port = service.port.value_or(443), .source = text}); diff --git a/src/routes.hpp b/src/routes.hpp index d702b2d..63cce51 100644 --- a/src/routes.hpp +++ b/src/routes.hpp @@ -31,6 +31,7 @@ struct ServiceRecords bool aliased = false; ///< at least one AliasMode record was followed std::vector records; ///< the ServiceMode records of \c owner std::vector notes; ///< what went wrong on the way, none of which is fatal + bool lookup_failed = false; ///< no answer, which is not to say that there are no records }; /// One way of connecting, which is HTTP/3 so far. The TLS name and the authority stay the host's. @@ -48,6 +49,7 @@ struct Plan { std::vector routes; ///< in the order to try them; empty if there is no usable way std::vector rejected; ///< what the records offered, or we knew of, and why not + bool h3_out_of_reach = false; ///< a record offers HTTP/3, but on a target that can't be used }; /** diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 991e284..0943de7 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -57,3 +57,8 @@ set_tests_properties(http_probe_refuses_a_url PROPERTIES add_test(NAME http_probe_refuses_a_file_it_can_not_read COMMAND http-probe --file /nonexistent/list) set_tests_properties(http_probe_refuses_a_file_it_can_not_read PROPERTIES PASS_REGULAR_EXPRESSION "can't read") + +add_test(NAME http_probe_refuses_a_concurrency_of_zero + COMMAND http-probe --concurrency 0 localhost) +set_tests_properties(http_probe_refuses_a_concurrency_of_zero PROPERTIES + PASS_REGULAR_EXPRESSION "concurrency of 0") diff --git a/test/prober_test.cpp b/test/prober_test.cpp index 9d701c7..b500f0f 100644 --- a/test/prober_test.cpp +++ b/test/prober_test.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -121,6 +122,14 @@ TEST_F(Probe, RefusesToRunWithoutHosts) finish(); } +TEST_F(Probe, RefusesAConcurrencyOfZero) +{ + auto configuration = config(); + configuration.concurrency = 0; + EXPECT_THROW(probe(std::move(configuration)), std::invalid_argument); + finish(); +} + TEST_F(Probe, RefusesAnythingButAHostName) { for (const auto* host : {"https://localhost/", "localhost/x", "localhost:443", "*.localhost", @@ -167,7 +176,7 @@ TEST_F(Probe, RequestsTheHostAtThePortOfItsRecord) ASSERT_EQ(results.size(), 1u); const auto& result = results[0]; - EXPECT_EQ(result.error, "") << notes(result); + EXPECT_EQ(result.outcome, Result::Outcome::answered) << notes(result); EXPECT_EQ(result.host, "localhost"); EXPECT_EQ(result.status, 200); EXPECT_EQ(result.route, std::format("h3 localhost:{}", site.port())); @@ -219,7 +228,7 @@ TEST_F(Probe, ReportsAnErrorStatusAsAResponse) ASSERT_EQ(results.size(), 1u); EXPECT_EQ(results[0].status, 503); - EXPECT_EQ(results[0].error, ""); + EXPECT_EQ(results[0].outcome, Result::Outcome::answered); } TEST_F(Probe, RequestsEachHostOnce) @@ -242,11 +251,92 @@ TEST_F(Probe, GoesOnAfterAHostThatFails) ASSERT_EQ(results.size(), 2u); EXPECT_EQ(results[0].host, "nowhere.example.test"); - EXPECT_NE(results[0].error, ""); + EXPECT_EQ(results[0].outcome, Result::Outcome::no_http3); EXPECT_EQ(results[1].host, "localhost"); EXPECT_EQ(results[1].status, 200); } +// ================================================================================================= +// A list of hosts +// ================================================================================================= + +TEST_F(Probe, ConnectsToTheHostsWithHttp3AndToNoOthers) +{ + // + // The other names exist, with records or without, and their addresses are the server's. It takes + // no more than connecting to any of them to show up in its requests. + // + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.dns().add_address("h2.example.test", "127.0.0.1"); + site.dns().add_https("h2.example.test", 1, ".", {alpn_h2}); + site.dns().add_address("plain.example.test", "127.0.0.1"); + site.page("/", {.body = "hello"}); + + auto configuration = config({"h2.example.test", "plain.example.test", "localhost"}); + configuration.concurrency = 1; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 3u); + EXPECT_EQ(results[0].outcome, Result::Outcome::no_http3); + EXPECT_EQ(results[1].outcome, Result::Outcome::no_http3); + EXPECT_EQ(results[2].outcome, Result::Outcome::answered) << notes(results[2]); + EXPECT_THAT(site.targets(), ElementsAre("/")); + EXPECT_EQ(site.requests()[0].authority, "localhost"); +} + +TEST_F(Probe, ReportsEachHostOnceHoweverManyThereAre) +{ + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.page("/", {.body = "hello"}); + + std::vector hosts{"localhost"}; + for (int i = 0; i < 100; ++i) + hosts.push_back(std::format("host{}.example.test", i)); + + auto configuration = config(hosts); + configuration.concurrency = 7; + probe(std::move(configuration)); + + std::set reported; + for (const auto& result : results) + reported.insert(result.host); + EXPECT_EQ(results.size(), hosts.size()); + EXPECT_EQ(reported.size(), hosts.size()); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, DoesNotLetASlowHostHoldUpTheOthers) +{ + boost::asio::ip::udp::socket silent(context, {boost::asio::ip::udp::v6(), 0}); + abandon_quic_sessions = true; + site.dns().add_https("localhost", 1, ".", h3_at(silent.local_endpoint().port())); + + auto configuration = config({"localhost", "nowhere.example.test"}); + configuration.timeout = 200ms; + configuration.concurrency = 2; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 2u); + EXPECT_EQ(results[0].host, "nowhere.example.test"); + EXPECT_EQ(results[1].host, "localhost"); +} + +TEST_F(Probe, WorksOnOneHostAtATimeIfToldTo) +{ + boost::asio::ip::udp::socket silent(context, {boost::asio::ip::udp::v6(), 0}); + abandon_quic_sessions = true; + site.dns().add_https("localhost", 1, ".", h3_at(silent.local_endpoint().port())); + + auto configuration = config({"localhost", "nowhere.example.test"}); + configuration.timeout = 200ms; + configuration.concurrency = 1; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 2u); + EXPECT_EQ(results[0].host, "localhost"); + EXPECT_EQ(results[1].host, "nowhere.example.test"); +} + // ================================================================================================= // How the HTTPS records decide about the connection // ================================================================================================= @@ -295,8 +385,7 @@ TEST_F(Probe, FailsWithoutARecord) probe(config()); ASSERT_EQ(results.size(), 1u); - EXPECT_EQ(results[0].status, 0); - EXPECT_EQ(results[0].error, "no way to connect"); + EXPECT_EQ(results[0].outcome, Result::Outcome::no_http3); EXPECT_THAT(notes(results[0]), HasSubstr("TLS over TCP")); EXPECT_THAT(site.requests(), IsEmpty()); } @@ -312,7 +401,7 @@ TEST_F(Probe, DoesNotConnectToATargetOtherThanTheHost) probe(config()); ASSERT_EQ(results.size(), 1u); - EXPECT_EQ(results[0].error, "no way to connect"); + EXPECT_EQ(results[0].outcome, Result::Outcome::out_of_reach); EXPECT_THAT(notes(results[0]), HasSubstr("elsewhere.example.test")); EXPECT_THAT(site.requests(), IsEmpty()); } @@ -329,7 +418,7 @@ TEST_F(Probe, RefusesACertificateItDoesNotTrust) probe(std::move(configuration)); ASSERT_EQ(results.size(), 1u); - EXPECT_EQ(results[0].status, 0); + EXPECT_EQ(results[0].outcome, Result::Outcome::failed); EXPECT_THAT(results[0].error, HasSubstr("connect")); EXPECT_THAT(site.requests(), IsEmpty()) << "no request survives the handshake"; } @@ -359,7 +448,7 @@ TEST_F(Probe, RecognizesAnAliasLoop) probe(config()); ASSERT_EQ(results.size(), 1u); - EXPECT_EQ(results[0].error, "no way to connect"); + EXPECT_EQ(results[0].outcome, Result::Outcome::no_http3); EXPECT_THAT(notes(results[0]), HasSubstr("too long or has a loop")); } @@ -372,7 +461,9 @@ TEST_F(Probe, IgnoresServiceRecordsNextToAnAlias) probe(config()); ASSERT_EQ(results.size(), 1u); - EXPECT_EQ(results[0].status, 0) << "the alias has no records of its own"; + EXPECT_EQ(results[0].outcome, Result::Outcome::no_http3) + << "the alias has no records of its own"; + EXPECT_TRUE(results[0].aliased); EXPECT_THAT(site.requests(), IsEmpty()); } @@ -391,7 +482,7 @@ TEST_F(Probe, GivesUpOnAServerThatDoesNotAnswer) probe(std::move(configuration)); ASSERT_EQ(results.size(), 1u); - EXPECT_EQ(results[0].status, 0); + EXPECT_EQ(results[0].outcome, Result::Outcome::failed); EXPECT_THAT(results[0].error, HasSubstr("timed out")); } @@ -400,7 +491,7 @@ TEST_F(Probe, ReportsANameThatDoesNotExist) probe(config({"nowhere.example.test"})); ASSERT_EQ(results.size(), 1u); - EXPECT_EQ(results[0].error, "no way to connect"); + EXPECT_EQ(results[0].outcome, Result::Outcome::no_http3); EXPECT_THAT(site.requests(), IsEmpty()); } @@ -412,7 +503,7 @@ TEST_F(Probe, ReportsAFailedLookup) probe(config()); ASSERT_EQ(results.size(), 1u); - EXPECT_EQ(results[0].error, "no way to connect"); + EXPECT_EQ(results[0].outcome, Result::Outcome::lookup_failed) << "not the same as no HTTP/3"; EXPECT_THAT(notes(results[0]), HasSubstr("HTTPS lookup")); EXPECT_THAT(site.requests(), IsEmpty()); } diff --git a/test/routes_test.cpp b/test/routes_test.cpp index 2db8e71..c6e4fd6 100644 --- a/test/routes_test.cpp +++ b/test/routes_test.cpp @@ -108,6 +108,46 @@ TEST(Plan, RejectsATargetOtherThanTheHost) EXPECT_THAT(plan.routes, IsEmpty()); ASSERT_EQ(plan.rejected.size(), 1u); EXPECT_THAT(plan.rejected[0], HasSubstr("cdn.example.net")); + EXPECT_TRUE(plan.h3_out_of_reach); +} + +TEST(Plan, DoesNotCallItOutOfReachWithoutHttp3) +{ + const auto plan = plan_routes("example.com", found({record(1, "cdn.example.net", {"alpn=h2"})})); + + EXPECT_FALSE(plan.h3_out_of_reach); +} + +TEST(Plan, UsesTheRecordItCanOfAMixedSet) +{ + // + // As facebook.com has it. + // + const auto plan = plan_routes( + "example.com", + found({record(1, ".", {"alpn=h2,h3"}), record(2, "fallback.example.net", {"alpn=h2,h3"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443")); + ASSERT_EQ(plan.rejected.size(), 1u); + EXPECT_THAT(plan.rejected[0], HasSubstr("fallback.example.net")); +} + +TEST(Plan, DoesNotTakeSomethingThatOnlyStartsWithH3) +{ + const auto plan = plan_routes("example.com", found({record(1, ".", {"alpn=h3-29,h2"})})); + + EXPECT_THAT(plan.routes, IsEmpty()); +} + +TEST(Plan, IgnoresWhatItDoesNotKnowUnlessItIsMandatory) +{ + // + // key13930 is real, seen on one of the hosts in c-ares-async's list. + // + const auto plan = + plan_routes("example.com", found({record(1, ".", {"alpn=h3", "key13930=01", "ech=dead"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443")); } TEST(Plan, TakesAnExplicitTargetThatIsTheHost) From f98c9e3031354992e239b1b1d76123a93eebafa2 Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Mon, 2 Nov 2026 22:33:43 +0000 Subject: [PATCH 06/27] chore: add colours and logging Colours as in c-ares-async's example: host names in yellow, IPv4 in magenta and IPv6 in blue, for terminals only unless there is --color. On top of that, the status by its class and the outcomes that went wrong. The formatting is in output.cpp, where it can be tested without a terminal. -v logs to stderr through spdlog's default logger, which is where anyhttp logs already go: its requests and responses, the queries of c-ares, and the steps of the program in between. -vv adds all of QUIC. So --verbose no longer lists the hosts without HTTP/3, and --all does that now. The test site tries again when it finds a port taken. Its servers need the same number on TCP and UDP, which failed about once in 30 runs of the tests in parallel, with another test holding the other half. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 2 +- README.md | 9 +++- src/main.cpp | 98 +++++++++++++++++------------------ src/output.cpp | 121 +++++++++++++++++++++++++++++++++++++++++++ src/output.hpp | 29 +++++++++++ src/prober.cpp | 29 +++++++++++ test/CMakeLists.txt | 2 +- test/output_test.cpp | 112 +++++++++++++++++++++++++++++++++++++++ test/prober_test.cpp | 32 ++++++++++++ test/site.cpp | 41 ++++++++++++--- test/site.hpp | 2 + 11 files changed, 416 insertions(+), 61 deletions(-) create mode 100644 src/output.cpp create mode 100644 src/output.hpp create mode 100644 test/output_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a1ad48a..a7ff549 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,7 +61,7 @@ FetchContent_MakeAvailable(anyhttp c_ares_async) # set(http_probe_warnings -Wall -Wextra -Werror -Wno-missing-designated-field-initializers) -add_library(http_probe_core STATIC src/prober.cpp src/routes.cpp) +add_library(http_probe_core STATIC src/output.cpp src/prober.cpp src/routes.cpp) target_include_directories(http_probe_core PUBLIC src) target_link_libraries(http_probe_core PUBLIC anyhttp::anyhttp c_ares_async::c_ares_async) target_compile_options(http_probe_core PRIVATE ${http_probe_warnings}) diff --git a/README.md b/README.md index 2a597a5..73a0283 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,8 @@ and connects to those that offer HTTP/3. From each of them it requests `https:// prints the status, the route, the time it took and the header fields of the response, along with the host's records. A redirect is reported and not followed, and nothing else on the host is requested. -HTTP/3 is what it takes because anyhttp's client has TLS on HTTP/3 only, so far. There is a line of -output for every host that offers it and every lookup that failed, and `--verbose` adds the others. +HTTP/3 is what it takes because anyhttp's client has TLS on HTTP/3 only, so far. There is output for +every host that offers it and every lookup that failed, and `--all` adds the others. The counts at the end look like this, on stderr: ``` @@ -40,6 +40,11 @@ Aliases are followed, a record has to be compatible, and the lowest priority goe the address comes from anyhttp's own resolver and not from c-ares, so `--dns-server` and the address hints in the record do not apply to it. +**To see what is going on,** `-v` logs to stderr: the requests and responses from anyhttp, the queries +from c-ares and the steps in between. `-vv` adds all of QUIC. Hosts run at the same time mix in the +log, so `--concurrency 1` helps. Colours are for terminals only unless there is `--color`, except that +anyhttp puts them into some of its log messages regardless. + **Use `--dns-server`.** The resolver in the devcontainer answers with no data for HTTPS records, even where they exist, which looks just like no host having any. diff --git a/src/main.cpp b/src/main.cpp index 7d2c1a4..95106e3 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,13 +1,17 @@ // // http-probe [options] HOST... // +#include "output.hpp" #include "prober.hpp" #include #include +#include #include +#include + #include #include #include @@ -15,7 +19,6 @@ #include #include #include -#include #include #include #include @@ -38,7 +41,11 @@ followed, and nothing else on the host is requested. --ca-file PEM the CA certificates to verify servers with; default: the system's --concurrency N how many hosts to work on at the same time; default: 16 --timeout-ms N how long one request may take; default: 10000 - --verbose also print the hosts without HTTP/3, and why + --all also print the hosts without HTTP/3, and why + --color colour the output even where it does not go to a terminal, e.g. into 'less -R' + -v, --verbose log to stderr what is going on, the requests and responses of anyhttp and the + queries of c-ares among it; -vv for all of QUIC as well, which is a lot. To + make sense of it, use --concurrency 1. The exit code is 0 if every host with HTTP/3 answered, whatever the status, and every lookup got an answer. @@ -47,7 +54,9 @@ answer. struct Arguments { http_probe::Config config; - bool verbose = false; + bool all = false; + bool color = false; + int verbosity = 0; }; template @@ -73,9 +82,19 @@ Arguments parse_arguments(int argc, char* argv[]) continue; } - if (argument == "--verbose") + if (argument == "--all") + { + arguments.all = true; + continue; + } + if (argument == "--color") { - arguments.verbose = true; + arguments.color = true; + continue; + } + if (argument == "-v" || argument == "--verbose" || argument == "-vv") + { + arguments.verbosity += argument == "-vv" ? 2 : 1; continue; } @@ -107,60 +126,47 @@ Arguments parse_arguments(int argc, char* argv[]) return arguments; } -using Outcome = http_probe::Result::Outcome; - -void print(const http_probe::Result& result) +// +// Everywhere that anyhttp and c-ares log to, as both use the default logger. On stderr, so that the +// results can go somewhere by themselves. +// +void set_up_logging(int verbosity, bool color) { - switch (result.outcome) - { - case Outcome::answered: - std::println("{}: {} [{}] {} bytes in {}", result.host, result.status, result.route, - result.body_size, result.duration); - break; - case Outcome::failed: - std::println("{}: failed: {}", result.host, result.error); - break; - case Outcome::out_of_reach: - std::println("{}: HTTP/3 out of reach", result.host); - break; - case Outcome::no_http3: - std::println("{}: no HTTP/3", result.host); - break; - case Outcome::lookup_failed: - std::println("{}: lookup failed", result.host); - break; - } - - for (const auto& record : result.records) - std::println(" record: {}", http_probe::to_text(record)); - for (const auto& note : result.notes) - std::println(" note: {}", note); - for (const auto& [name, value] : result.fields) - std::println(" {}: {}", name, value); - - std::fflush(stdout); // a run can take a while, and it is worth watching + spdlog::set_default_logger(spdlog::stderr_color_mt( + "http-probe", color ? spdlog::color_mode::always : spdlog::color_mode::automatic)); + spdlog::set_pattern("%H:%M:%S.%e %^%l%$ %v"); + spdlog::set_level(verbosity >= 2 ? spdlog::level::trace + : verbosity == 1 ? spdlog::level::debug + : spdlog::level::err); } } // namespace int main(int argc, char* argv[]) { - spdlog::set_level(spdlog::level::err); // anyhttp logs through the default logger + using Outcome = http_probe::Result::Outcome; - std::map counts; + http_probe::Counts counts; std::size_t with_records = 0; try { - auto [config, verbose] = parse_arguments(argc, argv); + auto [config, all, color, verbosity] = parse_arguments(argc, argv); + set_up_logging(verbosity, color); + const bool own_dns_server = !config.dns_server.empty(); + const bool color_results = color || isatty(STDOUT_FILENO); + const bool color_summary = color || isatty(STDERR_FILENO); - const auto report = [&, verbose](const http_probe::Result& result) { + const auto report = [&](const http_probe::Result& result) { ++counts[result.outcome]; if (!result.records.empty() || result.aliased) ++with_records; - if (verbose || result.outcome != Outcome::no_http3) - print(result); + if (all || result.outcome != Outcome::no_http3) + { + std::print("{}", http_probe::format_result(result, color_results)); + std::fflush(stdout); // a run can take a while, and it is worth watching + } }; boost::asio::io_context context; @@ -173,13 +179,7 @@ int main(int argc, char* argv[]) // Here and not after run(): a host that did not answer can keep the context from running // dry for a while, see the README, and this is what the wait would be for. // - const auto count = [&](Outcome outcome) { return counts[outcome]; }; - std::println(stderr, - "{} answered, {} failed, {} with HTTP/3 out of reach, {} without HTTP/3, " - "{} lookups failed", - count(Outcome::answered), count(Outcome::failed), - count(Outcome::out_of_reach), count(Outcome::no_http3), - count(Outcome::lookup_failed)); + std::println(stderr, "{}", http_probe::format_summary(counts, color_summary)); if (with_records == 0 && !own_dns_server) std::println(stderr, "http-probe: none of the hosts has an HTTPS record. Not every " "resolver answers for them, so try --dns-server."); diff --git a/src/output.cpp b/src/output.cpp new file mode 100644 index 0000000..50e9c53 --- /dev/null +++ b/src/output.cpp @@ -0,0 +1,121 @@ +#include "output.hpp" + +#include +#include + +namespace http_probe +{ + +namespace +{ + +// ================================================================================================= + +namespace color_of +{ +constexpr std::string_view name = "33"; +constexpr std::string_view v4 = "35"; +constexpr std::string_view v6 = "34"; +constexpr std::string_view good = "32"; +constexpr std::string_view moved = "36"; +constexpr std::string_view bad = "31"; +constexpr std::string_view field = "1;34"; // as anyhttp logs header fields +constexpr std::string_view note = "2"; +} // namespace color_of + +std::string colored(std::string_view text, std::string_view code, bool color) +{ + return color ? std::format("\x1b[{}m{}\x1b[0m", code, text) : std::string(text); +} + +std::string status_text(int status, bool color) +{ + const auto code = status >= 200 && status < 300 ? color_of::good + : status >= 300 && status < 400 ? color_of::moved + : color_of::bad; + return colored(std::to_string(status), code, color); +} + +// +// to_text() with colours: the target, and the values of the address hints. +// +std::string record_text(const c_ares_async::https_record& record, bool color) +{ + auto text = std::format("HTTPS {} {}", record.priority, + record.target.empty() || record.target == "." + ? "." + : colored(record.target, color_of::name, color)); + + for (std::string_view param : record.params) + { + const bool v4 = param.starts_with("ipv4hint="); + if (v4 || param.starts_with("ipv6hint=")) + { + const auto value = param.find('=') + 1; + text += std::format(" {}{}", param.substr(0, value), + colored(param.substr(value), v4 ? color_of::v4 : color_of::v6, color)); + } + else + text += std::format(" {}", param); + } + return text; +} + +// ================================================================================================= + +} // namespace + +std::string format_result(const Result& result, bool color) +{ + using Outcome = Result::Outcome; + + const auto host = colored(result.host, color_of::name, color); + std::string text; + switch (result.outcome) + { + case Outcome::answered: + text = std::format("{}: {} [{}] {} bytes in {}\n", host, status_text(result.status, color), + result.route, result.body_size, result.duration); + break; + case Outcome::failed: + text = + std::format("{}: {}: {}\n", host, colored("failed", color_of::bad, color), result.error); + break; + case Outcome::out_of_reach: + text = std::format("{}: {}\n", host, colored("HTTP/3 out of reach", color_of::moved, color)); + break; + case Outcome::no_http3: + text = std::format("{}: no HTTP/3\n", host); + break; + case Outcome::lookup_failed: + text = std::format("{}: {}\n", host, colored("lookup failed", color_of::bad, color)); + break; + } + + for (const auto& record : result.records) + text += std::format(" record: {}\n", record_text(record, color)); + for (const auto& note : result.notes) + text += colored(std::format(" note: {}", note), color_of::note, color) + "\n"; + for (const auto& [name, value] : result.fields) + text += std::format(" {}: {}\n", colored(name, color_of::field, color), value); + return text; +} + +std::string format_summary(Counts counts, bool color) +{ + using Outcome = Result::Outcome; + + const auto count = [&](Outcome outcome, std::string_view text, std::string_view code) { + const auto line = std::format("{} {}", counts[outcome], text); + return counts[outcome] ? colored(line, code, color) : line; + }; + return std::format( + "{}, {}, {}, {} without HTTP/3, {}", count(Outcome::answered, "answered", color_of::good), + count(Outcome::failed, "failed", color_of::bad), + count(Outcome::out_of_reach, "with HTTP/3 out of reach", color_of::moved), + counts[Outcome::no_http3], count(Outcome::lookup_failed, "lookups failed", color_of::bad)); +} + +// ================================================================================================= + +} // namespace http_probe diff --git a/src/output.hpp b/src/output.hpp new file mode 100644 index 0000000..851fb48 --- /dev/null +++ b/src/output.hpp @@ -0,0 +1,29 @@ +// +// What the program prints, in the colours of c-ares-async's example where it can: host names are +// yellow, and addresses come the way "ip -color address" gives them, IPv4 in magenta and IPv6 in +// blue. Without colour, the text is the same but for the escape sequences. +// +#pragma once + +#include "prober.hpp" + +#include +#include +#include + +namespace http_probe +{ + +// ================================================================================================= + +using Counts = std::map; + +/// All there is to say about one host, a line each and with a line break at the end. +std::string format_result(const Result& result, bool color); + +/// How it went overall, without a line break. +std::string format_summary(Counts counts, bool color); + +// ================================================================================================= + +} // namespace http_probe diff --git a/src/prober.cpp b/src/prober.cpp index 52b32cf..577a36e 100644 --- a/src/prober.cpp +++ b/src/prober.cpp @@ -10,6 +10,8 @@ #include #include +#include + #include #include #include @@ -32,6 +34,26 @@ constexpr std::string_view user_agent = "http-probe/0.0.1 (+https://github.com/p constexpr int max_alias_chain = 8; +// +// The resolver does not write anywhere by itself. Its messages go where those of anyhttp go, which +// is the default logger of spdlog. +// +void log_to_spdlog(c_ares_async::log_level level, std::string_view message) +{ + using enum c_ares_async::log_level; + switch (level) + { + case debug: + return spdlog::debug("{}", message); + case info: + return spdlog::info("{}", message); + case warning: + return spdlog::warn("{}", message); + case error: + return spdlog::error("{}", message); + } +} + // ================================================================================================= } // namespace @@ -91,6 +113,8 @@ Prober::Prober(anyhttp::Executor executor, Config config, Observer observer) hosts_.push_back(std::move(name)); } + resolver_.set_logger(&log_to_spdlog); + if (!config_.dns_server.empty()) if (const auto ec = resolver_.set_servers(config_.dns_server)) throw std::invalid_argument( @@ -141,6 +165,7 @@ anyhttp::Task Prober::resolve_service(const std::string& host) // Outside of any timeout: the image has c-ares 1.34.5, and cancelling a lookup is what // GHSA-6wfj-rwm7-3542 is about. c-ares has timeouts of its own. // + spdlog::debug("{}: asking for the HTTPS records of '{}'", host, found.owner); auto [ec, records] = co_await resolver_.resolve_https(found.owner); if (ec) { @@ -198,6 +223,8 @@ anyhttp::Task Prober::probe(const std::string& host) const auto plan = plan_routes(host, found); result.notes = plan.rejected; + for (const auto& rejected : plan.rejected) + spdlog::debug("{}: rejected: {}", host, rejected); if (plan.routes.empty()) { @@ -210,12 +237,14 @@ anyhttp::Task Prober::probe(const std::string& host) result.outcome = Result::Outcome::failed; for (const auto& route : plan.routes) { + spdlog::debug("{}: connecting by {}, for {}", host, route.to_string(), route.source); const auto start = std::chrono::steady_clock::now(); auto response = co_await attempt(route, host); if (response.ec) { result.error = std::format("[{}] {}", route.to_string(), response.error); result.notes.push_back(result.error); + spdlog::debug("{}: {}", host, result.error); continue; } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 0943de7..71c27f0 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -29,7 +29,7 @@ target_include_directories(test_site PUBLIC .) target_link_libraries(test_site PUBLIC http_probe_core fake_dns) target_compile_options(test_site PRIVATE ${http_probe_warnings}) -add_executable(http_probe_tests prober_test.cpp routes_test.cpp) +add_executable(http_probe_tests output_test.cpp prober_test.cpp routes_test.cpp) target_link_libraries(http_probe_tests PRIVATE test_site GTest::gmock GTest::gtest_main) target_compile_options(http_probe_tests PRIVATE ${http_probe_warnings}) diff --git a/test/output_test.cpp b/test/output_test.cpp new file mode 100644 index 0000000..bd95c32 --- /dev/null +++ b/test/output_test.cpp @@ -0,0 +1,112 @@ +// +// What the program prints: with colour, and with nothing of it left without. +// +#include "output.hpp" + +#include +#include + +using namespace http_probe; +using namespace std::chrono_literals; +using Outcome = Result::Outcome; +using testing::HasSubstr; +using testing::Not; + +namespace +{ + +Result answered(int status = 200) +{ + return {.host = "example.com", + .outcome = Outcome::answered, + .records = {{.priority = 1, + .target = ".", + .params = {"alpn=h3", "ipv4hint=192.0.2.1", "ipv6hint=2001:db8::1"}}}, + .status = status, + .route = "h3 example.com:443", + .fields = {{"server", "test"}}, + .body_size = 5, + .duration = 12ms}; +} + +} // namespace + +TEST(Output, SaysWhatCameBack) +{ + EXPECT_EQ(format_result(answered(), false), + "example.com: 200 [h3 example.com:443] 5 bytes in 12ms\n" + " record: HTTPS 1 . alpn=h3 ipv4hint=192.0.2.1 ipv6hint=2001:db8::1\n" + " server: test\n"); +} + +TEST(Output, SaysWhyThereIsNoResponse) +{ + EXPECT_EQ(format_result({.host = "example.com", + .outcome = Outcome::failed, + .error = "[h3 example.com:443] timed out", + .notes = {"some note"}}, + false), + "example.com: failed: [h3 example.com:443] timed out\n note: some note\n"); + EXPECT_EQ(format_result({.host = "example.com", .outcome = Outcome::out_of_reach}, false), + "example.com: HTTP/3 out of reach\n"); + EXPECT_EQ(format_result({.host = "example.com", .outcome = Outcome::no_http3}, false), + "example.com: no HTTP/3\n"); + EXPECT_EQ(format_result({.host = "example.com", .outcome = Outcome::lookup_failed}, false), + "example.com: lookup failed\n"); +} + +TEST(Output, ColoursTheWayCAresAsyncDoes) +{ + const auto text = format_result(answered(), true); + + EXPECT_THAT(text, HasSubstr("\x1b[33mexample.com\x1b[0m:")) << "host names are yellow"; + EXPECT_THAT(text, HasSubstr("ipv4hint=\x1b[35m192.0.2.1\x1b[0m")) << "IPv4 is magenta"; + EXPECT_THAT(text, HasSubstr("ipv6hint=\x1b[34m2001:db8::1\x1b[0m")) << "IPv6 is blue"; +} + +TEST(Output, ColoursATargetAsAHostName) +{ + auto result = answered(); + result.records[0].target = "svc.example.net"; + + EXPECT_THAT(format_result(result, true), HasSubstr("\x1b[33msvc.example.net\x1b[0m")); + EXPECT_THAT(format_result(result, false), HasSubstr("HTTPS 1 svc.example.net alpn=h3")); +} + +TEST(Output, ColoursTheStatusByItsClass) +{ + EXPECT_THAT(format_result(answered(204), true), HasSubstr("\x1b[32m204\x1b[0m")); + EXPECT_THAT(format_result(answered(301), true), HasSubstr("\x1b[36m301\x1b[0m")); + EXPECT_THAT(format_result(answered(404), true), HasSubstr("\x1b[31m404\x1b[0m")); + EXPECT_THAT(format_result(answered(503), true), HasSubstr("\x1b[31m503\x1b[0m")); +} + +TEST(Output, HasNoColourWithoutColour) +{ + for (const auto& result : + {answered(), Result{.host = "h", .outcome = Outcome::failed, .notes = {"n"}}, + Result{.host = "h", .outcome = Outcome::lookup_failed}}) + EXPECT_THAT(format_result(result, false), Not(HasSubstr("\x1b"))); + + const Counts counts{{Outcome::answered, 1}, {Outcome::failed, 1}}; + EXPECT_THAT(format_summary(counts, false), Not(HasSubstr("\x1b"))); +} + +TEST(Output, CountsTheOutcomes) +{ + EXPECT_EQ(format_summary({{Outcome::answered, 3}, + {Outcome::failed, 1}, + {Outcome::no_http3, 900}, + {Outcome::lookup_failed, 17}}, + false), + "3 answered, 1 failed, 0 with HTTP/3 out of reach, 900 without HTTP/3, " + "17 lookups failed"); +} + +TEST(Output, DrawsAttentionOnlyToWhatHappened) +{ + const auto text = format_summary({{Outcome::answered, 1}}, true); + + EXPECT_THAT(text, HasSubstr("\x1b[32m1 answered\x1b[0m")); + EXPECT_THAT(text, HasSubstr(", 0 failed,")) << "nothing to see, no colour"; +} diff --git a/test/prober_test.cpp b/test/prober_test.cpp index b500f0f..51b563c 100644 --- a/test/prober_test.cpp +++ b/test/prober_test.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -17,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -467,6 +469,36 @@ TEST_F(Probe, IgnoresServiceRecordsNextToAnAlias) EXPECT_THAT(site.requests(), IsEmpty()); } +// ================================================================================================= +// What can be seen of it +// ================================================================================================= + +TEST_F(Probe, LogsTheHttpAndTheLookupsAtDebugLevel) +{ + // + // What -v is for. The wording is up to the libraries, so this is only about it being there. + // + std::ostringstream log; + const auto previous = spdlog::default_logger(); + spdlog::set_default_logger(std::make_shared( + "test", std::make_shared(log))); + spdlog::set_level(spdlog::level::debug); + + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.page("/", {.body = "hello"}); + probe(config()); + + spdlog::set_default_logger(previous); + quiet(); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 200); + + EXPECT_THAT(log.str(), HasSubstr("asking for the HTTPS records of 'localhost'")) << "our own"; + EXPECT_THAT(log.str(), HasSubstr("GET https://localhost/")) << "anyhttp's request"; + EXPECT_THAT(log.str(), HasSubstr("content-type")) << "anyhttp's response"; +} + // ================================================================================================= // What can go wrong // ================================================================================================= diff --git a/test/site.cpp b/test/site.cpp index 85daecf..8b4c523 100644 --- a/test/site.cpp +++ b/test/site.cpp @@ -1,6 +1,7 @@ #include "site.hpp" #include +#include #include "routes.hpp" @@ -30,13 +31,7 @@ Site::Site(const anyhttp::Executor& executor) : requests_(std::make_sharedlocal_endpoint().port(); server_->on_request([this, requests = requests_]( @@ -66,8 +61,38 @@ Site::Site(const anyhttp::Executor& executor) : requests_(std::make_shared Date: Mon, 2 Nov 2026 22:33:43 +0000 Subject: [PATCH 07/27] chore: add the list of 999 hosts from c-ares-async To have something to run the program on, as it is in c-ares-async's example/resolve. The tests don't use it. Co-Authored-By: Claude Opus 5.5 --- example/hostnames.txt | 999 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 999 insertions(+) create mode 100644 example/hostnames.txt diff --git a/example/hostnames.txt b/example/hostnames.txt new file mode 100644 index 0000000..80422eb --- /dev/null +++ b/example/hostnames.txt @@ -0,0 +1,999 @@ +fonts.googleapis.com +facebook.com +twitter.com +google.com +youtube.com +s.w.org +instagram.com +googletagmanager.com +linkedin.com +ajax.googleapis.com +plus.google.com +gmpg.org +pinterest.com +fonts.gstatic.com +wordpress.org +en.wikipedia.org +youtu.be +maps.google.com +itunes.apple.com +github.com +bit.ly +play.google.com +goo.gl +docs.google.com +cdnjs.cloudflare.com +vimeo.com +support.google.com +google-analytics.com +maps.googleapis.com +flickr.com +vk.com +t.co +reddit.com +amazon.com +medium.com +sites.google.com +drive.google.com +creativecommons.org +microsoft.com +developers.google.com +adobe.com +soundcloud.com +theguardian.com +apis.google.com +ec.europa.eu +lh3.googleusercontent.com +chrome.google.com +cloudflare.com +nytimes.com +maxcdn.bootstrapcdn.com +support.microsoft.com +blogger.com +forbes.com +s3.amazonaws.com +code.jquery.com +dropbox.com +translate.google.com +paypal.com +apps.apple.com +tinyurl.com +etsy.com +theatlantic.com +m.facebook.com +archive.org +amzn.to +cnn.com +policies.google.com +commons.wikimedia.org +issuu.com +i.imgur.com +wordpress.com +wp.me +businessinsider.com +yelp.com +mail.google.com +support.apple.com +t.me +apple.com +washingtonpost.com +bbc.com +gstatic.com +imgur.com +amazon.de +bbc.co.uk +googleads.g.doubleclick.net +mozilla.org +eventbrite.com +slideshare.net +w3.org +forms.gle +platform.twitter.com +accounts.google.com +telegraph.co.uk +messenger.com +web.archive.org +secure.gravatar.com +usatoday.com +huffingtonpost.com +stackoverflow.com +fb.com +npr.org +techcrunch.com +wired.com +eepurl.com +reuters.com +arxiv.org +i0.wp.com +finance.yahoo.com +player.vimeo.com +amazon.fr +developer.mozilla.org +tumblr.com +cnbc.com +booking.com +zoom.us +tools.google.com +wa.me +paypal.me +slack.com +adwords.google.com +independent.co.uk +discord.gg +open.spotify.com +tripadvisor.com +gsuite.google.com +behance.net +thinkwithgoogle.com +who.int +cloud.google.com +calendar.google.com +ibm.com +analytics.google.com +lh5.googleusercontent.com +docs.microsoft.com +groups.google.com +mixcloud.com +google.co.in +latimes.com +gist.github.com +download.macromedia.com +dailymail.co.uk +myaccount.google.com +line.me +unsplash.com +addons.mozilla.org +msdn.microsoft.com +storage.googleapis.com +pagead2.googlesyndication.com +cdn.jsdelivr.net +g.co +meetup.com +bloomberg.com +books.google.com +vox.com +imdb.com +arstechnica.com +support.mozilla.org +amazon.co.uk +statista.com +google.ca +nbcnews.com +steemit.com +upload.wikimedia.org +aws.amazon.com +amazon.co.jp +w3schools.com +weibo.com +twitch.tv +moz.com +drupal.org +nypost.com +fr.wikipedia.org +ebay.com +sourceforge.net +godaddy.com +ncbi.nlm.nih.gov +wsj.com +thenextweb.com +cdc.gov +photos.google.com +trends.google.com +opera.com +ftc.gov +j.mp +about.me +gov.uk +i.ytimg.com +squareup.com +google.co.uk +buzzfeed.com +developer.apple.com +pixabay.com +quora.com +cnet.com +google.com.au +myspace.com +theglobeandmail.com +shopify.com +whitehouse.gov +marketingplatform.google.com +kickstarter.com +freelancer.com +evernote.com +gmail.com +static.wixstatic.com +fortune.com +wix.com +dailymotion.com +goodreads.com +podcasts.google.com +google.co.jp +support.cloudflare.com +surveymonkey.com +cbsnews.com +entrepreneur.com +bing.com +engadget.com +lh4.googleusercontent.com +tandfonline.com +cargocollective.com +join.slack.com +api.whatsapp.com +shutterstock.com +pbs.twimg.com +blog.google +ads.google.com +time.com +lifehacker.com +patreon.com +scribd.com +lh6.googleusercontent.com +feeds.feedburner.com +trello.com +ted.com +helpx.adobe.com +researchgate.net +fda.gov +instructables.com +google.de +digitalocean.com +dribbble.com +ssl.gstatic.com +variety.com +indiegogo.com +salesforce.com +uk.linkedin.com +nasa.gov +codepen.io +newyorker.com +abcnews.go.com +snapchat.com +googleadservices.com +sciencedirect.com +ft.com +nature.com +gofundme.com +500px.com +qz.com +dl.dropboxusercontent.com +store.steampowered.com +webmasters.googleblog.com +podcasts.apple.com +oracle.com +m.youtube.com +gartner.com +web.facebook.com +blogs.msdn.com +mailchi.mp +digg.com +skype.com +periscope.tv +google.com.br +googleblog.blogspot.com +de.wikipedia.org +search.google.com +npmjs.com +motherboard.vice.com +aboutads.info +pexels.com +sxsw.com +fiverr.com +code.google.com +cbc.ca +venturebeat.com +hangouts.google.com +mashable.com +upwork.com +bitly.com +music.apple.com +loc.gov +news.google.com +spotify.com +netflix.com +hbr.org +pbs.org +stumbleupon.com +walmart.com +foursquare.com +amazon.ca +tiny.cc +services.google.com +xkcd.com +target.com +developers.facebook.com +nginx.com +canva.com +s7.addthis.com +elmundo.es +i2.wp.com +gizmodo.com +expedia.com +picasaweb.google.com +money.cnn.com +timesofindia.indiatimes.com +yandex.ru +vice.com +de-de.facebook.com +pastebin.com +samsung.com +rt.com +ifttt.com +dev.mysql.com +giphy.com +use.fontawesome.com +houzz.com +developer.android.com +wikipedia.org +speakerdeck.com +productforums.google.com +digitaltrends.com +steamcommunity.com +canada.ca +networkadvertising.org +linktr.ee +calendly.com +lg.com +yahoo.com +last.fm +amazon.es +businesswire.com +deezer.com +dx.doi.org +udemy.com +edition.cnn.com +1.bp.blogspot.com +l.facebook.com +dl.dropbox.com +store.google.com +gumroad.com +azure.microsoft.com +i1.wp.com +ubuntu.com +google.ru +makeuseof.com +raw.githubusercontent.com +airbnb.com +bluehost.com +deviantart.com +pewresearch.org +google.cn +profiles.google.com +login.microsoftonline.com +google.nl +google.es +fastcompany.com +is.gd +telegram.me +mediafire.com +themeforest.net +openstreetmap.org +pewinternet.org +photos.app.goo.gl +buff.ly +events.google.com +link.springer.com +bizjournals.com +residentadvisor.net +golang.org +vogue.com +s3-eu-west-1.amazonaws.com +boardgamegeek.com +un.org +i.redd.it +ad.doubleclick.net +ow.ly +get.google.com +pt.slideshare.net +buffer.com +mozilla.com +chromium.org +patheos.com +ancestry.com +youtube-nocookie.com +barnesandnoble.com +amzn.com +hulu.com +greenpeace.org +spiegel.de +apps.facebook.com +census.gov +app.box.com +windows.microsoft.com +yandex.com +sketchfab.com +avvo.com +searchengineland.com +shareasale.com +abc.net.au +trustpilot.com +plaza.rakuten.co.jp +marriott.com +amazon.it +ca.linkedin.com +uber.com +coursera.org +adage.com +foxnews.com +flic.kr +developer.chrome.com +pl.wikipedia.org +change.org +spreaker.com +filezilla-project.org +flipboard.com +pcworld.com +news.yahoo.com +pitchfork.com +weforum.org +istockphoto.com +9to5mac.com +1drv.ms +uspto.gov +irs.gov +copyblogger.com +example.com +maps.google.co.jp +fonts.google.com +siteground.com +sony.net +ja.wikipedia.org +creativemarket.com +smithsonianmag.com +mobile.twitter.com +acm.org +accenture.com +git-scm.com +bbb.org +cdn.shopify.com +schema.org +psychologytoday.com +redcross.org +nodejs.org +europa.eu +ameblo.jp +gitlab.com +sciencemag.org +hollywoodreporter.com +freshbooks.com +heise.de +chicagotribune.com +epa.gov +anchor.fm +telegram.org +ikea.com +bloglovin.com +eventbrite.co.uk +googlewebmastercentral.blogspot.com +huffpost.com +weebly.com +producthunt.com +google.fr +click.linksynergy.com +vanityfair.com +diigo.com +about.fb.com +ouest-france.fr +namecheap.com +codecanyon.net +appstore.com +tools.ietf.org +rockpapershotgun.com +columbia.edu +constantcontact.com +webmd.com +eur-lex.europa.eu +aliexpress.com +business.facebook.com +material.io +neilpatel.com +discordapp.com +m.me +design.google +francetvinfo.fr +slashgear.com +4shared.com +python.org +amazon.com.au +storify.com +elegantthemes.com +europe1.fr +pwc.com +activecampaign.com +freepik.com +ravelry.com +apnews.com +propublica.org +smugmug.com +inc.com +google.it +zazzle.com +dol.gov +scoop.it +bit.do +yadi.sk +eclipse.org +economist.com +xing.com +hbo.com +nationalgeographic.com +jetbrains.com +iconfinder.com +cafepress.com +strava.com +es.wikipedia.org +kiva.org +doi.org +boredpanda.com +justgiving.com +pinterest.ca +stats.wp.com +zdnet.com +sproutsocial.com +rebrand.ly +ranker.com +a.co +waze.com +nydailynews.com +marketwatch.com +mailchimp.com +livescience.com +flaticon.com +ilpost.it +connect.facebook.net +4.bp.blogspot.com +maps.google.co.nz +s0.wp.com +amazon.com.br +ustream.tv +lulu.com +socialmediatoday.com +market.android.com +rollingstone.com +journals.sagepub.com +zillow.com +xinhuanet.com +affiliate-program.amazon.com +politico.com +pnas.org +laughingsquid.com +access.redhat.com +s.ytimg.com +it.linkedin.com +dashlane.com +copyright.gov +news.bbc.co.uk +mentalfloss.com +zeit.de +google.cz +artsandculture.google.com +startnext.com +launchpad.net +lemonde.fr +sciencedaily.com +edx.org +smashwords.com +philips.co.uk +smile.amazon.com +intel.com +livestream.com +brookings.edu +thehill.com +popsci.com +blog.us.playstation.com +fb.me +academic.oup.com +php.net +wetransfer.com +blog.livedoor.jp +amazon.in +families.google.com +obsproject.com +t.qq.com +android.com +blogs.windows.com +hostgator.com +google.gr +theverge.com +aub.edu.lb +digiday.com +gimp.org +treasury.gov +geocities.com +healthline.com +3.bp.blogspot.com +coinbase.com +gnu.org +abc.com +aljazeera.com +getresponse.com +dev.to +thesun.co.uk +lynda.com +sellfy.com +prnt.sc +help.ubuntu.com +whatsapp.com +www8.hp.com +notion.so +vmware.com +kraken.com +vine.co +marthastewart.com +discord.me +dailycaller.com +mega.nz +media.giphy.com +flavors.me +blip.tv +prntscr.com +journals.plos.org +gravatar.com +discogs.com +europarl.europa.eu +redhat.com +bugs.chromium.org +vanmiubeauty.com +metro.co.uk +google.ie +relapse.com +earth.google.com +bandcamp.com +br.linkedin.com +gopro.com +t.ly +get.adobe.com +news.mit.edu +billboard.com +britannica.com +onlinelibrary.wiley.com +techsmith.com +sfgate.com +redbubble.com +kstatic.googleusercontent.com +repubblica.it +disqus.com +khanacademy.org +skfb.ly +androidauthority.com +accessify.com +tensorflow.org +chrisjdavis.org +box.net +poetryfoundation.org +law.cornell.edu +penguinrandomhouse.com +refinery29.com +starwars.com +fr.linkedin.com +video.google.com +prnewswire.com +springer.com +zalo.me +reverbnation.com +getpocket.com +webdesign.about.com +goo.gle +note.mu +thetimes.co.uk +fao.org +bol.com +yoursite.com +sophos.com +automattic.com +podbean.com +sublimetext.com +archives.gov +vizio.com +thelancet.com +guardian.co.uk +news.nationalgeographic.com +tunein.com +stitcher.com +coinmarketcap.com +amnestyusa.org +docs.wixstatic.com +cambridge.org +validator.w3.org +asus.com +problogger.net +worldbank.org +ticketportal.cz +dw.com +static.googleusercontent.com +www-01.ibm.com +business.google.com +es.linkedin.com +songkick.com +adweek.com +lifehack.org +apple.co +sutterhealth.org +globalnews.ca +breitbart.com +thumbtack.com +eff.org +snopes.com +crowdrise.com +hkrsa.asia +metmuseum.org +pixlr.com +institutvajrayogini.fr +rhtlbn.mosgorcredit.ru +itchyfeetonthecheap.com +reacts.ru +db.tt +udacity.com +we.tl +blogtalkradio.com +audacityteam.org +youcaring.com +hp.com +0.gravatar.com +health.harvard.edu +meta.wikimedia.org +google.ch +nhs.uk +mixi.jp +symfony.com +business.linkedin.com +mtv.com +1.gravatar.com +cyber.law.harvard.edu +bitcointalk.org +esa.int +slate.com +oecd.org +raspberrypi.org +sfexaminer.com +warriorforum.com +g.page +bhphotovideo.com +axios.com +pinterest.co.uk +unesco.org +msn.com +code.visualstudio.com +netbeans.org +blockchain.info +untappd.com +espn.com +cbs.com +michigan.gov +oprah.com +ok.ru +mhlw.go.jp +newegg.com +2.bp.blogspot.com +pipes.yahoo.com +themarthablog.com +yummly.com +bitpay.com +buymeacoffee.com +deepl.com +blogs.scientificamerican.com +audible.com +bitbucket.org +it.wikipedia.org +in.linkedin.com +cse.google.com +use.typekit.net +ru.wikipedia.org +faz.net +adf.ly +cancerresearchuk.org +moma.org +bigthink.com +bandsintown.com +addthis.com +support.office.com +upi.com +zen.yandex.ru +collegehumor.com +lefigaro.fr +gplus.to +themify.me +rtve.es +nba.com +myfitnesspal.com +chris.pirillo.com +eonline.com +science.sciencemag.org +ietf.org +ko-fi.com +otto.de +tkqlhce.com +unity3d.com +blog.naver.com +scratch.mit.edu +freewebs.com +drift.com +postmates.com +geek.com +geocities.jp +beian.gov.cn +meet.google.com +flattr.com +webroot.com +thoughtcatalog.com +google.pl +seroundtable.com +checkpoint.com +form.jotform.com +xbox.com +infusionsoft.com +g1.globo.com +capterra.com +ea.com +hawaii.edu +stock.adobe.com +osha.gov +buzzfeednews.com +jstor.org +adssettings.google.com +iheart.com +en-gb.facebook.com +foodnetwork.com +eventim.de +economictimes.indiatimes.com +presseportal.de +google.se +skillshare.com +puu.sh +examiner.com +stripe.com +firstdata.com +en.advertisercommunity.com +download.microsoft.com +s-media-cache-ak0.pinimg.com +ctt.ec +cia.gov +docker.com +homedepot.com +humblebundle.com +chiark.greenend.org.uk +pandora.com +google.co.za +payhip.com +thingiverse.com +inkscape.org +adobe.ly +css-tricks.com +ja-jp.facebook.com +jamanetwork.com +maps.gstatic.com +wattpad.com +chase.com +cisco.com +bmj.com +louvre.fr +aclu.org +squarespace.com +gum.co +idealo.de +imore.com +rakuten.com +fbi.gov +united.com +google.pt +event.on24.com +nejm.org +colorado.edu +franchising.com +logitech.com +redbull.com +tf1.fr +keep.google.com +hostinger.com +nbc.com +stats.g.doubleclick.net +ign.com +blog.hubspot.com +1.envato.market +accessdata.fda.gov +google.be +help.apple.com +technorati.com +pond5.com +ucl.ac.uk +a2hosting.com +amzn.asia +nicovideo.jp +indiewire.com +mp.weixin.qq.com +codeproject.com +1.usa.gov +gleam.io +space.com +gitter.im +lh5.ggpht.com +leparisien.fr +kotaku.com +img.youtube.com +huffingtonpost.co.uk +agoda.com +flow.microsoft.com +chronicle.com +createspace.com +lh3.ggpht.com +verizon.com +teespring.com +data.worldbank.org +blog.goo.ne.jp +kobo.com +globalsign.com +money.yandex.ru +animoto.com +google.co.nz +privacy.microsoft.com +online.wsj.com +patft.uspto.gov +rottentomatoes.com +cell.com +de.linkedin.com +health.com +geni.us +news.harvard.edu +lesechos.fr +fas.org +smashingmagazine.com +7-zip.org +realvnc.com +news.discovery.com +mlb.com +autodesk.com +popularmechanics.com +forms.office.com +rdio.com +sendspace.com +synology.com +angelfire.com +france24.com +us.battle.net +stjude.org +denverpost.com +japantimes.co.jp +stocktwits.com +blogs.adobe.com +envato.com +airtable.com +ticketmaster.com +ocw.mit.edu +whc.unesco.org +neh.gov +purl.org +nfl.com +monster.com +codex.wordpress.org +dafont.com +tesla.com +lmgtfy.com +olympic.org +technet.microsoft.com +baidu.com +desktop.github.com +overcast.fm +allmusic.com +cdbaby.com +vsco.co +nvidia.com +stadt-bremerhaven.de +faa.gov +blogs.yahoo.co.jp +bild.de +lenovo.com +funnyordie.com +ssl.google-analytics.com +abebooks.com +spectrum.ieee.org +thinkgeek.com +snip.ly +pixiv.net +buzzsprout.com +patents.google.com +orcid.org +cdn.ampproject.org +blog.feedspot.com +vr.google.com +opinionator.blogs.nytimes.com +ericsson.com From 81316404a6d8cc82a2c1bd2dc302e98bb555c2f1 Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Mon, 2 Nov 2026 22:54:31 +0000 Subject: [PATCH 08/27] refactor: parse the command line with Boost.ProgramOptions As resolve in c-ares-async and server in anyhttp do. This adds --help, and with it --file=x, "--" and shortened option names. Giving an option with a single value twice is now an error, and hosts from --file come after those on the command line. The numbers are read as signed, because lexical_cast takes -1 for the largest value of an unsigned type. Boost is linked statically under clang, for the reason anyhttp does. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 7 +- src/main.cpp | 179 +++++++++++++++++++++++++++----------------- test/CMakeLists.txt | 17 +++++ 3 files changed, 132 insertions(+), 71 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a7ff549..ce29895 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,8 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") add_compile_options($<$:-stdlib=libc++>) add_link_options(-stdlib=libc++) list(PREPEND CMAKE_PREFIX_PATH /opt/libc++) + # As in anyhttp: the shared ones share their names with the libstdc++ ones in /usr/local/lib. + set(Boost_USE_STATIC_LIBS ON) endif() # I/O runtime: ASIO or COROSIO. COROSIO would have to declare the capy and corosio pins here, too. @@ -53,6 +55,9 @@ FetchContent_Declare(c_ares_async SYSTEM) FetchContent_MakeAvailable(anyhttp c_ares_async) +# Only for the command line. Neither library needs this component, so neither asks for it here. +find_package(Boost 1.88 REQUIRED COMPONENTS program_options) + # -------------------------------------------------------------------------------------------------- # @@ -67,7 +72,7 @@ target_link_libraries(http_probe_core PUBLIC anyhttp::anyhttp c_ares_async::c_ar target_compile_options(http_probe_core PRIVATE ${http_probe_warnings}) add_executable(http-probe src/main.cpp) -target_link_libraries(http-probe PRIVATE http_probe_core) +target_link_libraries(http-probe PRIVATE http_probe_core Boost::program_options) target_compile_options(http-probe PRIVATE ${http_probe_warnings}) enable_testing() diff --git a/src/main.cpp b/src/main.cpp index 95106e3..7055601 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -6,47 +6,41 @@ #include #include +#include #include #include #include -#include +#include #include #include #include #include #include #include +#include #include #include #include #include #include +namespace po = boost::program_options; + namespace { -constexpr std::string_view usage = R"(usage: http-probe [options] HOST... +constexpr std::string_view about = R"(usage: http-probe [options] HOST... http-probe [options] --file LIST Looks up the HTTPS records of each host (RFC 9460), and requests https://HOST/ once from those that offer HTTP/3, printing the status and header fields of the response. Redirects are reported and not followed, and nothing else on the host is requested. +)"; - --file LIST read the hosts from a file, one per line, as names or as URLs - --dns-server ADDR the DNS server to ask, like 192.0.2.1 or 192.0.2.1:5353; default: the system's, - which may well not answer for HTTPS records - --ca-file PEM the CA certificates to verify servers with; default: the system's - --concurrency N how many hosts to work on at the same time; default: 16 - --timeout-ms N how long one request may take; default: 10000 - --all also print the hosts without HTTP/3, and why - --color colour the output even where it does not go to a terminal, e.g. into 'less -R' - -v, --verbose log to stderr what is going on, the requests and responses of anyhttp and the - queries of c-ares among it; -vv for all of QUIC as well, which is a lot. To - make sense of it, use --concurrency 1. - +constexpr std::string_view exit_code = R"( The exit code is 0 if every host with HTTP/3 answered, whatever the status, and every lookup got an answer. )"; @@ -54,76 +48,110 @@ answer. struct Arguments { http_probe::Config config; + std::vector files; + bool help = false; bool all = false; bool color = false; int verbosity = 0; }; -template -T number(std::string_view option, std::string_view text) +// +// The numbers are read as signed ones, because lexical_cast takes -1 for the largest value of an +// unsigned type, which would pass as a choice. +// +int not_negative(std::string_view option, int value) { - T value{}; - const auto [end, ec] = std::from_chars(text.data(), text.data() + text.size(), value); - if (ec != std::errc{} || end != text.data() + text.size()) - throw std::invalid_argument(std::format("{}: '{}' is not a number", option, text)); + if (value < 0) + throw std::invalid_argument(std::format("{} can't be negative", option)); return value; } -Arguments parse_arguments(int argc, char* argv[]) +// +// The options, stored into 'arguments', which has to outlive parsing them. +// +po::options_description describe_options(Arguments& arguments) { - Arguments arguments; auto& config = arguments.config; - for (int i = 1; i < argc; ++i) + + po::options_description desc("Options", 100, 50); + auto opts = desc.add_options(); + opts("help,h", po::bool_switch(&arguments.help), "produce this help message"); + opts("host", po::value(&config.hosts)->value_name("HOST"), + "a host to request, the same as giving HOST by itself"); + opts("file", po::value(&arguments.files)->value_name("LIST"), + "read the hosts from a file, one per line, as names or as URLs"); + opts("dns-server", po::value(&config.dns_server)->value_name("ADDR"), + "the DNS server to ask, like 192.0.2.1 or 192.0.2.1:5353; default: the system's, which may " + "well not answer for HTTPS records"); + opts("ca-file", po::value(&config.tls_ca_file)->value_name("PEM"), + "the CA certificates to verify servers with; default: the system's"); + opts("concurrency", + po::value() + ->value_name("N") + ->default_value(static_cast(config.concurrency)) + ->notifier( + [&config](int value) { config.concurrency = not_negative("--concurrency", value); }), + "how many hosts to work on at the same time"); + opts("timeout-ms", + po::value() + ->value_name("N") + ->default_value(static_cast(config.timeout.count())) + ->notifier([&config](int value) { + config.timeout = std::chrono::milliseconds(not_negative("--timeout-ms", value)); + }), + "how long one request may take"); + opts("all", po::bool_switch(&arguments.all), "also print the hosts without HTTP/3, and why"); + opts("color", po::bool_switch(&arguments.color), + "colour the output even where it does not go to a terminal, e.g. into 'less -R'"); + opts("verbose,v", po::value>()->zero_tokens()->composing(), + "log to stderr what is going on, the requests and responses of anyhttp and the queries of " + "c-ares among it; -vv for all of QUIC as well, which is a lot. To make sense of it, use " + "--concurrency 1."); + return desc; +} + +void print_usage(std::ostream& out, const po::options_description& options) +{ + out << about << '\n' << options << exit_code; +} + +void parse_arguments(int argc, char* argv[], const po::options_description& options, + Arguments& arguments) +{ + try { - const std::string_view argument = argv[i]; - if (!argument.starts_with("-")) - { - config.hosts.emplace_back(argument); - continue; - } + po::positional_options_description positional; + positional.add("host", -1); - if (argument == "--all") - { - arguments.all = true; - continue; - } - if (argument == "--color") - { - arguments.color = true; - continue; - } - if (argument == "-v" || argument == "--verbose" || argument == "-vv") - { - arguments.verbosity += argument == "-vv" ? 2 : 1; - continue; - } + const auto parsed = + po::command_line_parser(argc, argv).options(options).positional(positional).run(); - if (i + 1 == argc) - throw std::invalid_argument(std::format("{} needs a value, or is unknown", argument)); - const std::string_view value = argv[++i]; + po::variables_map vm; + po::store(parsed, vm); + po::notify(vm); - if (argument == "--file") - { - std::ifstream list{std::string(value)}; - if (!list) - throw std::invalid_argument(std::format("can't read '{}'", value)); - std::vector skipped; - config.hosts.append_range(http_probe::read_hosts(list, skipped)); - for (const auto& line : skipped) - std::println(stderr, "http-probe: skipping '{}': not a host name", line); - } - else if (argument == "--dns-server") - config.dns_server = value; - else if (argument == "--ca-file") - config.tls_ca_file = value; - else if (argument == "--concurrency") - config.concurrency = number(argument, value); - else if (argument == "--timeout-ms") - config.timeout = std::chrono::milliseconds(number(argument, value)); - else - throw std::invalid_argument(std::format("unknown option '{}'", argument)); + // 'verbose' takes no argument, so its parsed value is always empty: count occurrences + arguments.verbosity = static_cast(std::ranges::count_if( + parsed.options, [](const po::option& option) { return option.string_key == "verbose"; })); + } + catch (const po::error& ex) + { + throw std::invalid_argument(ex.what()); + } + + if (arguments.help) + return; + + for (const auto& file : arguments.files) + { + std::ifstream list{file}; + if (!list) + throw std::invalid_argument(std::format("can't read '{}'", file)); + std::vector skipped; + arguments.config.hosts.append_range(http_probe::read_hosts(list, skipped)); + for (const auto& line : skipped) + std::println(stderr, "http-probe: skipping '{}': not a host name", line); } - return arguments; } // @@ -149,9 +177,19 @@ int main(int argc, char* argv[]) http_probe::Counts counts; std::size_t with_records = 0; + Arguments arguments; + const auto options = describe_options(arguments); + try { - auto [config, all, color, verbosity] = parse_arguments(argc, argv); + parse_arguments(argc, argv, options, arguments); + if (arguments.help) + { + print_usage(std::cout, options); + return 0; + } + + auto& [config, files, help, all, color, verbosity] = arguments; set_up_logging(verbosity, color); const bool own_dns_server = !config.dns_server.empty(); @@ -188,7 +226,8 @@ int main(int argc, char* argv[]) } catch (const std::invalid_argument& ex) { - std::println(stderr, "http-probe: {}\n\n{}", ex.what(), usage); + std::println(stderr, "http-probe: {}\n", ex.what()); + print_usage(std::cerr, options); return 2; } catch (const std::exception& ex) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 71c27f0..7c800d6 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -62,3 +62,20 @@ add_test(NAME http_probe_refuses_a_concurrency_of_zero COMMAND http-probe --concurrency 0 localhost) set_tests_properties(http_probe_refuses_a_concurrency_of_zero PROPERTIES PASS_REGULAR_EXPRESSION "concurrency of 0") + +add_test(NAME http_probe_refuses_a_negative_concurrency + COMMAND http-probe --concurrency=-1 localhost) +set_tests_properties(http_probe_refuses_a_negative_concurrency PROPERTIES + PASS_REGULAR_EXPRESSION "--concurrency can't be negative") + +add_test(NAME http_probe_refuses_a_negative_timeout COMMAND http-probe --timeout-ms -1 localhost) +set_tests_properties(http_probe_refuses_a_negative_timeout PROPERTIES + PASS_REGULAR_EXPRESSION "--timeout-ms can't be negative") + +add_test(NAME http_probe_refuses_an_unknown_option COMMAND http-probe --no-such-option localhost) +set_tests_properties(http_probe_refuses_an_unknown_option PROPERTIES + PASS_REGULAR_EXPRESSION "unrecognised option '--no-such-option'") + +add_test(NAME http_probe_describes_its_options COMMAND http-probe --help) +set_tests_properties(http_probe_describes_its_options PROPERTIES + PASS_REGULAR_EXPRESSION "--dns-server ADDR") From e83a126dc1a119223f319e6b2ad49fb06685aa97 Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Mon, 2 Nov 2026 23:11:22 +0000 Subject: [PATCH 09/27] docs: ask for library changes with issues, reviewed by Peter Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 71a3adb..1ea8b2e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,6 +28,9 @@ any I/O, and [prober.cpp](src/prober.cpp) does the rest. The tests in [test/](te - Changes to a library belong in that library's repository and session, with a test in its suite. Do not work around a library bug in this code: get it fixed there, then move the pin. +- Ask for a library change with an issue in that library's repository, using `gh`. Peter reviews the + full text first, and the same goes for comments on issues and for closing them. The repositories + are public, so reproduce on loopback and name no hosts from the lists. - Moving a pin in [CMakeLists.txt](CMakeLists.txt) is a commit of its own. CI builds from the pins only, so a change here that needs a library change has to wait for that change to be pushed. - The `dev` preset builds the sibling checkouts as they are, on whatever branch they are on, and can @@ -43,5 +46,4 @@ As in anyhttp: clang-format with 3-space indent and 100 columns, members `foo_`, `snake_case`, types `CamelCase`. Commit subjects are prefixed `fix:`, `refactor:`, `docs:`, `chore:` or `test:`. -Commit or push only when asked. `gh`'s token is read-only, so push over SSH to git@github.com; Peter -opens the pull requests. +Commit or push only when asked. Push over SSH to git@github.com; Peter opens the pull requests. From 136047b981108712084354eb486559c5fe474199 Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Tue, 3 Nov 2026 07:01:01 +0000 Subject: [PATCH 10/27] chore: add readonly flag to bind mounts in devcontainer configuration --- .devcontainer/devcontainer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index defaab7..b7fd2de 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -75,8 +75,8 @@ "mounts": [ "source=cpp-devcontainer-bashhistory,target=/commandhistory,type=volume", - "source=${localEnv:HOME}/src/anyhttp,target=/workspaces/anyhttp,type=bind,consistency=cached", - "source=${localEnv:HOME}/src/c-ares-async,target=/workspaces/c-ares-async,type=bind,consistency=cached", + "source=${localEnv:HOME}/src/anyhttp,target=/workspaces/anyhttp,type=bind,consistency=cached,readonly", + "source=${localEnv:HOME}/src/c-ares-async,target=/workspaces/c-ares-async,type=bind,consistency=cached,readonly", "source=${localEnv:HOME}/.claude,target=/home/vscode/.claude,type=bind,consistency=cached", "source=${localEnv:HOME}/.claude.json,target=/home/vscode/.claude.json,type=bind,consistency=cached" ], From e6fc4dde1fce3a36aa970809a79554874374aae7 Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Tue, 3 Nov 2026 07:24:37 +0000 Subject: [PATCH 11/27] chore: move the anyhttp pin to master, which has TLS over TCP Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ce29895..2140d76 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,12 +42,10 @@ set(C_ARES_ASYNC_TLS ${TLS_LIBRARY}) # there is the 'dev' preset, which sets FETCHCONTENT_SOURCE_DIR_*. SYSTEM, so that the warnings of # this project do not apply to the headers of the libraries. # -# anyhttp: the head of branch 'packaging', move this to master once that is merged -# include(FetchContent) FetchContent_Declare(anyhttp GIT_REPOSITORY https://github.com/pgit/anyhttp.git - GIT_TAG af39b5c71490978d545c1b47ee8e149f9a7082c9 + GIT_TAG 9c7627d78cc350376e6f2e6872b251d854a565d8 SYSTEM) FetchContent_Declare(c_ares_async GIT_REPOSITORY https://github.com/pgit/c-ares-async.git From 01d7eb5ef45a68931223c92a57e08268c7470628 Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Tue, 3 Nov 2026 07:24:37 +0000 Subject: [PATCH 12/27] fix: connect to any host over TLS, and never send a second request anyhttp's client has TLS over TCP now, so a host no longer has to offer HTTP/3. Within a record, HTTP/3 goes first and TCP on the same port next, and a host without a usable record gets TCP on port 443. Names that do not exist, failed lookups, targets other than the host and records saying that the service is not available are still not connected to. This lifts the HTTP/3-only rule in CLAUDE.md, as decided by Peter. The next route is now only for a connection that failed. Before, a timeout during a request moved on to the next route as well, which could send the host a second request. Sessions now end with shutdown(), outside of the timeout. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 3 +- README.md | 42 ++++--- src/main.cpp | 14 +-- src/output.cpp | 12 +- src/prober.cpp | 125 +++++++++++++-------- src/prober.hpp | 32 ++++-- src/routes.cpp | 92 +++++++++++---- src/routes.hpp | 22 +++- test/output_test.cpp | 11 +- test/prober_test.cpp | 260 +++++++++++++++++++++++++++++++++++++------ test/routes_test.cpp | 156 +++++++++++++++++++------- test/site.cpp | 3 + test/site.hpp | 3 +- 13 files changed, 580 insertions(+), 195 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1ea8b2e..2f17bb6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,7 +11,8 @@ These hold for all code in this repository, including tests and experiments. - **One request per host, and nothing more.** c-ares-async's lists of hosts are fine as input here, which is what makes this rule matter: they have 1000 mixed sites, some adult or spam-looking. So - nothing is followed: no links and no redirects. Only hosts that offer HTTP/3 are connected to. + nothing is followed: no links, no redirects and no Alt-Svc. Another route is for a connection that + failed: once a request has gone out, that was it for the host, however it ended. - **Tests connect to local servers on loopback only.** - **If the program ever crawls, it refuses to run without an explicit host allowlist,** with robots.txt and per-host rate limits built in from the start, not added later. diff --git a/README.md b/README.md index 73a0283..4849ff3 100644 --- a/README.md +++ b/README.md @@ -16,29 +16,36 @@ http-probe --dns-server 1.1.1.1 --file hostnames.txt ``` It looks up the HTTPS records of each host in DNS ([RFC 9460](https://www.rfc-editor.org/rfc/rfc9460)), -and connects to those that offer HTTP/3. From each of them it requests `https:///` once, and -prints the status, the route, the time it took and the header fields of the response, along with the -host's records. A redirect is reported and not followed, and nothing else on the host is requested. +and requests `https:///` once, over TLS, in the way that the records say. It prints the status, +the route, the time it took and the header fields of the response, along with the host's records. A +redirect is reported and not followed, and nothing else on the host is requested. -HTTP/3 is what it takes because anyhttp's client has TLS on HTTP/3 only, so far. There is output for -every host that offers it and every lookup that failed, and `--all` adds the others. -The counts at the end look like this, on stderr: +There is output for every host but those that do not exist, which `--all` adds. The counts at the end +look like this, on stderr: ``` -12 answered, 1 failed, 0 with HTTP/3 out of reach, 180 without HTTP/3, 2 lookups failed +190 answered, 1 failed, 3 out of reach, 12 not found, 2 lookups failed ``` | The host | Outcome | |-----------------------------------------------|---------------------------------------------| -| has a record with `h3` in its ALPN set | answered or failed, to the port of the record | -| has that only on a target that is not itself | out of reach: anyhttp can't connect to one name and verify another | -| has records without `h3`, or none at all | without HTTP/3, not connected to | -| does not exist | without HTTP/3, not connected to | -| gets no answer from DNS | lookup failed, so there is no telling | +| has a record on its own name | answered or failed, by the protocols and the port of the record | +| has no record, or none that is compatible | answered or failed, over TCP on port 443, HTTP/2 or HTTP/1.1 as the server likes | +| has records for other targets only, or an alias to another name | out of reach, not connected to: anyhttp can't connect to one name and verify another | +| has a record that says the service is not available | out of reach, not connected to | +| does not exist | not found, not connected to | +| gets no answer from DNS | lookup failed, so there is no telling, not connected to | -Aliases are followed, a record has to be compatible, and the lowest priority goes first. Over HTTP/3, -the address comes from anyhttp's own resolver and not from c-ares, so `--dns-server` and the address -hints in the record do not apply to it. +Aliases are followed, a record has to be compatible, and the lowest priority goes first. Within a +record, HTTP/3 goes first, and then TCP on the same port: HTTP/2 if the record has `h2`, otherwise +HTTP/1.1, which is in every record that does not say `no-default-alpn`. The route in the output is +what the server went for, which may be HTTP/1.1 where HTTP/2 was offered. + +**One request means one.** The next route is for when connecting failed. Once a request has gone out, +the host has been asked, so that a timeout or a response cut short ends it there, as a failure. + +The address comes from anyhttp's own resolver and not from c-ares, so `--dns-server` and the address +hints in the record do not apply to it. Certificates are always verified, against the host's name. **To see what is going on,** `-v` logs to stderr: the requests and responses from anyhttp, the queries from c-ares and the steps in between. `-vv` adds all of QUIC. Hosts run at the same time mix in the @@ -48,9 +55,12 @@ anyhttp puts them into some of its log messages regardless. **Use `--dns-server`.** The resolver in the devcontainer answers with no data for HTTPS records, even where they exist, which looks just like no host having any. -**The program can take a while to end.** If a host with HTTP/3 did not answer, anyhttp goes on sending +**The program can take a while to end.** If a host did not answer over HTTP/3, anyhttp goes on sending to it for 30 seconds after the timeout. The counts are printed when the work is done, ahead of that. +**A response goes to memory as a whole,** as anyhttp has no way of getting at it piece by piece with a +GET yet. So only `--timeout-ms` limits how large one can get. + To see it work without going anywhere, `demo_site` runs a site and a DNS server on loopback, and prints the command to request it with: diff --git a/src/main.cpp b/src/main.cpp index 7055601..f6a02d4 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -35,14 +35,14 @@ namespace constexpr std::string_view about = R"(usage: http-probe [options] HOST... http-probe [options] --file LIST -Looks up the HTTPS records of each host (RFC 9460), and requests https://HOST/ once from those that -offer HTTP/3, printing the status and header fields of the response. Redirects are reported and not -followed, and nothing else on the host is requested. +Looks up the HTTPS records of each host (RFC 9460), and requests https://HOST/ once, the way they +say or over TCP on port 443 if they don't, printing the status and header fields of the response. +Redirects are reported and not followed, and nothing else on the host is requested. )"; constexpr std::string_view exit_code = R"( -The exit code is 0 if every host with HTTP/3 answered, whatever the status, and every lookup got an -answer. +The exit code is 0 if every host that was connected to answered, whatever the status, and every +lookup got an answer. )"; struct Arguments @@ -100,7 +100,7 @@ po::options_description describe_options(Arguments& arguments) config.timeout = std::chrono::milliseconds(not_negative("--timeout-ms", value)); }), "how long one request may take"); - opts("all", po::bool_switch(&arguments.all), "also print the hosts without HTTP/3, and why"); + opts("all", po::bool_switch(&arguments.all), "also print the hosts that do not exist"); opts("color", po::bool_switch(&arguments.color), "colour the output even where it does not go to a terminal, e.g. into 'less -R'"); opts("verbose,v", po::value>()->zero_tokens()->composing(), @@ -200,7 +200,7 @@ int main(int argc, char* argv[]) ++counts[result.outcome]; if (!result.records.empty() || result.aliased) ++with_records; - if (all || result.outcome != Outcome::no_http3) + if (all || result.outcome != Outcome::not_found) { std::print("{}", http_probe::format_result(result, color_results)); std::fflush(stdout); // a run can take a while, and it is worth watching diff --git a/src/output.cpp b/src/output.cpp index 50e9c53..83366dd 100644 --- a/src/output.cpp +++ b/src/output.cpp @@ -82,10 +82,10 @@ std::string format_result(const Result& result, bool color) std::format("{}: {}: {}\n", host, colored("failed", color_of::bad, color), result.error); break; case Outcome::out_of_reach: - text = std::format("{}: {}\n", host, colored("HTTP/3 out of reach", color_of::moved, color)); + text = std::format("{}: {}\n", host, colored("out of reach", color_of::moved, color)); break; - case Outcome::no_http3: - text = std::format("{}: no HTTP/3\n", host); + case Outcome::not_found: + text = std::format("{}: not found\n", host); break; case Outcome::lookup_failed: text = std::format("{}: {}\n", host, colored("lookup failed", color_of::bad, color)); @@ -110,10 +110,10 @@ std::string format_summary(Counts counts, bool color) return counts[outcome] ? colored(line, code, color) : line; }; return std::format( - "{}, {}, {}, {} without HTTP/3, {}", count(Outcome::answered, "answered", color_of::good), + "{}, {}, {}, {} not found, {}", count(Outcome::answered, "answered", color_of::good), count(Outcome::failed, "failed", color_of::bad), - count(Outcome::out_of_reach, "with HTTP/3 out of reach", color_of::moved), - counts[Outcome::no_http3], count(Outcome::lookup_failed, "lookups failed", color_of::bad)); + count(Outcome::out_of_reach, "out of reach", color_of::moved), counts[Outcome::not_found], + count(Outcome::lookup_failed, "lookups failed", color_of::bad)); } // ================================================================================================= diff --git a/src/prober.cpp b/src/prober.cpp index 577a36e..ad7f0bc 100644 --- a/src/prober.cpp +++ b/src/prober.cpp @@ -169,8 +169,9 @@ anyhttp::Task Prober::resolve_service(const std::string& host) auto [ec, records] = co_await resolver_.resolve_https(found.owner); if (ec) { - if (ec != c_ares_async::make_ares_error(ARES_ENODATA) && - ec != c_ares_async::make_ares_error(ARES_ENOTFOUND)) + if (ec == c_ares_async::make_ares_error(ARES_ENOTFOUND)) + found.not_found = !found.aliased; // the target of an alias is another matter + else if (ec != c_ares_async::make_ares_error(ARES_ENODATA)) { found.lookup_failed = true; found.notes.push_back( @@ -192,11 +193,11 @@ anyhttp::Task Prober::resolve_service(const std::string& host) if (alias->target.empty() || alias->target == ".") { // - // "The service is not available" (section 2.5.1), which is advisory. Treated like no - // record at all, what comes of that is the connection attempt's to find out. + // "The service is not available" (section 2.5.1). This is advisory, but one who does not + // have to request anything can as well take the host's word for it. // found.notes.push_back(std::format("'{}' says the service is not available", found.owner)); - found.aliased = false; + found.unavailable = true; co_return found; } @@ -221,16 +222,16 @@ anyhttp::Task Prober::probe(const std::string& host) result.aliased = found.aliased; result.records = found.records; - const auto plan = plan_routes(host, found); + const auto plan = plan_routes(host, found, config_.default_port); result.notes = plan.rejected; for (const auto& rejected : plan.rejected) spdlog::debug("{}: rejected: {}", host, rejected); if (plan.routes.empty()) { - result.outcome = plan.h3_out_of_reach ? Result::Outcome::out_of_reach - : found.lookup_failed ? Result::Outcome::lookup_failed - : Result::Outcome::no_http3; + result.outcome = found.lookup_failed ? Result::Outcome::lookup_failed + : found.not_found ? Result::Outcome::not_found + : Result::Outcome::out_of_reach; co_return result; } @@ -238,82 +239,114 @@ anyhttp::Task Prober::probe(const std::string& host) for (const auto& route : plan.routes) { spdlog::debug("{}: connecting by {}, for {}", host, route.to_string(), route.source); - const auto start = std::chrono::steady_clock::now(); auto response = co_await attempt(route, host); if (response.ec) { result.error = std::format("[{}] {}", route.to_string(), response.error); result.notes.push_back(result.error); spdlog::debug("{}: {}", host, result.error); + if (response.request_sent) + break; // the host may have seen it, and another route would make it two continue; } result.outcome = Result::Outcome::answered; result.error.clear(); result.status = static_cast(response.message.result_int()); - result.route = route.to_string(); + auto spoken = route; + spoken.protocol = response.protocol; + result.route = spoken.to_string(); for (const auto& field : response.message) result.fields.emplace_back(field.name_string(), field.value()); result.body_size = response.message.body().size(); - result.duration = std::chrono::duration_cast( - std::chrono::steady_clock::now() - start); + result.duration = response.duration; break; } co_return result; } // -// Connects as the route says and asks for the host's root. Everything is reported rather than -// thrown: when_either() would otherwise sit out the timeout, as what throws does not count as -// having finished first. +// Connects as the route says and asks for the host's root, within the timeout, and after that ends +// the connection in an orderly way, which has limits of its own and is none of the timeout's +// business: the response is there by then. // anyhttp::Task Prober::attempt(const Route& route, const std::string& host) { anyhttp::client::Config client_config; - client_config.protocol = anyhttp::Protocol::h3; + client_config.protocol = route.protocol; client_config.tls_ca_file = config_.tls_ca_file; // // The name of the route, as that is also what anyhttp verifies the certificate against, which - // leaves it to its own resolver to look up the address. + // leaves it to its own resolver to look up the address. The scheme is what makes it TLS over + // TCP, too. // client_config.url.set_scheme("https"); client_config.url.set_host_address(route.host); client_config.url.set_port_number(route.port); - const auto run = [](anyhttp::Executor executor, anyhttp::client::Config client_config, - boost::urls::url url) -> anyhttp::Task { - try - { - anyhttp::client::Client client(executor, std::move(client_config)); - auto [connect_ec, session] = co_await client.connect(); - if (connect_ec) - co_return Response{.ec = connect_ec, - .error = std::format("connect: {}", connect_ec.message())}; - - anyhttp::Fields headers; - headers.set(http::field::user_agent, user_agent); - auto [get_ec, message] = co_await session.get(std::move(url), std::move(headers)); - co_return Response{.ec = get_ec, - .message = std::move(message), - .error = get_ec ? std::format("request: {}", get_ec.message()) : ""}; - } - catch (const std::exception& ex) - { - co_return Response{.ec = make_error_code(boost::system::errc::connection_aborted), - .error = ex.what()}; - } - }; + const auto start = std::chrono::steady_clock::now(); + anyhttp::client::Client client(executor_, std::move(client_config)); + anyhttp::Session session; + bool request_sent = false; + // + // These stay around for as long as exchange() needs them: when_either() waits for what it has + // cancelled to finish. + // auto result = co_await anyhttp::when_either( - run(executor_, std::move(client_config), boost::urls::url(std::format("https://{}/", host))), + exchange(client, session, request_sent, boost::urls::url(std::format("https://{}/", host))), anyhttp::delay(config_.timeout)); - if (auto* response = std::get_if<0>(&result)) - co_return std::move(*response); + auto* exchanged = std::get_if<0>(&result); + if (!exchanged) + co_return Response{.ec = make_error_code(boost::system::errc::timed_out), + .error = std::format("timed out after {}", config_.timeout), + .request_sent = request_sent}; + + auto response = std::move(*exchanged); + response.request_sent = request_sent; + response.duration = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + + if (!response.ec) + if (auto [ec] = co_await session.shutdown(); ec) + spdlog::debug("{}: shutdown: {}", host, ec.message()); - co_return Response{.ec = make_error_code(boost::system::errc::timed_out), - .error = std::format("timed out after {}", config_.timeout)}; + co_return response; +} + +// +// Everything is reported rather than thrown: when_either() would otherwise sit out the timeout, as +// what throws does not count as having finished first. +// +anyhttp::Task Prober::exchange(anyhttp::client::Client& client, + anyhttp::Session& session, bool& request_sent, + boost::urls::url url) +{ + try + { + auto [connect_ec, connected] = co_await client.connect(); + if (connect_ec) + co_return Response{.ec = connect_ec, + .error = std::format("connect: {}", connect_ec.message())}; + + session = std::move(connected); + + anyhttp::Fields headers; + headers.set(http::field::user_agent, user_agent); + request_sent = true; + auto [get_ec, message] = co_await session.get(std::move(url), std::move(headers)); + co_return Response{.ec = get_ec, + .message = std::move(message), + .error = get_ec ? std::format("request: {}", get_ec.message()) : "", + .protocol = session.protocol()}; + } + catch (const std::exception& ex) + { + co_return Response{.ec = make_error_code(boost::system::errc::connection_aborted), + .error = ex.what()}; + } } // ================================================================================================= diff --git a/src/prober.hpp b/src/prober.hpp index 026f2c3..c0caad9 100644 --- a/src/prober.hpp +++ b/src/prober.hpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -39,6 +40,9 @@ struct Config /// The CA certificates to verify servers against, as a PEM file. Empty for the system's. std::string tls_ca_file; + + /// Where to go for HTTPS when the records don't say. Only tests have a use for anything else. + std::uint16_t default_port = 443; }; /// Whether \p host is a name and nothing but a name: no URL, port, path or wildcard, and not an @@ -57,20 +61,20 @@ struct Result { enum class Outcome { - answered, ///< it has HTTP/3, and a response arrived, whatever its status - failed, ///< it has HTTP/3, and no response could be had - out_of_reach, ///< it has HTTP/3, but on a target that can't be used, see plan_routes() - no_http3, ///< its records offer no HTTP/3, or it has none, or the name does not exist - lookup_failed, ///< DNS gave no answer, so there is no telling + answered, ///< a response arrived, whatever its status + failed, ///< it was connected to, and no response could be had + out_of_reach, ///< not connected to, its records say so or lead elsewhere, see plan_routes() + not_found, ///< not connected to, the name does not exist + lookup_failed, ///< not connected to, DNS gave no answer, so there is no telling }; std::string host; - Outcome outcome = Outcome::no_http3; + Outcome outcome = Outcome::failed; bool aliased = false; ///< its records are those of another name std::vector records; ///< the records that were considered int status = 0; ///< with \c answered - std::string route; ///< with \c answered: what the response came over + std::string route; ///< with \c answered: what the response came over, as the server chose it std::vector> fields; ///< with \c answered std::size_t body_size = 0; ///< with \c answered std::chrono::milliseconds duration{}; ///< with \c answered: of the attempt that got through @@ -80,10 +84,11 @@ struct Result }; /** - * Looks up the HTTPS records of each of Config::hosts with the resolver of c-ares-async, and for - * those that offer HTTP/3, sends one request (see plan_routes()). A response is what comes back, - * whatever its status: a redirect is reported and not followed, nothing in the body is looked at, - * and nothing else on the host is requested. + * Looks up the HTTPS records of each of Config::hosts with the resolver of c-ares-async, and sends + * one request to each host that can be connected to (see plan_routes()). A response is what comes + * back, whatever its status: a redirect is reported and not followed, nothing in the body is looked + * at, and nothing else on the host is requested. The next route is for a connection that did not + * come about: once a request has gone out, that was it for this host, however it ended. * * Hosts are worked on Config::concurrency at a time, so results arrive as they are ready. All of * this is on the one executor, which is also what calls the observer. @@ -105,12 +110,17 @@ class Prober anyhttp::error_code ec; anyhttp::client::Message message; std::string error; + anyhttp::Protocol protocol = anyhttp::Protocol::h3; ///< what the server chose to speak + bool request_sent = false; + std::chrono::milliseconds duration{}; }; anyhttp::Task work(); anyhttp::Task resolve_service(const std::string& host); anyhttp::Task probe(const std::string& host); anyhttp::Task attempt(const Route& route, const std::string& host); + anyhttp::Task exchange(anyhttp::client::Client& client, anyhttp::Session& session, + bool& request_sent, boost::urls::url url); anyhttp::Executor executor_; Config config_; diff --git a/src/routes.cpp b/src/routes.cpp index be5116c..f8c39cb 100644 --- a/src/routes.cpp +++ b/src/routes.cpp @@ -15,9 +15,6 @@ namespace // ================================================================================================= -constexpr std::string_view no_tls_over_tcp = - "needs TLS over TCP, which anyhttp does not have yet (only HTTP/3 has TLS)"; - // // What a ServiceMode record says, once its parameters have been read. // @@ -29,6 +26,23 @@ struct Service std::optional incompatible; // why this record has to be ignored, see section 8 bool offers(std::string_view protocol) const { return std::ranges::contains(alpn, protocol); } + + // + // What to connect with, in the order to try them, which is at most one connection per transport. + // anyhttp's h2 offers HTTP/1.1 along with it, so that a server still has the choice over TCP. + // + std::vector protocols() const + { + using enum anyhttp::Protocol; + std::vector result; + if (offers("h3")) + result.push_back(h3); + if (offers("h2")) + result.push_back(h2); + else if (offers("http/1.1")) + result.push_back(h1); + return result; + } }; std::vector split_list(std::string_view list) @@ -108,12 +122,25 @@ std::string to_text(const c_ares_async::https_record& record) return text; } -std::string Route::to_string() const { return std::format("h3 {}:{}", host, port); } +std::string Route::to_string() const +{ + using enum anyhttp::Protocol; + return std::format("{} {}:{}", protocol == h3 ? "h3" : protocol == h2 ? "h2" : "h1", host, port); +} -Plan plan_routes(std::string_view host, const ServiceRecords& found) +Plan plan_routes(std::string_view host, const ServiceRecords& found, std::uint16_t default_port) { Plan plan{.rejected = found.notes}; + if (found.lookup_failed || found.not_found) + return plan; + + if (found.unavailable) + { + plan.out_of_reach = true; + return plan; + } + auto records = found.records; std::ranges::stable_sort(records, {}, &c_ares_async::https_record::priority); @@ -127,34 +154,59 @@ Plan plan_routes(std::string_view host, const ServiceRecords& found) plan.rejected.push_back(std::format("{}: {}", text, *service.incompatible)); continue; } + + const auto protocols = service.protocols(); + if (protocols.empty()) + { + plan.rejected.push_back(std::format("{}: no protocol that anyhttp speaks", text)); + continue; + } any_compatible = true; const auto target = normalize_host(is_own_name(record.target) ? found.owner : record.target); - if (!service.offers("h3")) - plan.rejected.push_back(std::format("{}: {}", text, no_tls_over_tcp)); - else if (target != host) + if (target != host) { - plan.h3_out_of_reach = true; plan.rejected.push_back(std::format( "{}: target '{}' is not the host, and anyhttp can only verify the name it connects to", text, target)); + continue; + } + + for (const auto protocol : protocols) + { + Route route{.protocol = protocol, + .host = target, + .port = service.port.value_or(default_port), + .source = text}; + const auto same = [&](const Route& other) { + return other.protocol == route.protocol && other.port == route.port; + }; + if (std::ranges::none_of(plan.routes, same)) // a second try would only fail the same way + plan.routes.push_back(std::move(route)); } - else - plan.routes.push_back( - Route{.host = target, .port = service.port.value_or(443), .source = text}); } + if (!plan.routes.empty()) + return plan; + // - // After an alias, the name it led to is an endpoint even without any parameters (section 3), - // and that is what is left for a name without any usable record, too. Either way it comes down - // to the default ALPN set. + // After an alias, the name it led to is an endpoint even without any parameters (section 3). // - if (found.aliased) - plan.rejected.push_back( - std::format("alias target '{}' without parameters: {}", found.owner, no_tls_over_tcp)); - else if (!any_compatible) - plan.rejected.push_back(std::format("no usable HTTPS record: {}", no_tls_over_tcp)); + if (found.aliased && !any_compatible) + plan.rejected.push_back(std::format( + "alias target '{}' is not the host, and anyhttp can only verify the name it connects to", + found.owner)); + + if (found.aliased || any_compatible) + { + plan.out_of_reach = true; + return plan; + } + plan.routes.push_back(Route{.protocol = anyhttp::Protocol::h2, + .host = std::string(host), + .port = default_port, + .source = "no usable HTTPS record"}); return plan; } diff --git a/src/routes.hpp b/src/routes.hpp index 63cce51..9d989dc 100644 --- a/src/routes.hpp +++ b/src/routes.hpp @@ -4,6 +4,8 @@ // #pragma once +#include + #include #include @@ -32,11 +34,14 @@ struct ServiceRecords std::vector records; ///< the ServiceMode records of \c owner std::vector notes; ///< what went wrong on the way, none of which is fatal bool lookup_failed = false; ///< no answer, which is not to say that there are no records + bool not_found = false; ///< the host itself does not exist + bool unavailable = false; ///< an alias to "." says that the service is not available }; -/// One way of connecting, which is HTTP/3 so far. The TLS name and the authority stay the host's. +/// One way of connecting, always over TLS. The TLS name and the authority stay the host's. struct Route { + anyhttp::Protocol protocol = anyhttp::Protocol::h3; std::string host; ///< the name to connect to std::uint16_t port = 443; std::string source; ///< the record that led to this route @@ -49,18 +54,23 @@ struct Plan { std::vector routes; ///< in the order to try them; empty if there is no usable way std::vector rejected; ///< what the records offered, or we knew of, and why not - bool h3_out_of_reach = false; ///< a record offers HTTP/3, but on a target that can't be used + bool out_of_reach = false; ///< the records say not to connect, or only to a target we can't use }; /** * Says how to connect to https://\p host/, given what was found for it. * - * What is usable is limited by anyhttp as much as by the records. TLS exists on HTTP/3 only, so - * a host can be reached through a record that has "h3" in its ALPN set and in no other way. And + * Records go by priority, and within a record, HTTP/3 comes ahead of TCP. A name without a usable + * record, the most common thing there is, gets \p default_port over TCP, and so does a record that + * does not say which port (section 9.1 on both of them). + * * anyhttp verifies the certificate against the host it connects to, while RFC 9460 (section 9.1) - * wants that to be the origin, so a record is usable only if it stays on the host's own name. + * wants that to be the origin, so a record is usable only if it stays on the host's own name. If + * none of them does, the host is out of reach: the records do say where to go, and it is not here. + * A name that does not exist and a lookup that failed have no routes either. */ -Plan plan_routes(std::string_view host, const ServiceRecords& found); +Plan plan_routes(std::string_view host, const ServiceRecords& found, + std::uint16_t default_port = 443); // ================================================================================================= diff --git a/test/output_test.cpp b/test/output_test.cpp index bd95c32..e7325fd 100644 --- a/test/output_test.cpp +++ b/test/output_test.cpp @@ -48,9 +48,9 @@ TEST(Output, SaysWhyThereIsNoResponse) false), "example.com: failed: [h3 example.com:443] timed out\n note: some note\n"); EXPECT_EQ(format_result({.host = "example.com", .outcome = Outcome::out_of_reach}, false), - "example.com: HTTP/3 out of reach\n"); - EXPECT_EQ(format_result({.host = "example.com", .outcome = Outcome::no_http3}, false), - "example.com: no HTTP/3\n"); + "example.com: out of reach\n"); + EXPECT_EQ(format_result({.host = "example.com", .outcome = Outcome::not_found}, false), + "example.com: not found\n"); EXPECT_EQ(format_result({.host = "example.com", .outcome = Outcome::lookup_failed}, false), "example.com: lookup failed\n"); } @@ -96,11 +96,10 @@ TEST(Output, CountsTheOutcomes) { EXPECT_EQ(format_summary({{Outcome::answered, 3}, {Outcome::failed, 1}, - {Outcome::no_http3, 900}, + {Outcome::not_found, 900}, {Outcome::lookup_failed, 17}}, false), - "3 answered, 1 failed, 0 with HTTP/3 out of reach, 900 without HTTP/3, " - "17 lookups failed"); + "3 answered, 1 failed, 0 out of reach, 900 not found, 17 lookups failed"); } TEST(Output, DrawsAttentionOnlyToWhatHappened) diff --git a/test/prober_test.cpp b/test/prober_test.cpp index 51b563c..9de9d80 100644 --- a/test/prober_test.cpp +++ b/test/prober_test.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -37,14 +38,30 @@ using testing::Pair; namespace { -std::vector h3_at(std::uint16_t port) +fake_dns_server::svc_param port_param(std::uint16_t port) { - return {{ARES_SVCB_PARAM_ALPN, {2, 'h', '3'}}, - {ARES_SVCB_PARAM_PORT, - {static_cast(port >> 8), static_cast(port & 0xff)}}}; + return {ARES_SVCB_PARAM_PORT, + {static_cast(port >> 8), static_cast(port & 0xff)}}; } +const fake_dns_server::svc_param alpn_h3{ARES_SVCB_PARAM_ALPN, {2, 'h', '3'}}; const fake_dns_server::svc_param alpn_h2{ARES_SVCB_PARAM_ALPN, {2, 'h', '2'}}; +const fake_dns_server::svc_param alpn_h1{ARES_SVCB_PARAM_ALPN, + {8, 'h', 't', 't', 'p', '/', '1', '.', '1'}}; +const fake_dns_server::svc_param no_default_alpn{ARES_SVCB_PARAM_NO_DEFAULT_ALPN, {}}; + +// +// One protocol and no other, for a test to know which one it is getting. A record with nothing but +// an "alpn" also has the default set, and that means trying HTTP/1.1 as well. +// +std::vector only(fake_dns_server::svc_param alpn, std::uint16_t port) +{ + return {std::move(alpn), no_default_alpn, port_param(port)}; +} + +std::vector h3_at(std::uint16_t port) { return only(alpn_h3, port); } +std::vector h2_at(std::uint16_t port) { return only(alpn_h2, port); } +std::vector h1_at(std::uint16_t port) { return only(alpn_h1, port); } class Probe : public testing::Test { @@ -54,7 +71,8 @@ class Probe : public testing::Test return {.hosts = std::move(hosts), .timeout = 1000ms, .dns_server = site.dns_address(), - .tls_ca_file = site.ca_file().string()}; + .tls_ca_file = site.ca_file().string(), + .default_port = site.port()}; } void probe(Config config) @@ -253,7 +271,7 @@ TEST_F(Probe, GoesOnAfterAHostThatFails) ASSERT_EQ(results.size(), 2u); EXPECT_EQ(results[0].host, "nowhere.example.test"); - EXPECT_EQ(results[0].outcome, Result::Outcome::no_http3); + EXPECT_EQ(results[0].outcome, Result::Outcome::not_found); EXPECT_EQ(results[1].host, "localhost"); EXPECT_EQ(results[1].status, 200); } @@ -262,28 +280,23 @@ TEST_F(Probe, GoesOnAfterAHostThatFails) // A list of hosts // ================================================================================================= -TEST_F(Probe, ConnectsToTheHostsWithHttp3AndToNoOthers) +TEST_F(Probe, DoesNotConnectToANameThatDoesNotExist) { // - // The other names exist, with records or without, and their addresses are the server's. It takes - // no more than connecting to any of them to show up in its requests. + // Which would also take the lookup of its address to the resolver of the system, and away from + // loopback, for every test that has a name like this. // - site.dns().add_https("localhost", 1, ".", h3_at(site.port())); - site.dns().add_address("h2.example.test", "127.0.0.1"); - site.dns().add_https("h2.example.test", 1, ".", {alpn_h2}); - site.dns().add_address("plain.example.test", "127.0.0.1"); + site.dns().add_address("localhost", "127.0.0.1"); site.page("/", {.body = "hello"}); - auto configuration = config({"h2.example.test", "plain.example.test", "localhost"}); + auto configuration = config({"nowhere.example.test", "localhost"}); configuration.concurrency = 1; probe(std::move(configuration)); - ASSERT_EQ(results.size(), 3u); - EXPECT_EQ(results[0].outcome, Result::Outcome::no_http3); - EXPECT_EQ(results[1].outcome, Result::Outcome::no_http3); - EXPECT_EQ(results[2].outcome, Result::Outcome::answered) << notes(results[2]); + ASSERT_EQ(results.size(), 2u); + EXPECT_EQ(results[0].outcome, Result::Outcome::not_found); + EXPECT_EQ(results[1].outcome, Result::Outcome::answered) << notes(results[1]); EXPECT_THAT(site.targets(), ElementsAre("/")); - EXPECT_EQ(site.requests()[0].authority, "localhost"); } TEST_F(Probe, ReportsEachHostOnceHoweverManyThereAre) @@ -343,18 +356,90 @@ TEST_F(Probe, WorksOnOneHostAtATimeIfToldTo) // How the HTTPS records decide about the connection // ================================================================================================= -TEST_F(Probe, SkipsARecordItCanNotUse) +TEST_F(Probe, SpeaksHttp2OverTlsWhereTheRecordSaysSo) { - site.dns().add_https("localhost", 1, ".", {alpn_h2}); - site.dns().add_https("localhost", 2, ".", h3_at(site.port())); + site.dns().add_https("localhost", 1, ".", h2_at(site.port())); + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 200) << notes(results[0]); + EXPECT_EQ(results[0].route, std::format("h2 localhost:{}", site.port())); + EXPECT_EQ(results[0].body_size, 5u); + ASSERT_EQ(site.requests().size(), 1u); + EXPECT_EQ(site.requests()[0].authority, "localhost"); + EXPECT_THAT(site.requests()[0].user_agent, HasSubstr("http-probe/")); +} + +TEST_F(Probe, SpeaksHttp1OverTlsWhereTheRecordSaysSo) +{ + site.dns().add_https("localhost", 1, ".", h1_at(site.port())); site.page("/", {.body = "hello"}); probe(config()); ASSERT_EQ(results.size(), 1u); EXPECT_EQ(results[0].status, 200) << notes(results[0]); - EXPECT_THAT(notes(results[0]), HasSubstr("alpn=h2")); - EXPECT_THAT(notes(results[0]), HasSubstr("TLS over TCP")); + EXPECT_EQ(results[0].route, std::format("h1 localhost:{}", site.port())); + EXPECT_EQ(results[0].body_size, 5u); + ASSERT_EQ(site.requests().size(), 1u); + EXPECT_THAT(site.requests()[0].authority, HasSubstr("localhost")); +} + +TEST_F(Probe, PrefersHttp3OfWhatARecordOffers) +{ + site.dns().add_https("localhost", 1, ".", {alpn_h3, port_param(site.port())}); + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].route, std::format("h3 localhost:{}", site.port())) << notes(results[0]); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, GoesOverTcpToTheDefaultPortWithoutARecord) +{ + site.dns().add_address("localhost", "127.0.0.1"); + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::answered) << notes(results[0]); + EXPECT_EQ(results[0].route, std::format("h2 localhost:{}", site.port())); + EXPECT_THAT(results[0].records, IsEmpty()); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, DoesNotFollowARedirectOverHttp2) +{ + site.dns().add_https("localhost", 1, ".", h2_at(site.port())); + site.page("/", {.status = 302, .location = "/moved"}); + site.page("/moved", {.body = "not asked for"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 302) << notes(results[0]); + EXPECT_THAT(results[0].fields, Contains(Pair("location", "/moved"))); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, DoesNotFollowARedirectOverHttp1) +{ + site.dns().add_https("localhost", 1, ".", h1_at(site.port())); + site.page("/", {.status = 302, .location = "/moved"}); + site.page("/moved", {.body = "not asked for"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 302) << notes(results[0]); + // As it came, and only HTTP/2 and HTTP/3 have field names in lower case on the wire. + EXPECT_THAT(results[0].fields, Contains(Pair("Location", "/moved"))); + EXPECT_THAT(site.targets(), ElementsAre("/")); } TEST_F(Probe, FallsBackToTheNextRecordWhenAConnectionFails) @@ -380,16 +465,59 @@ TEST_F(Probe, FallsBackToTheNextRecordWhenAConnectionFails) EXPECT_THAT(notes(results[0]), HasSubstr("timed out")); } -TEST_F(Probe, FailsWithoutARecord) +TEST_F(Probe, FallsBackFromHttp3ToTcpWhenAConnectionFails) { + boost::asio::ip::udp::socket silent(context, {boost::asio::ip::udp::v6(), 0}); + abandon_quic_sessions = true; + + site.dns().add_https("localhost", 1, ".", h3_at(silent.local_endpoint().port())); + site.dns().add_https("localhost", 2, ".", h2_at(site.port())); site.page("/", {.body = "hello"}); - probe(config()); + auto configuration = config(); + configuration.timeout = 200ms; + probe(std::move(configuration)); ASSERT_EQ(results.size(), 1u); - EXPECT_EQ(results[0].outcome, Result::Outcome::no_http3); - EXPECT_THAT(notes(results[0]), HasSubstr("TLS over TCP")); - EXPECT_THAT(site.requests(), IsEmpty()); + EXPECT_EQ(results[0].status, 200) << notes(results[0]); + EXPECT_EQ(results[0].route, std::format("h2 localhost:{}", site.port())); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, TriesNoOtherRouteOnceARequestHasGoneOut) +{ + // + // The server has it, it is just not done with it. Everything after this would be a second + // request to the same host: over HTTP/1.1 on the same record, and over HTTP/2 on the next. + // + site.dns().add_https("localhost", 1, ".", {alpn_h3, port_param(site.port())}); + site.dns().add_https("localhost", 2, ".", h2_at(site.port())); + site.page("/", {.body = "too late", .delay = 400ms}); + + auto configuration = config(); + configuration.timeout = 100ms; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::failed); + EXPECT_THAT(results[0].error, HasSubstr("timed out")); + EXPECT_THAT(results[0].error, HasSubstr("h3")); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, TriesNoOtherRouteOnceARequestHasGoneOutOverTcp) +{ + site.dns().add_https("localhost", 1, ".", h2_at(site.port())); + site.dns().add_https("localhost", 2, ".", h1_at(site.port())); + site.page("/", {.body = "too late", .delay = 400ms}); + + auto configuration = config(); + configuration.timeout = 100ms; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::failed); + EXPECT_THAT(site.targets(), ElementsAre("/")); } TEST_F(Probe, DoesNotConnectToATargetOtherThanTheHost) @@ -425,6 +553,23 @@ TEST_F(Probe, RefusesACertificateItDoesNotTrust) EXPECT_THAT(site.requests(), IsEmpty()) << "no request survives the handshake"; } +TEST_F(Probe, RefusesACertificateItDoesNotTrustOverTcp) +{ + Site other(context.get_executor()); + site.dns().add_https("localhost", 1, ".", h2_at(site.port())); + site.page("/", {.body = "hello"}); + + auto configuration = config(); + configuration.tls_ca_file = other.ca_file().string(); + other.stop(); + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::failed); + EXPECT_THAT(results[0].error, HasSubstr("connect")); + EXPECT_THAT(site.requests(), IsEmpty()) << "no request survives the handshake"; +} + TEST_F(Probe, FollowsAnAliasToItsRecords) { // @@ -446,12 +591,17 @@ TEST_F(Probe, RecognizesAnAliasLoop) { site.dns().add_https("localhost", 0, "alias.example.test"); site.dns().add_https("alias.example.test", 0, "localhost"); + site.page("/", {.body = "hello"}); probe(config()); + // + // As if it had no record at all (section 3.1), which leaves the default port. + // ASSERT_EQ(results.size(), 1u); - EXPECT_EQ(results[0].outcome, Result::Outcome::no_http3); EXPECT_THAT(notes(results[0]), HasSubstr("too long or has a loop")); + EXPECT_EQ(results[0].outcome, Result::Outcome::answered) << notes(results[0]); + EXPECT_EQ(results[0].route, std::format("h2 localhost:{}", site.port())); } TEST_F(Probe, IgnoresServiceRecordsNextToAnAlias) @@ -463,12 +613,25 @@ TEST_F(Probe, IgnoresServiceRecordsNextToAnAlias) probe(config()); ASSERT_EQ(results.size(), 1u); - EXPECT_EQ(results[0].outcome, Result::Outcome::no_http3) + EXPECT_EQ(results[0].outcome, Result::Outcome::out_of_reach) << "the alias has no records of its own"; EXPECT_TRUE(results[0].aliased); EXPECT_THAT(site.requests(), IsEmpty()); } +TEST_F(Probe, TakesAHostsWordForItThatTheServiceIsNotAvailable) +{ + site.dns().add_https("localhost", 0, "."); + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::out_of_reach); + EXPECT_THAT(notes(results[0]), HasSubstr("not available")); + EXPECT_THAT(site.requests(), IsEmpty()); +} + // ================================================================================================= // What can be seen of it // ================================================================================================= @@ -518,12 +681,45 @@ TEST_F(Probe, GivesUpOnAServerThatDoesNotAnswer) EXPECT_THAT(results[0].error, HasSubstr("timed out")); } +TEST_F(Probe, GivesUpOnATcpServerThatDoesNotAnswer) +{ + // + // It takes connections, as the kernel does that for it, and leaves the handshake hanging. + // + boost::asio::ip::tcp::acceptor silent(context, {boost::asio::ip::tcp::v6(), 0}); + site.dns().add_https("localhost", 1, ".", h2_at(silent.local_endpoint().port())); + + auto configuration = config(); + configuration.timeout = 100ms; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::failed); + EXPECT_THAT(results[0].error, HasSubstr("timed out")); +} + +TEST_F(Probe, ReportsAPortThatIsClosed) +{ + std::uint16_t closed = 0; + { + boost::asio::ip::tcp::acceptor acceptor(context, {boost::asio::ip::tcp::v6(), 0}); + closed = acceptor.local_endpoint().port(); + } + site.dns().add_https("localhost", 1, ".", h2_at(closed)); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::failed); + EXPECT_THAT(results[0].error, HasSubstr("connect")); +} + TEST_F(Probe, ReportsANameThatDoesNotExist) { probe(config({"nowhere.example.test"})); ASSERT_EQ(results.size(), 1u); - EXPECT_EQ(results[0].outcome, Result::Outcome::no_http3); + EXPECT_EQ(results[0].outcome, Result::Outcome::not_found); EXPECT_THAT(site.requests(), IsEmpty()); } diff --git a/test/routes_test.cpp b/test/routes_test.cpp index c6e4fd6..c029f43 100644 --- a/test/routes_test.cpp +++ b/test/routes_test.cpp @@ -52,70 +52,133 @@ TEST(ToText, ReadsLikeAZoneFile) // ------------------------------------------------------------------------------------------------- -TEST(Plan, ConnectsOverHttp3WhereTheRecordOffersIt) +TEST(Plan, ConnectsOverHttp3FirstAndThenOverTcp) { const auto plan = plan_routes("example.com", found({record(1, ".", {"alpn=h3,h2"})})); + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443", "h2 example.com:443")); +} + +TEST(Plan, TakesTheDefaultSetIntoAccount) +{ + // + // "alpn=h3" alone does not mean HTTP/3 alone: the default set is still there (section 7.1.2). + // + const auto plan = plan_routes("example.com", found({record(1, ".", {"alpn=h3"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443", "h1 example.com:443")); +} + +TEST(Plan, HasNoTcpWithNoDefaultAlpn) +{ + const auto plan = + plan_routes("example.com", found({record(1, ".", {"alpn=h3", "no-default-alpn"})})); + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443")); } +TEST(Plan, ConnectsOverTcpWhereTheRecordOffersNoHttp3) +{ + EXPECT_THAT(routes(plan_routes("example.com", found({record(1, ".", {"alpn=h2"})}))), + ElementsAre("h2 example.com:443")); + EXPECT_THAT(routes(plan_routes("example.com", found({record(1, ".", {"alpn=http/1.1"})}))), + ElementsAre("h1 example.com:443")); + EXPECT_THAT(routes(plan_routes("example.com", found({record(1, ".")}))), + ElementsAre("h1 example.com:443")); +} + TEST(Plan, GoesToThePortOfTheRecord) { const auto plan = plan_routes("example.com", found({record(1, ".", {"alpn=h3", "port=8443"})})); - EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:8443")); + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:8443", "h1 example.com:8443")); } TEST(Plan, TriesTheLowestPriorityFirst) { - const auto plan = plan_routes("example.com", found({record(3, ".", {"alpn=h3", "port=3"}), - record(1, ".", {"alpn=h3", "port=1"}), - record(2, ".", {"alpn=h3", "port=2"})})); + const auto plan = + plan_routes("example.com", found({record(3, ".", {"alpn=h3", "no-default-alpn", "port=3"}), + record(1, ".", {"alpn=h3", "no-default-alpn", "port=1"}), + record(2, ".", {"alpn=h2", "no-default-alpn", "port=2"})})); EXPECT_THAT(routes(plan), - ElementsAre("h3 example.com:1", "h3 example.com:2", "h3 example.com:3")); + ElementsAre("h3 example.com:1", "h2 example.com:2", "h3 example.com:3")); } -TEST(Plan, RejectsARecordWithoutHttp3) +TEST(Plan, DoesNotTryTheSameThingTwice) { + // + // Records that differ only in what is of no use here, like their address hints. + // const auto plan = - plan_routes("example.com", - found({record(1, ".", {"alpn=h2"}), record(2, ".", {"alpn=h3", "port=8443"})})); + plan_routes("example.com", found({record(1, ".", {"alpn=h3", "ipv4hint=192.0.2.1"}), + record(2, ".", {"alpn=h3", "ipv4hint=192.0.2.2"})})); - EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:8443")); - ASSERT_EQ(plan.rejected.size(), 1u); - EXPECT_THAT(plan.rejected[0], HasSubstr("alpn=h2")); - EXPECT_THAT(plan.rejected[0], HasSubstr("TLS over TCP")); + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443", "h1 example.com:443")); +} + +TEST(Plan, GoesOverTcpToTheDefaultPortWithoutARecord) +{ + EXPECT_THAT(routes(plan_routes("example.com", found({}))), ElementsAre("h2 example.com:443")); + EXPECT_THAT(plan_routes("example.com", found({})).rejected, IsEmpty()) << "nothing odd about it"; + EXPECT_THAT(routes(plan_routes("example.com", found({}), 8443)), + ElementsAre("h2 example.com:8443")); } -TEST(Plan, HasNoRouteWithoutARecord) +TEST(Plan, RecordsThatDoNotSayGoToTheDefaultPort) { - const auto plan = plan_routes("example.com", found({})); + EXPECT_THAT(routes(plan_routes("example.com", found({record(1, ".", {"alpn=h2"})}), 8443)), + ElementsAre("h2 example.com:8443")); +} + +TEST(Plan, HasNoRouteForANameThatDoesNotExist) +{ + auto records = found({}); + records.not_found = true; + + const auto plan = plan_routes("example.com", records); EXPECT_THAT(plan.routes, IsEmpty()); - ASSERT_EQ(plan.rejected.size(), 1u); - EXPECT_THAT(plan.rejected[0], HasSubstr("TLS over TCP")); + EXPECT_FALSE(plan.out_of_reach); } -TEST(Plan, RejectsATargetOtherThanTheHost) +TEST(Plan, HasNoRouteAfterALookupThatFailed) { - // - // The certificate has to be valid for the origin (section 9.1), and anyhttp would check it - // against what it connects to. - // - const auto plan = plan_routes("example.com", found({record(1, "cdn.example.net", {"alpn=h3"})})); + auto records = found({}); + records.lookup_failed = true; + + const auto plan = plan_routes("example.com", records); EXPECT_THAT(plan.routes, IsEmpty()); - ASSERT_EQ(plan.rejected.size(), 1u); - EXPECT_THAT(plan.rejected[0], HasSubstr("cdn.example.net")); - EXPECT_TRUE(plan.h3_out_of_reach); + EXPECT_FALSE(plan.out_of_reach); } -TEST(Plan, DoesNotCallItOutOfReachWithoutHttp3) +TEST(Plan, TakesAHostsWordForItThatTheServiceIsNotAvailable) { - const auto plan = plan_routes("example.com", found({record(1, "cdn.example.net", {"alpn=h2"})})); + auto records = found({}); + records.unavailable = true; - EXPECT_FALSE(plan.h3_out_of_reach); + const auto plan = plan_routes("example.com", records); + + EXPECT_THAT(plan.routes, IsEmpty()); + EXPECT_TRUE(plan.out_of_reach); +} + +TEST(Plan, RejectsATargetOtherThanTheHost) +{ + // + // The certificate has to be valid for the origin (section 9.1), and anyhttp would check it + // against what it connects to. + // + for (const auto* alpn : {"alpn=h3", "alpn=h2"}) + { + const auto plan = plan_routes("example.com", found({record(1, "cdn.example.net", {alpn})})); + + EXPECT_THAT(plan.routes, IsEmpty()); + ASSERT_EQ(plan.rejected.size(), 1u); + EXPECT_THAT(plan.rejected[0], HasSubstr("cdn.example.net")); + EXPECT_TRUE(plan.out_of_reach); + } } TEST(Plan, UsesTheRecordItCanOfAMixedSet) @@ -127,16 +190,17 @@ TEST(Plan, UsesTheRecordItCanOfAMixedSet) "example.com", found({record(1, ".", {"alpn=h2,h3"}), record(2, "fallback.example.net", {"alpn=h2,h3"})})); - EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443")); + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443", "h2 example.com:443")); ASSERT_EQ(plan.rejected.size(), 1u); EXPECT_THAT(plan.rejected[0], HasSubstr("fallback.example.net")); + EXPECT_FALSE(plan.out_of_reach); } TEST(Plan, DoesNotTakeSomethingThatOnlyStartsWithH3) { const auto plan = plan_routes("example.com", found({record(1, ".", {"alpn=h3-29,h2"})})); - EXPECT_THAT(plan.routes, IsEmpty()); + EXPECT_THAT(routes(plan), ElementsAre("h2 example.com:443")); } TEST(Plan, IgnoresWhatItDoesNotKnowUnlessItIsMandatory) @@ -147,14 +211,14 @@ TEST(Plan, IgnoresWhatItDoesNotKnowUnlessItIsMandatory) const auto plan = plan_routes("example.com", found({record(1, ".", {"alpn=h3", "key13930=01", "ech=dead"})})); - EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443")); + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443", "h1 example.com:443")); } TEST(Plan, TakesAnExplicitTargetThatIsTheHost) { - const auto plan = plan_routes("example.com", found({record(1, "EXAMPLE.com.", {"alpn=h3"})})); + const auto plan = plan_routes("example.com", found({record(1, "EXAMPLE.com.", {"alpn=h2"})})); - EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443")); + EXPECT_THAT(routes(plan), ElementsAre("h2 example.com:443")); } TEST(Plan, TakesTheOwnNameOfARecordAtTheEndOfAnAliasAsThatName) @@ -166,6 +230,7 @@ TEST(Plan, TakesTheOwnNameOfARecordAtTheEndOfAnAliasAsThatName) plan_routes("example.com", found({record(1, ".", {"alpn=h3"})}, "other.example", true)); EXPECT_THAT(plan.routes, IsEmpty()); + EXPECT_TRUE(plan.out_of_reach); EXPECT_THAT(plan.rejected.front(), HasSubstr("other.example")); } @@ -174,33 +239,38 @@ TEST(Plan, SaysSoAboutTheTargetOfAnAlias) const auto plan = plan_routes("example.com", found({}, "other.example", true)); EXPECT_THAT(plan.routes, IsEmpty()); + EXPECT_TRUE(plan.out_of_reach); ASSERT_EQ(plan.rejected.size(), 1u); EXPECT_THAT(plan.rejected[0], HasSubstr("other.example")); } -TEST(Plan, HasNoDefaultSetToFallBackOnWithNoDefaultAlpn) +TEST(Plan, TreatsARecordWithNoProtocolItSpeaksAsNoRecord) { const auto plan = plan_routes("example.com", found({record(1, ".", {"no-default-alpn"})})); - EXPECT_THAT(plan.routes, IsEmpty()); + EXPECT_THAT(routes(plan), ElementsAre("h2 example.com:443")); + EXPECT_THAT(plan.rejected.front(), HasSubstr("no protocol")); } -TEST(Plan, IgnoresARecordWithAMandatoryKeyItDoesNotKnow) +TEST(Plan, TreatsARecordWithAMandatoryKeyItDoesNotKnowAsNoRecord) { + // + // Section 8: what is left is the name without a record, and that goes for all that follow. + // const auto plan = plan_routes("example.com", found({record(1, ".", {"mandatory=alpn,key65280", "alpn=h3", "key65280=01"})})); - EXPECT_THAT(plan.routes, IsEmpty()); + EXPECT_THAT(routes(plan), ElementsAre("h2 example.com:443")); EXPECT_THAT(plan.rejected.front(), HasSubstr("key65280")); } -TEST(Plan, IgnoresARecordThatRequiresEch) +TEST(Plan, TreatsARecordThatRequiresEchAsNoRecord) { const auto plan = plan_routes("example.com", found({record(1, ".", {"mandatory=ech", "alpn=h3", "ech=dead"})})); - EXPECT_THAT(plan.routes, IsEmpty()); + EXPECT_THAT(routes(plan), ElementsAre("h2 example.com:443")); } TEST(Plan, AcceptsMandatoryKeysItCanLeaveAlone) @@ -209,14 +279,14 @@ TEST(Plan, AcceptsMandatoryKeysItCanLeaveAlone) "example.com", found({record(1, ".", {"mandatory=alpn,ipv4hint", "alpn=h3", "ipv4hint=192.0.2.1"})})); - EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443")); + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443", "h1 example.com:443")); } -TEST(Plan, IgnoresARecordWithAnInvalidPort) +TEST(Plan, TreatsARecordWithAnInvalidPortAsNoRecord) { const auto plan = plan_routes("example.com", found({record(1, ".", {"alpn=h3", "port=x"})})); - EXPECT_THAT(plan.routes, IsEmpty()); + EXPECT_THAT(routes(plan), ElementsAre("h2 example.com:443")); } TEST(Plan, PassesOnWhatWentWrongInTheLookup) diff --git a/test/site.cpp b/test/site.cpp index 8b4c523..68513a9 100644 --- a/test/site.cpp +++ b/test/site.cpp @@ -52,6 +52,9 @@ Site::Site(const anyhttp::Executor& executor) : requests_(std::make_sharedsecond : Page{.status = 404}; + if (page.delay.count() > 0) + co_await anyhttp::delay(page.delay); + anyhttp::Fields fields; fields.set(http::field::content_type, page.content_type); if (!page.location.empty()) diff --git a/test/site.hpp b/test/site.hpp index b3a7368..281f0a9 100644 --- a/test/site.hpp +++ b/test/site.hpp @@ -2,7 +2,7 @@ // A web site with DNS of its own, all on loopback, for the program to be pointed at: an anyhttp // server on one port, for every protocol, and a DNS server with whatever zone the test wants. The // server has a certificate for "localhost", the one name that also resolves without our DNS server, -// which anyhttp's own resolver needs for HTTP/3. +// which anyhttp's own resolver needs. // #pragma once @@ -34,6 +34,7 @@ class Site std::string body; std::string content_type = "text/html"; std::string location; + std::chrono::milliseconds delay{}; ///< how long the server takes to get to it }; /// One request that arrived. From b8e48ddb34416465907375515d15d4a57e555548 Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Tue, 3 Nov 2026 07:44:37 +0000 Subject: [PATCH 13/27] chore: take both libraries from GitHub only, at their pins Remove the dev preset and the mounts of the sibling checkouts, so that everything builds the way any other user gets it. Whatever other agents do in the libraries now arrives here with a pin move and not before. With a single preset left, its tree moves to build/, where clangd looks. Co-Authored-By: Claude Opus 5.5 --- .devcontainer/devcontainer.json | 2 -- CLAUDE.md | 11 ++++++----- CMakeLists.txt | 5 ++--- CMakePresets.json | 24 ++---------------------- README.md | 10 +++------- 5 files changed, 13 insertions(+), 39 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b7fd2de..dafc6ca 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -75,8 +75,6 @@ "mounts": [ "source=cpp-devcontainer-bashhistory,target=/commandhistory,type=volume", - "source=${localEnv:HOME}/src/anyhttp,target=/workspaces/anyhttp,type=bind,consistency=cached,readonly", - "source=${localEnv:HOME}/src/c-ares-async,target=/workspaces/c-ares-async,type=bind,consistency=cached,readonly", "source=${localEnv:HOME}/.claude,target=/home/vscode/.claude,type=bind,consistency=cached", "source=${localEnv:HOME}/.claude.json,target=/home/vscode/.claude.json,type=bind,consistency=cached" ], diff --git a/CLAUDE.md b/CLAUDE.md index 2f17bb6..1404155 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -3,7 +3,7 @@ It requests `https:///` once per host and reports what came back, connecting the way the host's HTTPS records say. It is built on two libraries of the same author, each in a repository of its own: anyhttp for HTTP and c-ares-async for DNS. -[README.md](README.md) has the presets and the build commands. +[README.md](README.md) has the build commands. ## Safety rules @@ -32,10 +32,11 @@ any I/O, and [prober.cpp](src/prober.cpp) does the rest. The tests in [test/](te - Ask for a library change with an issue in that library's repository, using `gh`. Peter reviews the full text first, and the same goes for comments on issues and for closing them. The repositories are public, so reproduce on loopback and name no hosts from the lists. -- Moving a pin in [CMakeLists.txt](CMakeLists.txt) is a commit of its own. CI builds from the pins - only, so a change here that needs a library change has to wait for that change to be pushed. -- The `dev` preset builds the sibling checkouts as they are, on whatever branch they are on, and can - differ from the pins. Check with the `pinned` preset before committing. +- Moving a pin in [CMakeLists.txt](CMakeLists.txt) is a commit of its own. Everything builds from + the pins, so a change here that needs a library change has to wait for that change to be pushed. +- The libraries come from GitHub, as for any other user, and there are no checkouts of them to build + against. Other agents working on them do not interfere with this repository: nothing they do + arrives here until a pin moves. To read a library at its pin, look in `build/_deps/`. - Everything in one process has to agree on the standard library, the TLS library and the I/O runtime. [CMakeLists.txt](CMakeLists.txt) decides these for both libraries, so change them there. Do not configure a tree by hand: `cc` and `c++` are GCC on this host, and a tree without clang diff --git a/CMakeLists.txt b/CMakeLists.txt index 2140d76..35d1428 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,9 +38,8 @@ set(C_ARES_ASYNC_TLS ${TLS_LIBRARY}) # -------------------------------------------------------------------------------------------------- # -# Pinned to a commit, and moving a pin is a commit of its own. To build against a checkout instead -# there is the 'dev' preset, which sets FETCHCONTENT_SOURCE_DIR_*. SYSTEM, so that the warnings of -# this project do not apply to the headers of the libraries. +# Pinned to a commit, and moving a pin is a commit of its own. SYSTEM, so that the warnings of this +# project do not apply to the headers of the libraries. # include(FetchContent) FetchContent_Declare(anyhttp diff --git a/CMakePresets.json b/CMakePresets.json index 667b0f3..7d4fddc 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -18,30 +18,15 @@ "CMAKE_EXPORT_COMPILE_COMMANDS": "ON" } }, - { - "name": "dev", - "displayName": "Debug, against sibling checkouts", - "description": "What http-probe is developed with: the two libraries as they are in their checkouts next to this one, whatever the pins say. Keep this tree in build/, clangd reads its compile_commands.json", - "inherits": "base", - "binaryDir": "${sourceDir}/build", - "cacheVariables": { - "FETCHCONTENT_SOURCE_DIR_ANYHTTP": "${sourceDir}/../anyhttp", - "FETCHCONTENT_SOURCE_DIR_C_ARES_ASYNC": "${sourceDir}/../c-ares-async" - } - }, { "name": "pinned", "displayName": "Debug, from the pins", - "description": "The libraries at the commits CMakeLists.txt pins, fetched from GitHub: what CI builds", + "description": "The libraries at the commits CMakeLists.txt pins, fetched from GitHub. Keep this tree in build/, clangd reads its compile_commands.json", "inherits": "base", - "binaryDir": "${sourceDir}/build-pinned" + "binaryDir": "${sourceDir}/build" } ], "buildPresets": [ - { - "name": "dev", - "configurePreset": "dev" - }, { "name": "pinned", "configurePreset": "pinned" @@ -60,11 +45,6 @@ "noTestsAction": "error" } }, - { - "name": "dev", - "configurePreset": "dev", - "inherits": "base" - }, { "name": "pinned", "configurePreset": "pinned", diff --git a/README.md b/README.md index 4849ff3..323fde8 100644 --- a/README.md +++ b/README.md @@ -72,12 +72,8 @@ prints the command to request it with: ## Build C++23 and CMake 3.30 or later. The devcontainer in [.devcontainer](.devcontainer) has the compilers -and every dependency that the two libraries need. There are two presets: - -| Preset | Tree | Libraries come from | -|----------|-----------------|-----------------------------------------------------------------| -| `dev` | `build/` | the checkouts next to this one, as they are | -| `pinned` | `build-pinned/` | GitHub, at the commits pinned in [CMakeLists.txt](CMakeLists.txt); what CI builds | +and every dependency that the two libraries need. The libraries themselves are fetched from GitHub, +at the commits pinned in [CMakeLists.txt](CMakeLists.txt): ```sh cmake --preset pinned @@ -85,4 +81,4 @@ cmake --build --preset pinned ctest --preset pinned ``` -Both presets use clang, libc++, OpenSSL and the Asio runtime. +The tree is `build/`, with clang, libc++, OpenSSL and the Asio runtime. From 02255270c7af9c4767796bb202c7c54a3c5f2f20 Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Tue, 3 Nov 2026 07:47:03 +0000 Subject: [PATCH 14/27] chore: move the c-ares-async pin to master, which only has its wording changed Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 35d1428..2c3f299 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,7 +48,7 @@ FetchContent_Declare(anyhttp SYSTEM) FetchContent_Declare(c_ares_async GIT_REPOSITORY https://github.com/pgit/c-ares-async.git - GIT_TAG 2fb55f687f210d3da816e24aab9cefb61d6b4673 + GIT_TAG 6d46c6c8e65231f23a250bcec5dd00b3a1d9ba34 SYSTEM) FetchContent_MakeAvailable(anyhttp c_ares_async) From 3297e93b40c2d9af6c60d27a6f53a6d974611ab9 Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Tue, 3 Nov 2026 12:17:44 +0000 Subject: [PATCH 15/27] chore: move the anyhttp pin to h2-no-server-push, where a pushed response no longer crashes the client Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c3f299..b85eec4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,7 +44,7 @@ set(C_ARES_ASYNC_TLS ${TLS_LIBRARY}) include(FetchContent) FetchContent_Declare(anyhttp GIT_REPOSITORY https://github.com/pgit/anyhttp.git - GIT_TAG 9c7627d78cc350376e6f2e6872b251d854a565d8 + GIT_TAG 649443c8883cb74ecaf306db0efb77c64c6ed606 SYSTEM) FetchContent_Declare(c_ares_async GIT_REPOSITORY https://github.com/pgit/c-ares-async.git From a581f452ba3229eda8f2d6c752c106b1ce9fae85 Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Tue, 3 Nov 2026 12:19:30 +0000 Subject: [PATCH 16/27] fix: take the address from c-ares, and say which one was connected to The address came from anyhttp's own resolver, so --dns-server did not apply to it and nothing could tell which one had been contacted. Now c-ares looks it up and each attempt gets one address through client::Config::endpoints. The output has the address that answered, or the last one tried on a failure. Addresses are tried in the order c-ares sorts them, within the timeout of the route, until one can be connected to. Once a request has gone out, that ends it for the host, on this address as on any other route. A host without an address now counts as a failed lookup and is not connected to. Address hints get their colour per address, leaving the commas without. Co-Authored-By: Claude Opus 5.5 --- README.md | 13 ++++--- src/output.cpp | 38 +++++++++++++++---- src/prober.cpp | 84 +++++++++++++++++++++++++++--------------- src/prober.hpp | 14 +++++-- test/output_test.cpp | 37 ++++++++++++++++++- test/prober_test.cpp | 87 ++++++++++++++++++++++++++++++++++++++++++++ test/site.hpp | 4 +- 7 files changed, 228 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 323fde8..de73aa7 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,9 @@ http-probe --dns-server 1.1.1.1 --file hostnames.txt It looks up the HTTPS records of each host in DNS ([RFC 9460](https://www.rfc-editor.org/rfc/rfc9460)), and requests `https:///` once, over TLS, in the way that the records say. It prints the status, -the route, the time it took and the header fields of the response, along with the host's records. A -redirect is reported and not followed, and nothing else on the host is requested. +the route, the IP address it connected to, the time it took and the header fields of the response, +along with the host's records. A redirect is reported and not followed, and nothing else on the host +is requested. There is output for every host but those that do not exist, which `--all` adds. The counts at the end look like this, on stderr: @@ -34,7 +35,7 @@ look like this, on stderr: | has records for other targets only, or an alias to another name | out of reach, not connected to: anyhttp can't connect to one name and verify another | | has a record that says the service is not available | out of reach, not connected to | | does not exist | not found, not connected to | -| gets no answer from DNS | lookup failed, so there is no telling, not connected to | +| gets no answer from DNS, or has no address | lookup failed, so there is no telling, not connected to | Aliases are followed, a record has to be compatible, and the lowest priority goes first. Within a record, HTTP/3 goes first, and then TCP on the same port: HTTP/2 if the record has `h2`, otherwise @@ -44,8 +45,10 @@ what the server went for, which may be HTTP/1.1 where HTTP/2 was offered. **One request means one.** The next route is for when connecting failed. Once a request has gone out, the host has been asked, so that a timeout or a response cut short ends it there, as a failure. -The address comes from anyhttp's own resolver and not from c-ares, so `--dns-server` and the address -hints in the record do not apply to it. Certificates are always verified, against the host's name. +The address comes from c-ares, so `--dns-server` applies to it, and `/etc/hosts` as well. The address +hints in a record do not. Addresses are tried one at a time, in the order c-ares sorts them, within +the timeout of the route, until one of them can be connected to. A failure says which address it was +on. Certificates are always verified, against the host's name. **To see what is going on,** `-v` logs to stderr: the requests and responses from anyhttp, the queries from c-ares and the steps in between. `-vv` adds all of QUIC. Hosts run at the same time mix in the diff --git a/src/output.cpp b/src/output.cpp index 83366dd..ddc595a 100644 --- a/src/output.cpp +++ b/src/output.cpp @@ -1,6 +1,7 @@ #include "output.hpp" #include +#include #include namespace http_probe @@ -36,6 +37,27 @@ std::string status_text(int status, bool color) return colored(std::to_string(status), code, color); } +std::string address_text(std::string_view address, bool color) +{ + const bool v6 = address.find(':') != address.npos; + return colored(address, v6 ? color_of::v6 : color_of::v4, color); +} + +// +// The value of an address hint: the addresses have a colour, the commas between them do not. +// +std::string address_list_text(std::string_view list, bool color) +{ + std::string text; + for (const auto address : std::views::split(list, ',')) + { + if (!text.empty()) + text += ','; + text += address_text(std::string_view(address), color); + } + return text; +} + // // to_text() with colours: the target, and the values of the address hints. // @@ -48,12 +70,11 @@ std::string record_text(const c_ares_async::https_record& record, bool color) for (std::string_view param : record.params) { - const bool v4 = param.starts_with("ipv4hint="); - if (v4 || param.starts_with("ipv6hint=")) + if (param.starts_with("ipv4hint=") || param.starts_with("ipv6hint=")) { const auto value = param.find('=') + 1; text += std::format(" {}{}", param.substr(0, value), - colored(param.substr(value), v4 ? color_of::v4 : color_of::v6, color)); + address_list_text(param.substr(value), color)); } else text += std::format(" {}", param); @@ -74,12 +95,15 @@ std::string format_result(const Result& result, bool color) switch (result.outcome) { case Outcome::answered: - text = std::format("{}: {} [{}] {} bytes in {}\n", host, status_text(result.status, color), - result.route, result.body_size, result.duration); + text = std::format("{}: {} [{}] at {}, {} bytes in {}\n", host, + status_text(result.status, color), result.route, + address_text(result.address, color), result.body_size, result.duration); break; case Outcome::failed: - text = - std::format("{}: {}: {}\n", host, colored("failed", color_of::bad, color), result.error); + text = std::format( + "{}: {}{}: {}\n", host, colored("failed", color_of::bad, color), + result.address.empty() ? "" : std::format(" at {}", address_text(result.address, color)), + result.error); break; case Outcome::out_of_reach: text = std::format("{}: {}\n", host, colored("out of reach", color_of::moved, color)); diff --git a/src/prober.cpp b/src/prober.cpp index ad7f0bc..0356bb1 100644 --- a/src/prober.cpp +++ b/src/prober.cpp @@ -235,52 +235,78 @@ anyhttp::Task Prober::probe(const std::string& host) co_return result; } + // + // Routes stay on the host's own name, see plan_routes(), so that this is good for all of them. + // Outside of any timeout, as with the records. Not having an address says nothing about the + // host, and no record says that it has to have one. + // + spdlog::debug("{}: asking for the addresses of '{}'", host, host); + const auto [lookup_ec, addresses] = co_await resolver_.resolve(host, ""); + if (lookup_ec || addresses.empty()) + { + result.outcome = Result::Outcome::lookup_failed; + result.notes.push_back(std::format("address lookup of '{}': {}", host, + lookup_ec ? lookup_ec.message() : "no addresses")); + co_return result; + } + result.outcome = Result::Outcome::failed; for (const auto& route : plan.routes) { - spdlog::debug("{}: connecting by {}, for {}", host, route.to_string(), route.source); - auto response = co_await attempt(route, host); - if (response.ec) + const auto deadline = std::chrono::steady_clock::now() + config_.timeout; + for (auto endpoint : addresses) { - result.error = std::format("[{}] {}", route.to_string(), response.error); - result.notes.push_back(result.error); - spdlog::debug("{}: {}", host, result.error); - if (response.request_sent) - break; // the host may have seen it, and another route would make it two - continue; - } + endpoint.port(route.port); + result.address = endpoint.address().to_string(); + spdlog::debug("{}: connecting to {} by {}, for {}", host, result.address, + route.to_string(), route.source); + auto response = co_await attempt(route, host, endpoint, deadline); + if (response.ec) + { + result.error = std::format("[{}] {}", route.to_string(), response.error); + result.notes.push_back(std::format("{} at {}", result.error, result.address)); + spdlog::debug("{}: {} at {}", host, result.error, result.address); + if (response.request_sent) + co_return result; // the host may have seen it, and anything else would make it two + if (std::chrono::steady_clock::now() >= deadline) + break; // the next address gets what is left of the timeout, which is nothing + continue; + } - result.outcome = Result::Outcome::answered; - result.error.clear(); - result.status = static_cast(response.message.result_int()); - auto spoken = route; - spoken.protocol = response.protocol; - result.route = spoken.to_string(); - for (const auto& field : response.message) - result.fields.emplace_back(field.name_string(), field.value()); - result.body_size = response.message.body().size(); - result.duration = response.duration; - break; + result.outcome = Result::Outcome::answered; + result.error.clear(); + result.status = static_cast(response.message.result_int()); + auto spoken = route; + spoken.protocol = response.protocol; + result.route = spoken.to_string(); + for (const auto& field : response.message) + result.fields.emplace_back(field.name_string(), field.value()); + result.body_size = response.message.body().size(); + result.duration = response.duration; + co_return result; + } } co_return result; } // -// Connects as the route says and asks for the host's root, within the timeout, and after that ends -// the connection in an orderly way, which has limits of its own and is none of the timeout's -// business: the response is there by then. +// Connects as the route says, to this one address so that there is no doubt about which it was, and +// asks for the host's root, by the deadline. After that it ends the connection in an orderly way, +// which has limits of its own and is none of the timeout's business: the response is there by then. // -anyhttp::Task Prober::attempt(const Route& route, const std::string& host) +anyhttp::Task Prober::attempt(const Route& route, const std::string& host, + const anyhttp::TcpEndpoint& endpoint, + std::chrono::steady_clock::time_point deadline) { anyhttp::client::Config client_config; client_config.protocol = route.protocol; client_config.tls_ca_file = config_.tls_ca_file; // - // The name of the route, as that is also what anyhttp verifies the certificate against, which - // leaves it to its own resolver to look up the address. The scheme is what makes it TLS over - // TCP, too. + // The name of the route is what anyhttp verifies the certificate against, wherever the + // connection goes. The scheme is what makes it TLS over TCP, too. // + client_config.endpoints = {endpoint}; client_config.url.set_scheme("https"); client_config.url.set_host_address(route.host); client_config.url.set_port_number(route.port); @@ -296,7 +322,7 @@ anyhttp::Task Prober::attempt(const Route& route, const std::s // auto result = co_await anyhttp::when_either( exchange(client, session, request_sent, boost::urls::url(std::format("https://{}/", host))), - anyhttp::delay(config_.timeout)); + anyhttp::delay(std::chrono::ceil(deadline - start))); auto* exchanged = std::get_if<0>(&result); if (!exchanged) diff --git a/src/prober.hpp b/src/prober.hpp index c0caad9..d641dc8 100644 --- a/src/prober.hpp +++ b/src/prober.hpp @@ -32,7 +32,8 @@ struct Config /// How many hosts to work on at the same time. std::size_t concurrency = 16; - /// How long the request may take on one route, from connecting to the end of the response. + /// How long the request may take on one route, from connecting to the end of the response, + /// including any addresses of the host that could not be connected to. std::chrono::milliseconds timeout{10000}; /// The DNS servers to ask, as resolver::set_servers() takes them. Empty for those of the system. @@ -79,13 +80,16 @@ struct Result std::size_t body_size = 0; ///< with \c answered std::chrono::milliseconds duration{}; ///< with \c answered: of the attempt that got through + /// The address that answered or, with \c failed, the last one contacted. Empty without one. + std::string address; std::string error; ///< with \c failed: what went wrong on the last route std::vector notes; ///< records that could not be used, routes that did not work }; /** - * Looks up the HTTPS records of each of Config::hosts with the resolver of c-ares-async, and sends - * one request to each host that can be connected to (see plan_routes()). A response is what comes + * Looks up the HTTPS records and the addresses of each of Config::hosts with the resolver of + * c-ares-async, and sends one request to each host that can be connected to (see plan_routes()), + * trying its addresses in the order they came in, on each route. A response is what comes * back, whatever its status: a redirect is reported and not followed, nothing in the body is looked * at, and nothing else on the host is requested. The next route is for a connection that did not * come about: once a request has gone out, that was it for this host, however it ended. @@ -118,7 +122,9 @@ class Prober anyhttp::Task work(); anyhttp::Task resolve_service(const std::string& host); anyhttp::Task probe(const std::string& host); - anyhttp::Task attempt(const Route& route, const std::string& host); + anyhttp::Task attempt(const Route& route, const std::string& host, + const anyhttp::TcpEndpoint& endpoint, + std::chrono::steady_clock::time_point deadline); anyhttp::Task exchange(anyhttp::client::Client& client, anyhttp::Session& session, bool& request_sent, boost::urls::url url); diff --git a/test/output_test.cpp b/test/output_test.cpp index e7325fd..eba10ba 100644 --- a/test/output_test.cpp +++ b/test/output_test.cpp @@ -26,7 +26,8 @@ Result answered(int status = 200) .route = "h3 example.com:443", .fields = {{"server", "test"}}, .body_size = 5, - .duration = 12ms}; + .duration = 12ms, + .address = "192.0.2.1"}; } } // namespace @@ -34,11 +35,30 @@ Result answered(int status = 200) TEST(Output, SaysWhatCameBack) { EXPECT_EQ(format_result(answered(), false), - "example.com: 200 [h3 example.com:443] 5 bytes in 12ms\n" + "example.com: 200 [h3 example.com:443] at 192.0.2.1, 5 bytes in 12ms\n" " record: HTTPS 1 . alpn=h3 ipv4hint=192.0.2.1 ipv6hint=2001:db8::1\n" " server: test\n"); } +TEST(Output, SaysWhichAddressFailed) +{ + EXPECT_EQ(format_result({.host = "example.com", + .outcome = Outcome::failed, + .address = "2001:db8::1", + .error = "[h3 example.com:443] timed out"}, + false), + "example.com: failed at 2001:db8::1: [h3 example.com:443] timed out\n"); +} + +TEST(Output, ColoursAnAddressByItsFamily) +{ + auto result = answered(); + EXPECT_THAT(format_result(result, true), HasSubstr("at \x1b[35m192.0.2.1\x1b[0m,")); + + result.address = "2001:db8::1"; + EXPECT_THAT(format_result(result, true), HasSubstr("at \x1b[34m2001:db8::1\x1b[0m,")); +} + TEST(Output, SaysWhyThereIsNoResponse) { EXPECT_EQ(format_result({.host = "example.com", @@ -64,6 +84,19 @@ TEST(Output, ColoursTheWayCAresAsyncDoes) EXPECT_THAT(text, HasSubstr("ipv6hint=\x1b[34m2001:db8::1\x1b[0m")) << "IPv6 is blue"; } +TEST(Output, LeavesTheCommasOfAnAddressListWithoutColour) +{ + auto result = answered(); + result.records[0].params = {"ipv4hint=192.0.2.1,192.0.2.2", "ipv6hint=2001:db8::1,2001:db8::2"}; + + const auto text = format_result(result, true); + EXPECT_THAT(text, HasSubstr("ipv4hint=\x1b[35m192.0.2.1\x1b[0m,\x1b[35m192.0.2.2\x1b[0m")); + EXPECT_THAT(text, HasSubstr("ipv6hint=\x1b[34m2001:db8::1\x1b[0m,\x1b[34m2001:db8::2\x1b[0m")); + + EXPECT_THAT(format_result(result, false), + HasSubstr("ipv4hint=192.0.2.1,192.0.2.2 ipv6hint=2001:db8::1,2001:db8::2")); +} + TEST(Output, ColoursATargetAsAHostName) { auto result = answered(); diff --git a/test/prober_test.cpp b/test/prober_test.cpp index 9de9d80..49629bb 100644 --- a/test/prober_test.cpp +++ b/test/prober_test.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -536,6 +537,92 @@ TEST_F(Probe, DoesNotConnectToATargetOtherThanTheHost) EXPECT_THAT(site.requests(), IsEmpty()); } +TEST_F(Probe, SaysWhichAddressAnswered) +{ + site.dns().add_https("localhost", 1, ".", {alpn_h3, port_param(site.port())}); + site.dns().add_https("localhost", 2, ".", h2_at(site.port())); + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + ASSERT_EQ(results[0].outcome, Result::Outcome::answered) << notes(results[0]); + EXPECT_TRUE(boost::asio::ip::make_address(results[0].address).is_loopback()) + << results[0].address; +} + +TEST_F(Probe, SaysWhichAddressAnsweredOverTcp) +{ + site.dns().add_https("localhost", 1, ".", h2_at(site.port())); + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + ASSERT_EQ(results[0].outcome, Result::Outcome::answered) << notes(results[0]); + EXPECT_TRUE(boost::asio::ip::make_address(results[0].address).is_loopback()) + << results[0].address; +} + +// +// No other resolver knows this name, so that getting as far as the certificate, which is not for +// this name, is what it takes to know that the addresses came from the DNS server. +// +TEST_F(Probe, ConnectsToTheAddressInDnsAndSaysWhichOne) +{ + site.dns().add_https("site.example.test", 1, ".", h2_at(site.port())); + site.dns().add_address("site.example.test", "127.0.0.1"); + + probe(config({"site.example.test"})); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::failed); + EXPECT_EQ(results[0].address, "127.0.0.1"); + EXPECT_THAT(results[0].error, HasSubstr("connect")); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(Probe, ConnectsToAnIPv6AddressInDns) +{ + site.dns().add_https("site.example.test", 1, ".", h2_at(site.port())); + site.dns().add_address("site.example.test", "::1"); + + probe(config({"site.example.test"})); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::failed); + EXPECT_EQ(results[0].address, "::1"); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(Probe, TriesTheNextAddressWhenAConnectionFails) +{ + site.dns().add_https("site.example.test", 1, ".", h2_at(site.port())); + site.dns().add_address("site.example.test", "127.0.0.1"); + site.dns().add_address("site.example.test", "::1"); + + probe(config({"site.example.test"})); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::failed); + EXPECT_THAT(notes(results[0]), HasSubstr("at 127.0.0.1")); + EXPECT_THAT(notes(results[0]), HasSubstr("at ::1")); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(Probe, DoesNotConnectWithoutAnAddress) +{ + site.dns().add_https("site.example.test", 1, ".", h2_at(site.port())); + + probe(config({"site.example.test"})); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::lookup_failed); + EXPECT_THAT(results[0].address, IsEmpty()); + EXPECT_THAT(notes(results[0]), HasSubstr("address lookup")); + EXPECT_THAT(site.requests(), IsEmpty()); +} + TEST_F(Probe, RefusesACertificateItDoesNotTrust) { Site other(context.get_executor()); // has a certificate of its own diff --git a/test/site.hpp b/test/site.hpp index 281f0a9..92f2c3d 100644 --- a/test/site.hpp +++ b/test/site.hpp @@ -1,8 +1,8 @@ // // A web site with DNS of its own, all on loopback, for the program to be pointed at: an anyhttp // server on one port, for every protocol, and a DNS server with whatever zone the test wants. The -// server has a certificate for "localhost", the one name that also resolves without our DNS server, -// which anyhttp's own resolver needs. +// server has a certificate for "localhost". That name comes from /etc/hosts and not from the DNS +// server, which a test can only have the last word on with a name that the certificate is not for. // #pragma once From 5196b85404a170c92f8b23f62ec0ffaa7a16f4c9 Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Tue, 3 Nov 2026 17:09:30 +0000 Subject: [PATCH 17/27] fix: try the IPv6 addresses of a host first c-ares goes by RFC 6724, which puts IPv4 first wherever the source address is not a global one, so IPv6 did not get a turn on such a machine. The addresses are still tried one at a time and not with Happy Eyeballs, so an IPv6 address that does not answer at all takes the whole timeout of the route. Co-Authored-By: Claude Opus 5.5 --- README.md | 7 ++++--- src/prober.cpp | 14 +++++++++++++- src/prober.hpp | 11 +++++++---- test/prober_test.cpp | 45 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index de73aa7..b723517 100644 --- a/README.md +++ b/README.md @@ -46,9 +46,10 @@ what the server went for, which may be HTTP/1.1 where HTTP/2 was offered. the host has been asked, so that a timeout or a response cut short ends it there, as a failure. The address comes from c-ares, so `--dns-server` applies to it, and `/etc/hosts` as well. The address -hints in a record do not. Addresses are tried one at a time, in the order c-ares sorts them, within -the timeout of the route, until one of them can be connected to. A failure says which address it was -on. Certificates are always verified, against the host's name. +hints in a record do not. Addresses are tried one at a time, IPv6 first and otherwise in the order +c-ares sorts them, within the timeout of the route, until one of them can be connected to. So an IPv6 +address that does not answer at all takes the whole timeout of the route, and IPv4 does not get a turn. +A failure says which address it was on. Certificates are always verified, against the host's name. **To see what is going on,** `-v` logs to stderr: the requests and responses from anyhttp, the queries from c-ares and the steps in between. `-vv` adds all of QUIC. Hosts run at the same time mix in the diff --git a/src/prober.cpp b/src/prober.cpp index 0356bb1..0820c8c 100644 --- a/src/prober.cpp +++ b/src/prober.cpp @@ -95,6 +95,12 @@ std::vector read_hosts(std::istream& list, std::vector return hosts; } +void prefer_ipv6(std::vector& addresses) +{ + std::ranges::stable_partition(addresses, + [](const auto& endpoint) { return endpoint.address().is_v6(); }); +} + Prober::Prober(anyhttp::Executor executor, Config config, Observer observer) : executor_(executor), config_(std::move(config)), observer_(std::move(observer)), resolver_(std::move(executor)) @@ -241,7 +247,7 @@ anyhttp::Task Prober::probe(const std::string& host) // host, and no record says that it has to have one. // spdlog::debug("{}: asking for the addresses of '{}'", host, host); - const auto [lookup_ec, addresses] = co_await resolver_.resolve(host, ""); + auto [lookup_ec, addresses] = co_await resolver_.resolve(host, ""); if (lookup_ec || addresses.empty()) { result.outcome = Result::Outcome::lookup_failed; @@ -250,6 +256,12 @@ anyhttp::Task Prober::probe(const std::string& host) co_return result; } + // + // One after the other, and not at the same time as with Happy Eyeballs. c-ares goes by RFC 6724, + // which puts IPv4 first wherever the source address is not a global one. + // + prefer_ipv6(addresses); + result.outcome = Result::Outcome::failed; for (const auto& route : plan.routes) { diff --git a/src/prober.hpp b/src/prober.hpp index d641dc8..e80b82b 100644 --- a/src/prober.hpp +++ b/src/prober.hpp @@ -57,6 +57,9 @@ bool is_host_name(std::string_view host); */ std::vector read_hosts(std::istream& list, std::vector& skipped); +/// Moves the IPv6 addresses to the front, each family keeping the order it was in. +void prefer_ipv6(std::vector& addresses); + /// What came of one host. struct Result { @@ -89,10 +92,10 @@ struct Result /** * Looks up the HTTPS records and the addresses of each of Config::hosts with the resolver of * c-ares-async, and sends one request to each host that can be connected to (see plan_routes()), - * trying its addresses in the order they came in, on each route. A response is what comes - * back, whatever its status: a redirect is reported and not followed, nothing in the body is looked - * at, and nothing else on the host is requested. The next route is for a connection that did not - * come about: once a request has gone out, that was it for this host, however it ended. + * trying its addresses one at a time on each route, IPv6 first. A response is what comes back, + * whatever its status: a redirect is reported and not followed, nothing in the body is looked at, + * and nothing else on the host is requested. The next route is for a connection that did not come + * about: once a request has gone out, that was it for this host, however it ended. * * Hosts are worked on Config::concurrency at a time, so results arrive as they are ready. All of * this is on the one executor, which is also what calls the observer. diff --git a/test/prober_test.cpp b/test/prober_test.cpp index 49629bb..2456f2c 100644 --- a/test/prober_test.cpp +++ b/test/prober_test.cpp @@ -610,6 +610,51 @@ TEST_F(Probe, TriesTheNextAddressWhenAConnectionFails) EXPECT_THAT(site.requests(), IsEmpty()); } +// +// This one would hold without prefer_ipv6(), as c-ares puts ::1 ahead of 127.0.0.1 by itself. An +// address that it would put behind IPv4 is not on loopback, so the order is tested on its own +// below. +// +TEST_F(Probe, TriesIPv6First) +{ + site.dns().add_https("site.example.test", 1, ".", h2_at(site.port())); + site.dns().add_address("site.example.test", "127.0.0.1"); + site.dns().add_address("site.example.test", "::1"); + + probe(config({"site.example.test"})); + + ASSERT_EQ(results.size(), 1u); + const auto text = notes(results[0]); + ASSERT_THAT(text, HasSubstr("at ::1")); + ASSERT_THAT(text, HasSubstr("at 127.0.0.1")); + EXPECT_LT(text.find("at ::1"), text.find("at 127.0.0.1")); + EXPECT_EQ(results[0].address, "127.0.0.1"); // the last one tried +} + +anyhttp::TcpEndpoint at(const char* address) +{ + return {boost::asio::ip::make_address(address), 443}; +} + +TEST(PreferIpv6, MovesIPv6ToTheFrontAndKeepsTheOrderWithinEachFamily) +{ + std::vector addresses{at("192.0.2.1"), at("2001:db8::1"), at("192.0.2.2"), at("2001:db8::2")}; + + prefer_ipv6(addresses); + + EXPECT_THAT(addresses, + ElementsAre(at("2001:db8::1"), at("2001:db8::2"), at("192.0.2.1"), at("192.0.2.2"))); +} + +TEST(PreferIpv6, LeavesASingleFamilyAlone) +{ + std::vector addresses{at("192.0.2.2"), at("192.0.2.1")}; + + prefer_ipv6(addresses); + + EXPECT_THAT(addresses, ElementsAre(at("192.0.2.2"), at("192.0.2.1"))); +} + TEST_F(Probe, DoesNotConnectWithoutAnAddress) { site.dns().add_https("site.example.test", 1, ".", h2_at(site.port())); From 27cf807633012a0fa77f091727380b897552f4be Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Tue, 3 Nov 2026 19:19:50 +0000 Subject: [PATCH 18/27] refactor: run on Capy and Corosio only The libraries take either runtime and need a layer for that. This is a program, so it decides on one and has no layer: no switches, one preset, and capy::task, capy::any_executor, corosio::endpoint and std::error_code directly instead of the aliases of anyhttp. CMakeLists.txt selects the runtime for both libraries and declares the pins of Capy and Corosio, the same commits as theirs. when_all() and corosio::timeout() replace the parallel group and when_either(). A failure now comes back as capy::results, so Response has no error code of its own. If a worker throws, when_all() stops the others, where the parallel group let them finish. Asio stays in the tests, for the DNS server of c-ares-async, which is on Asio whatever the runtime. Site polls its context from a coroutine, which keeps everything on one thread. The other use is anyhttp's write_eof(), which takes nothing but an Asio buffer. What a run_async() handler throws ends the process, so main() and the fixture hold on to it until run() returns. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 24 ++++++++- README.md | 2 +- src/main.cpp | 40 +++++++++------ src/prober.cpp | 109 ++++++++++++++++++--------------------- src/prober.hpp | 36 ++++++++----- test/demo_site.cpp | 23 +++++++-- test/prober_test.cpp | 120 +++++++++++++++++++++++++++++-------------- test/site.cpp | 110 ++++++++++++++++++++++++++------------- test/site.hpp | 16 +++++- 9 files changed, 306 insertions(+), 174 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b85eec4..9960b99 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,8 +24,12 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(Boost_USE_STATIC_LIBS ON) endif() -# I/O runtime: ASIO or COROSIO. COROSIO would have to declare the capy and corosio pins here, too. -set(ANYHTTP_API ASIO) +# +# I/O runtime: Capy and Corosio, and only those. The libraries take either, which is their business: +# this is a program, and it gets to decide. These shadow the cache entries of the libraries, which +# go on saying ASIO in CMakeCache.txt. +# +set(ANYHTTP_API COROSIO) set(C_ARES_ASYNC_API ${ANYHTTP_API}) # @@ -50,6 +54,22 @@ FetchContent_Declare(c_ares_async GIT_REPOSITORY https://github.com/pgit/c-ares-async.git GIT_TAG 6d46c6c8e65231f23a250bcec5dd00b3a1d9ba34 SYSTEM) + +# +# Both libraries pin these as well, to the same commits for now. The first declaration wins, so these +# are the ones both of them get, and here they are SYSTEM, too. +# +FetchContent_Declare(capy + GIT_REPOSITORY https://github.com/cppalliance/capy.git + GIT_TAG 24c54af2cf967ccbf8d3809af4bcd9c8f61f553b + EXCLUDE_FROM_ALL + SYSTEM) +FetchContent_Declare(corosio + GIT_REPOSITORY https://github.com/cppalliance/corosio.git + GIT_TAG 41d0de18b6e4fa1b397c38a866f2daf52c6c36b5 + EXCLUDE_FROM_ALL + SYSTEM) + FetchContent_MakeAvailable(anyhttp c_ares_async) # Only for the command line. Neither library needs this component, so neither asks for it here. diff --git a/README.md b/README.md index b723517..f212ac2 100644 --- a/README.md +++ b/README.md @@ -85,4 +85,4 @@ cmake --build --preset pinned ctest --preset pinned ``` -The tree is `build/`, with clang, libc++, OpenSSL and the Asio runtime. +The tree is `build/`, with clang, libc++, OpenSSL and Capy and Corosio as the runtime. diff --git a/src/main.cpp b/src/main.cpp index f6a02d4..c4b3925 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4,8 +4,8 @@ #include "output.hpp" #include "prober.hpp" -#include -#include +#include +#include #include #include @@ -207,22 +207,30 @@ int main(int argc, char* argv[]) } }; - boost::asio::io_context context; + boost::corosio::io_context context; http_probe::Prober prober(context.get_executor(), std::move(config), report); - boost::asio::co_spawn(context, prober.run(), [&](const std::exception_ptr& ex) { - if (ex) - std::rethrow_exception(ex); - - // - // Here and not after run(): a host that did not answer can keep the context from running - // dry for a while, see the README, and this is what the wait would be for. - // - std::println(stderr, "{}", http_probe::format_summary(counts, color_summary)); - if (with_records == 0 && !own_dns_server) - std::println(stderr, "http-probe: none of the hosts has an HTTPS record. Not every " - "resolver answers for them, so try --dns-server."); - }); + std::exception_ptr failed; + + // + // What a handler throws ends the process, so it stays in here until the context is through. + // Printing here and not after run(): a host that did not answer can keep the context from + // running dry for a while, see the README, and this is what the wait would be for. + // + boost::capy::run_async( + context.get_executor(), + [&]() noexcept { + std::println(stderr, "{}", http_probe::format_summary(counts, color_summary)); + if (with_records == 0 && !own_dns_server) + std::println(stderr, "http-probe: none of the hosts has an HTTPS record. Not every " + "resolver answers for them, so try --dns-server."); + }, + [&](std::exception_ptr ex) noexcept { + failed = ex; + context.stop(); + })(prober.run()); context.run(); + if (failed) + std::rethrow_exception(failed); } catch (const std::invalid_argument& ex) { diff --git a/src/prober.cpp b/src/prober.cpp index 0820c8c..12935c7 100644 --- a/src/prober.cpp +++ b/src/prober.cpp @@ -2,11 +2,11 @@ #include -#include -#include -#include -#include #include +#include +#include +#include +#include #include #include @@ -18,6 +18,7 @@ #include #include #include +#include #include namespace http = boost::beast::http; @@ -63,8 +64,7 @@ bool is_host_name(std::string_view host) const auto is_name_char = [](unsigned char c) { return std::isalnum(c) != 0 || c == '-' || c == '.' || c == '_'; }; - boost::system::error_code not_an_address; - boost::asio::ip::make_address(host, not_an_address); + const auto [not_an_address, address] = corosio::make_ip_address(host); return !host.empty() && host.size() <= 253 && std::ranges::all_of(host, is_name_char) && not_an_address; } @@ -95,13 +95,12 @@ std::vector read_hosts(std::istream& list, std::vector return hosts; } -void prefer_ipv6(std::vector& addresses) +void prefer_ipv6(std::vector& addresses) { - std::ranges::stable_partition(addresses, - [](const auto& endpoint) { return endpoint.address().is_v6(); }); + std::ranges::stable_partition(addresses, &corosio::endpoint::is_v6); } -Prober::Prober(anyhttp::Executor executor, Config config, Observer observer) +Prober::Prober(capy::any_executor executor, Config config, Observer observer) : executor_(executor), config_(std::move(config)), observer_(std::move(observer)), resolver_(std::move(executor)) { @@ -127,30 +126,23 @@ Prober::Prober(anyhttp::Executor executor, Config config, Observer observer) std::format("invalid DNS server '{}': {}", config_.dns_server, ec.message())); } -anyhttp::Task Prober::run() +capy::task<> Prober::run() { - namespace asio = boost::asio; - using Worker = decltype(asio::co_spawn(executor_, work(), asio::deferred)); - - std::vector workers; + std::vector>> workers; for (std::size_t i = 0; i < std::min(config_.concurrency, hosts_.size()); ++i) - workers.push_back(asio::co_spawn(executor_, work(), asio::deferred)); - - auto [order, exceptions] = co_await asio::experimental::make_parallel_group(std::move(workers)) - .async_wait(asio::experimental::wait_for_all(), asio::deferred); + workers.push_back(work()); - for (const auto& exception : exceptions) - if (exception) - std::rethrow_exception(exception); + std::ignore = co_await capy::when_all(std::move(workers)); } -anyhttp::Task Prober::work() +capy::task> Prober::work() { while (next_host_ < hosts_.size()) { const auto& host = hosts_[next_host_++]; observer_(co_await probe(host)); } + co_return capy::results<>{}; } // ------------------------------------------------------------------------------------------------- @@ -160,7 +152,7 @@ anyhttp::Task Prober::work() // that fails is not fatal: DNS is not protected here, and this is the choice section 3.1 leaves // for that. A name without a record is the common case and not even worth a note. // -anyhttp::Task Prober::resolve_service(const std::string& host) +capy::task Prober::resolve_service(const std::string& host) { ServiceRecords found{.owner = host}; std::set visited{found.owner}; @@ -220,7 +212,7 @@ anyhttp::Task Prober::resolve_service(const std::string& host) .owner = host, .notes = {std::format("alias chain of '{}' is too long or has a loop", host)}}; } -anyhttp::Task Prober::probe(const std::string& host) +capy::task Prober::probe(const std::string& host) { Result result{.host = host}; @@ -266,14 +258,14 @@ anyhttp::Task Prober::probe(const std::string& host) for (const auto& route : plan.routes) { const auto deadline = std::chrono::steady_clock::now() + config_.timeout; - for (auto endpoint : addresses) + for (const auto& address : addresses) { - endpoint.port(route.port); + const corosio::endpoint endpoint(address, route.port); result.address = endpoint.address().to_string(); spdlog::debug("{}: connecting to {} by {}, for {}", host, result.address, route.to_string(), route.source); - auto response = co_await attempt(route, host, endpoint, deadline); - if (response.ec) + auto [ec, response] = co_await attempt(route, host, endpoint, deadline); + if (ec) { result.error = std::format("[{}] {}", route.to_string(), response.error); result.notes.push_back(std::format("{} at {}", result.error, result.address)); @@ -306,9 +298,9 @@ anyhttp::Task Prober::probe(const std::string& host) // asks for the host's root, by the deadline. After that it ends the connection in an orderly way, // which has limits of its own and is none of the timeout's business: the response is there by then. // -anyhttp::Task Prober::attempt(const Route& route, const std::string& host, - const anyhttp::TcpEndpoint& endpoint, - std::chrono::steady_clock::time_point deadline) +capy::task> +Prober::attempt(const Route& route, const std::string& host, const corosio::endpoint& endpoint, + std::chrono::steady_clock::time_point deadline) { anyhttp::client::Config client_config; client_config.protocol = route.protocol; @@ -329,45 +321,45 @@ anyhttp::Task Prober::attempt(const Route& route, const std::s bool request_sent = false; // - // These stay around for as long as exchange() needs them: when_either() waits for what it has + // These stay around for as long as exchange() needs them: timeout() waits for what it has // cancelled to finish. // - auto result = co_await anyhttp::when_either( + auto [ec, response] = co_await corosio::timeout( exchange(client, session, request_sent, boost::urls::url(std::format("https://{}/", host))), - anyhttp::delay(std::chrono::ceil(deadline - start))); - - auto* exchanged = std::get_if<0>(&result); - if (!exchanged) - co_return Response{.ec = make_error_code(boost::system::errc::timed_out), - .error = std::format("timed out after {}", config_.timeout), - .request_sent = request_sent}; + deadline); - auto response = std::move(*exchanged); response.request_sent = request_sent; + if (ec == capy::cond::timeout) + { + response.error = std::format("timed out after {}", config_.timeout); + co_return {ec, std::move(response)}; + } + response.duration = std::chrono::duration_cast( std::chrono::steady_clock::now() - start); - if (!response.ec) - if (auto [ec] = co_await session.shutdown(); ec) - spdlog::debug("{}: shutdown: {}", host, ec.message()); + if (!ec) + if (auto [shutdown_ec] = co_await session.shutdown(); shutdown_ec) + spdlog::debug("{}: shutdown: {}", host, shutdown_ec.message()); - co_return response; + co_return {ec, std::move(response)}; } // -// Everything is reported rather than thrown: when_either() would otherwise sit out the timeout, as -// what throws does not count as having finished first. +// Everything is reported rather than thrown: one host that makes anyhttp throw does not get to end +// the run for the others. // -anyhttp::Task Prober::exchange(anyhttp::client::Client& client, - anyhttp::Session& session, bool& request_sent, - boost::urls::url url) +capy::task> Prober::exchange(anyhttp::client::Client& client, + anyhttp::Session& session, + bool& request_sent, + boost::urls::url url) { try { auto [connect_ec, connected] = co_await client.connect(); if (connect_ec) - co_return Response{.ec = connect_ec, - .error = std::format("connect: {}", connect_ec.message())}; + co_return {connect_ec, + Response{.error = std::format("connect: {}", connect_ec.message())}}; session = std::move(connected); @@ -375,15 +367,14 @@ anyhttp::Task Prober::exchange(anyhttp::client::Client& client headers.set(http::field::user_agent, user_agent); request_sent = true; auto [get_ec, message] = co_await session.get(std::move(url), std::move(headers)); - co_return Response{.ec = get_ec, - .message = std::move(message), - .error = get_ec ? std::format("request: {}", get_ec.message()) : "", - .protocol = session.protocol()}; + co_return {get_ec, + Response{.message = std::move(message), + .error = get_ec ? std::format("request: {}", get_ec.message()) : "", + .protocol = session.protocol()}}; } catch (const std::exception& ex) { - co_return Response{.ec = make_error_code(boost::system::errc::connection_aborted), - .error = ex.what()}; + co_return {std::make_error_code(std::errc::connection_aborted), Response{.error = ex.what()}}; } } diff --git a/src/prober.hpp b/src/prober.hpp index e80b82b..0b44f49 100644 --- a/src/prober.hpp +++ b/src/prober.hpp @@ -9,6 +9,11 @@ #include +#include +#include +#include +#include + #include #include #include @@ -22,6 +27,9 @@ namespace http_probe { +namespace capy = boost::capy; +namespace corosio = boost::corosio; + // ================================================================================================= struct Config @@ -58,7 +66,7 @@ bool is_host_name(std::string_view host); std::vector read_hosts(std::istream& list, std::vector& skipped); /// Moves the IPv6 addresses to the front, each family keeping the order it was in. -void prefer_ipv6(std::vector& addresses); +void prefer_ipv6(std::vector& addresses); /// What came of one host. struct Result @@ -107,14 +115,13 @@ class Prober /// Throws \c std::invalid_argument for a configuration that is not good to go, and does nothing /// else until run(). - Prober(anyhttp::Executor executor, Config config, Observer observer); + Prober(capy::any_executor executor, Config config, Observer observer); - anyhttp::Task run(); + capy::task<> run(); private: struct Response { - anyhttp::error_code ec; anyhttp::client::Message message; std::string error; anyhttp::Protocol protocol = anyhttp::Protocol::h3; ///< what the server chose to speak @@ -122,16 +129,17 @@ class Prober std::chrono::milliseconds duration{}; }; - anyhttp::Task work(); - anyhttp::Task resolve_service(const std::string& host); - anyhttp::Task probe(const std::string& host); - anyhttp::Task attempt(const Route& route, const std::string& host, - const anyhttp::TcpEndpoint& endpoint, - std::chrono::steady_clock::time_point deadline); - anyhttp::Task exchange(anyhttp::client::Client& client, anyhttp::Session& session, - bool& request_sent, boost::urls::url url); - - anyhttp::Executor executor_; + capy::task> work(); + capy::task resolve_service(const std::string& host); + capy::task probe(const std::string& host); + capy::task> attempt(const Route& route, const std::string& host, + const corosio::endpoint& endpoint, + std::chrono::steady_clock::time_point deadline); + capy::task> exchange(anyhttp::client::Client& client, + anyhttp::Session& session, bool& request_sent, + boost::urls::url url); + + capy::any_executor executor_; Config config_; Observer observer_; std::vector hosts_; diff --git a/test/demo_site.cpp b/test/demo_site.cpp index de7533c..41980fa 100644 --- a/test/demo_site.cpp +++ b/test/demo_site.cpp @@ -4,18 +4,31 @@ // #include "site.hpp" -#include -#include +#include +#include +#include +#include #include +#include #include +#include + +namespace capy = boost::capy; +namespace corosio = boost::corosio; + +static capy::task<> stop_on_signal(corosio::signal_set& signals, http_probe::test::Site& site) +{ + std::ignore = co_await signals.wait(); + site.stop(); +} int main() { spdlog::set_level(spdlog::level::err); - boost::asio::io_context context; + corosio::io_context context; http_probe::test::Site site(context.get_executor()); site.dns().add_https("localhost", 1, ".", @@ -30,7 +43,7 @@ int main() site.ca_file().string()); std::fflush(stdout); - boost::asio::signal_set signals(context, SIGINT, SIGTERM); - signals.async_wait([&](auto, auto) { site.stop(); }); + corosio::signal_set signals(context, SIGINT, SIGTERM); + capy::run_async(context.get_executor())(stop_on_signal(signals, site)); context.run(); } diff --git a/test/prober_test.cpp b/test/prober_test.cpp index 2456f2c..1a4ed8c 100644 --- a/test/prober_test.cpp +++ b/test/prober_test.cpp @@ -5,11 +5,13 @@ #include "prober.hpp" #include "site.hpp" -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include #include @@ -21,9 +23,12 @@ #include #include #include +#include #include #include #include +#include +#include #include #include @@ -36,9 +41,28 @@ using testing::HasSubstr; using testing::IsEmpty; using testing::Pair; +namespace boost::corosio +{ + +/// What a failed expectation shows of an endpoint: Corosio has no output operator for one. +inline void PrintTo(const endpoint& endpoint, std::ostream* out) +{ + *out << endpoint.address() << ':' << endpoint.port(); +} + +} // namespace boost::corosio + namespace { +corosio::ip_address make_address(std::string_view text) +{ + auto [ec, address] = corosio::make_ip_address(text); + if (ec) + throw std::system_error(ec, std::string(text)); + return address; +} + fake_dns_server::svc_param port_param(std::uint16_t port) { return {ARES_SVCB_PARAM_PORT, @@ -81,24 +105,49 @@ class Probe : public testing::Test Prober prober(context.get_executor(), std::move(config), [this](const Result& result) { results.push_back(result); }); - boost::asio::co_spawn( - context, - [&]() -> anyhttp::Task { - co_await prober.run(); - site.stop(); - // - // For the tests with a QUIC endpoint that never answers. anyhttp does not close a - // session whose connect() was cancelled: it goes on sending until its idle timeout of - // 30 seconds, and the context does not run dry before that. To go once that is fixed. - // - if (abandon_quic_sessions) - context.stop(); - }, - [](const std::exception_ptr& ex) { - if (ex) - std::rethrow_exception(ex); - }); + // + // In a variable: the coroutine refers to what the lambda captured, for as long as it runs. + // + const auto run = [&]() -> capy::task<> { + co_await prober.run(); + site.stop(); + // + // For the tests with a QUIC endpoint that never answers. anyhttp does not close a + // session whose connect() was cancelled: it goes on sending until its idle timeout of + // 30 seconds, and the context does not run dry before that. To go once that is fixed. + // + if (abandon_quic_sessions) + context.stop(); + }; + + std::exception_ptr failed; + capy::run_async( + context.get_executor(), []() noexcept {}, + [&](std::exception_ptr ex) noexcept { + failed = ex; + context.stop(); + })(run()); context.run(); + if (failed) + std::rethrow_exception(failed); + } + + // + // A port where nothing is ever going to answer, on any address that "localhost" may be. + // + corosio::udp_socket silent_udp_socket() + { + corosio::udp_socket socket(context); + if (const auto ec = socket.open(corosio::family::v6)) + throw std::system_error(ec, "open"); + if (const auto ec = socket.bind({corosio::ipv6_address::any(), 0})) + throw std::system_error(ec, "bind"); + return socket; + } + + corosio::tcp_acceptor silent_tcp_acceptor() + { + return corosio::tcp_acceptor(context, {corosio::ipv6_address::any(), 0}); } // @@ -125,7 +174,7 @@ class Probe : public testing::Test } int quiet_ = quiet(); // ahead of the server, which already has something to say when it starts - boost::asio::io_context context; + corosio::io_context context; Site site{context.get_executor()}; std::vector results; bool abandon_quic_sessions = false; @@ -323,7 +372,7 @@ TEST_F(Probe, ReportsEachHostOnceHoweverManyThereAre) TEST_F(Probe, DoesNotLetASlowHostHoldUpTheOthers) { - boost::asio::ip::udp::socket silent(context, {boost::asio::ip::udp::v6(), 0}); + auto silent = silent_udp_socket(); abandon_quic_sessions = true; site.dns().add_https("localhost", 1, ".", h3_at(silent.local_endpoint().port())); @@ -339,7 +388,7 @@ TEST_F(Probe, DoesNotLetASlowHostHoldUpTheOthers) TEST_F(Probe, WorksOnOneHostAtATimeIfToldTo) { - boost::asio::ip::udp::socket silent(context, {boost::asio::ip::udp::v6(), 0}); + auto silent = silent_udp_socket(); abandon_quic_sessions = true; site.dns().add_https("localhost", 1, ".", h3_at(silent.local_endpoint().port())); @@ -449,7 +498,7 @@ TEST_F(Probe, FallsBackToTheNextRecordWhenAConnectionFails) // Nothing answers on this port. The socket is ours for the length of the test, so that nothing // else will either. // - boost::asio::ip::udp::socket silent(context, {boost::asio::ip::udp::v6(), 0}); + auto silent = silent_udp_socket(); abandon_quic_sessions = true; site.dns().add_https("localhost", 1, ".", h3_at(silent.local_endpoint().port())); @@ -468,7 +517,7 @@ TEST_F(Probe, FallsBackToTheNextRecordWhenAConnectionFails) TEST_F(Probe, FallsBackFromHttp3ToTcpWhenAConnectionFails) { - boost::asio::ip::udp::socket silent(context, {boost::asio::ip::udp::v6(), 0}); + auto silent = silent_udp_socket(); abandon_quic_sessions = true; site.dns().add_https("localhost", 1, ".", h3_at(silent.local_endpoint().port())); @@ -547,8 +596,7 @@ TEST_F(Probe, SaysWhichAddressAnswered) ASSERT_EQ(results.size(), 1u); ASSERT_EQ(results[0].outcome, Result::Outcome::answered) << notes(results[0]); - EXPECT_TRUE(boost::asio::ip::make_address(results[0].address).is_loopback()) - << results[0].address; + EXPECT_TRUE(make_address(results[0].address).is_loopback()) << results[0].address; } TEST_F(Probe, SaysWhichAddressAnsweredOverTcp) @@ -560,8 +608,7 @@ TEST_F(Probe, SaysWhichAddressAnsweredOverTcp) ASSERT_EQ(results.size(), 1u); ASSERT_EQ(results[0].outcome, Result::Outcome::answered) << notes(results[0]); - EXPECT_TRUE(boost::asio::ip::make_address(results[0].address).is_loopback()) - << results[0].address; + EXPECT_TRUE(make_address(results[0].address).is_loopback()) << results[0].address; } // @@ -631,10 +678,7 @@ TEST_F(Probe, TriesIPv6First) EXPECT_EQ(results[0].address, "127.0.0.1"); // the last one tried } -anyhttp::TcpEndpoint at(const char* address) -{ - return {boost::asio::ip::make_address(address), 443}; -} +corosio::endpoint at(const char* address) { return {make_address(address), 443}; } TEST(PreferIpv6, MovesIPv6ToTheFrontAndKeepsTheOrderWithinEachFamily) { @@ -800,7 +844,7 @@ TEST_F(Probe, LogsTheHttpAndTheLookupsAtDebugLevel) TEST_F(Probe, GivesUpOnAServerThatDoesNotAnswer) { - boost::asio::ip::udp::socket silent(context, {boost::asio::ip::udp::v6(), 0}); + auto silent = silent_udp_socket(); abandon_quic_sessions = true; site.dns().add_https("localhost", 1, ".", h3_at(silent.local_endpoint().port())); @@ -818,7 +862,7 @@ TEST_F(Probe, GivesUpOnATcpServerThatDoesNotAnswer) // // It takes connections, as the kernel does that for it, and leaves the handshake hanging. // - boost::asio::ip::tcp::acceptor silent(context, {boost::asio::ip::tcp::v6(), 0}); + auto silent = silent_tcp_acceptor(); site.dns().add_https("localhost", 1, ".", h2_at(silent.local_endpoint().port())); auto configuration = config(); @@ -834,7 +878,7 @@ TEST_F(Probe, ReportsAPortThatIsClosed) { std::uint16_t closed = 0; { - boost::asio::ip::tcp::acceptor acceptor(context, {boost::asio::ip::tcp::v6(), 0}); + auto acceptor = silent_tcp_acceptor(); closed = acceptor.local_endpoint().port(); } site.dns().add_https("localhost", 1, ".", h2_at(closed)); diff --git a/test/site.cpp b/test/site.cpp index 68513a9..94f5aa1 100644 --- a/test/site.cpp +++ b/test/site.cpp @@ -1,6 +1,11 @@ #include "site.hpp" +#include #include +#include +#include +#include +#include #include #include "routes.hpp" @@ -10,7 +15,11 @@ #include #include #include +#include +#include +namespace capy = boost::capy; +namespace corosio = boost::corosio; namespace http = boost::beast::http; namespace http_probe::test @@ -18,7 +27,7 @@ namespace http_probe::test // ================================================================================================= -Site::Site(const anyhttp::Executor& executor) : requests_(std::make_shared>()) +Site::Site(const capy::any_executor& executor) : requests_(std::make_shared>()) { auto directory = (std::filesystem::temp_directory_path() / "http-probe-test-XXXXXX").string(); if (!mkdtemp(directory.data())) @@ -33,37 +42,38 @@ Site::Site(const anyhttp::Executor& executor) : requests_(std::make_sharedlocal_endpoint().port(); - - server_->on_request([this, requests = requests_]( - anyhttp::server::Request request, - anyhttp::server::Response response) -> anyhttp::Task { - requests->push_back(Request{ - .target = std::string(request.url().encoded_target()), - .authority = std::string(request.url().encoded_host_and_port()), - .user_agent = std::string(request.fields()[http::field::user_agent]), - .time = std::chrono::steady_clock::now(), + capy::run_async(executor, polling_.get_token())(poll_dns()); + + server_->on_request( + [this, requests = requests_](anyhttp::server::Request request, + anyhttp::server::Response response) -> capy::task<> { + requests->push_back(Request{ + .target = std::string(request.url().encoded_target()), + .authority = std::string(request.url().encoded_host_and_port()), + .user_agent = std::string(request.fields()[http::field::user_agent]), + .time = std::chrono::steady_clock::now(), + }); + + std::array buffer; + for (;;) + if (auto [ec, n] = co_await request.read_some(capy::make_buffer(buffer)); ec || n == 0) + break; + + const auto it = pages_.find(std::string(request.url().encoded_path())); + const auto page = it != pages_.end() ? it->second : Page{.status = 404}; + + if (page.delay.count() > 0) + std::ignore = co_await corosio::delay(page.delay); + + anyhttp::Fields fields; + fields.set(http::field::content_type, page.content_type); + if (!page.location.empty()) + fields.set(http::field::location, page.location); + + if (auto [ec] = co_await response.submit(page.status, std::move(fields)); ec) + co_return; + co_await response.write_eof(boost::asio::buffer(page.body)); // has no overload for Capy's }); - - std::array buffer; - for (;;) - if (auto [ec, n] = co_await request.read_some(boost::asio::buffer(buffer)); ec || n == 0) - break; - - const auto it = pages_.find(std::string(request.url().encoded_path())); - const auto page = it != pages_.end() ? it->second : Page{.status = 404}; - - if (page.delay.count() > 0) - co_await anyhttp::delay(page.delay); - - anyhttp::Fields fields; - fields.set(http::field::content_type, page.content_type); - if (!page.location.empty()) - fields.set(http::field::location, page.location); - - if (auto [ec] = co_await response.submit(page.status, std::move(fields)); ec) - co_return; - co_await response.write_eof(boost::asio::buffer(page.body)); - }); } // @@ -71,10 +81,14 @@ Site::Site(const anyhttp::Executor& executor) : requests_(std::make_shared Site::poll_dns() +{ + const std::stop_token stopped = co_await capy::this_coro::stop_token; + while (!stopped.stop_requested()) + { + dns_context_.restart(); + dns_context_.poll(); + std::ignore = co_await corosio::delay(std::chrono::milliseconds(1)); } } Site::~Site() { + polling_.request_stop(); std::error_code ignored; std::filesystem::remove_all(directory_, ignored); } @@ -126,6 +161,7 @@ void Site::stop() if (dns_) queries_ = dns_->queries(); dns_.reset(); + polling_.request_stop(); } // ================================================================================================= diff --git a/test/site.hpp b/test/site.hpp index 92f2c3d..2a4bc1d 100644 --- a/test/site.hpp +++ b/test/site.hpp @@ -4,10 +4,18 @@ // server has a certificate for "localhost". That name comes from /etc/hosts and not from the DNS // server, which a test can only have the last word on with a name that the certificate is not for. // +// The DNS server comes from the tests of c-ares-async, where it is on Asio whatever the runtime. So +// it has a context of its own, which a coroutine polls from the one thread that everything runs on: +// a server on a thread of its own would race whoever looks at what it was asked. +// #pragma once #include +#include +#include +#include + #include #include @@ -16,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -46,7 +55,7 @@ class Site std::chrono::steady_clock::time_point time; }; - explicit Site(const anyhttp::Executor& executor); + explicit Site(const boost::capy::any_executor& executor); ~Site(); /// A path that has no page gets a 404. @@ -70,7 +79,8 @@ class Site void stop(); private: - void start(const anyhttp::Executor& executor); + void start(const boost::capy::any_executor& executor); + boost::capy::task<> poll_dns(); std::filesystem::path directory_; std::filesystem::path certificate_file_; @@ -78,7 +88,9 @@ class Site std::map pages_; std::shared_ptr> requests_; std::optional server_; + boost::asio::io_context dns_context_; std::optional dns_; + std::stop_source polling_; std::vector queries_; // what the DNS server knew when it was stopped std::uint16_t port_ = 0; }; From 4790ed1994c8e95c9b0a06e2d6446e4a222856d2 Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Tue, 3 Nov 2026 19:19:57 +0000 Subject: [PATCH 19/27] test: drop the workaround for a cancelled HTTP/3 connect abandon_quic_sessions was there because anyhttp went on sending for 30 seconds after a cancelled connect, so the context did not run dry. The pin has ef20614, which ends that connection, and the tests now end without stopping the context. So the README has no such wait to warn about, and main() prints the counts after run() rather than from a handler. Only tested on Corosio: the Asio code on http-probe still has the workaround. Co-Authored-By: Claude Opus 5.5 --- README.md | 3 --- src/main.cpp | 17 +++++++---------- test/prober_test.cpp | 13 ------------- 3 files changed, 7 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index f212ac2..8bce29e 100644 --- a/README.md +++ b/README.md @@ -59,9 +59,6 @@ anyhttp puts them into some of its log messages regardless. **Use `--dns-server`.** The resolver in the devcontainer answers with no data for HTTPS records, even where they exist, which looks just like no host having any. -**The program can take a while to end.** If a host did not answer over HTTP/3, anyhttp goes on sending -to it for 30 seconds after the timeout. The counts are printed when the work is done, ahead of that. - **A response goes to memory as a whole,** as anyhttp has no way of getting at it piece by piece with a GET yet. So only `--timeout-ms` limits how large one can get. diff --git a/src/main.cpp b/src/main.cpp index c4b3925..b66bbd4 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -212,18 +212,10 @@ int main(int argc, char* argv[]) std::exception_ptr failed; // - // What a handler throws ends the process, so it stays in here until the context is through. - // Printing here and not after run(): a host that did not answer can keep the context from - // running dry for a while, see the README, and this is what the wait would be for. + // What a handler throws ends the process, so this one leaves it for after run(). // boost::capy::run_async( - context.get_executor(), - [&]() noexcept { - std::println(stderr, "{}", http_probe::format_summary(counts, color_summary)); - if (with_records == 0 && !own_dns_server) - std::println(stderr, "http-probe: none of the hosts has an HTTPS record. Not every " - "resolver answers for them, so try --dns-server."); - }, + context.get_executor(), []() noexcept {}, [&](std::exception_ptr ex) noexcept { failed = ex; context.stop(); @@ -231,6 +223,11 @@ int main(int argc, char* argv[]) context.run(); if (failed) std::rethrow_exception(failed); + + std::println(stderr, "{}", http_probe::format_summary(counts, color_summary)); + if (with_records == 0 && !own_dns_server) + std::println(stderr, "http-probe: none of the hosts has an HTTPS record. Not every " + "resolver answers for them, so try --dns-server."); } catch (const std::invalid_argument& ex) { diff --git a/test/prober_test.cpp b/test/prober_test.cpp index 1a4ed8c..c9da083 100644 --- a/test/prober_test.cpp +++ b/test/prober_test.cpp @@ -111,13 +111,6 @@ class Probe : public testing::Test const auto run = [&]() -> capy::task<> { co_await prober.run(); site.stop(); - // - // For the tests with a QUIC endpoint that never answers. anyhttp does not close a - // session whose connect() was cancelled: it goes on sending until its idle timeout of - // 30 seconds, and the context does not run dry before that. To go once that is fixed. - // - if (abandon_quic_sessions) - context.stop(); }; std::exception_ptr failed; @@ -177,7 +170,6 @@ class Probe : public testing::Test corosio::io_context context; Site site{context.get_executor()}; std::vector results; - bool abandon_quic_sessions = false; }; } // namespace @@ -373,7 +365,6 @@ TEST_F(Probe, ReportsEachHostOnceHoweverManyThereAre) TEST_F(Probe, DoesNotLetASlowHostHoldUpTheOthers) { auto silent = silent_udp_socket(); - abandon_quic_sessions = true; site.dns().add_https("localhost", 1, ".", h3_at(silent.local_endpoint().port())); auto configuration = config({"localhost", "nowhere.example.test"}); @@ -389,7 +380,6 @@ TEST_F(Probe, DoesNotLetASlowHostHoldUpTheOthers) TEST_F(Probe, WorksOnOneHostAtATimeIfToldTo) { auto silent = silent_udp_socket(); - abandon_quic_sessions = true; site.dns().add_https("localhost", 1, ".", h3_at(silent.local_endpoint().port())); auto configuration = config({"localhost", "nowhere.example.test"}); @@ -499,7 +489,6 @@ TEST_F(Probe, FallsBackToTheNextRecordWhenAConnectionFails) // else will either. // auto silent = silent_udp_socket(); - abandon_quic_sessions = true; site.dns().add_https("localhost", 1, ".", h3_at(silent.local_endpoint().port())); site.dns().add_https("localhost", 2, ".", h3_at(site.port())); @@ -518,7 +507,6 @@ TEST_F(Probe, FallsBackToTheNextRecordWhenAConnectionFails) TEST_F(Probe, FallsBackFromHttp3ToTcpWhenAConnectionFails) { auto silent = silent_udp_socket(); - abandon_quic_sessions = true; site.dns().add_https("localhost", 1, ".", h3_at(silent.local_endpoint().port())); site.dns().add_https("localhost", 2, ".", h2_at(site.port())); @@ -845,7 +833,6 @@ TEST_F(Probe, LogsTheHttpAndTheLookupsAtDebugLevel) TEST_F(Probe, GivesUpOnAServerThatDoesNotAnswer) { auto silent = silent_udp_socket(); - abandon_quic_sessions = true; site.dns().add_https("localhost", 1, ".", h3_at(silent.local_endpoint().port())); auto configuration = config(); From bdbbb3105aaa9a671310c2c1dc4a0bc13b598778 Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Tue, 3 Nov 2026 19:46:58 +0000 Subject: [PATCH 20/27] chore: drop an unused include and a move from a copied executor, add capy to the dictionary Co-Authored-By: Claude Opus 5.5 --- .cspell.json | 1 + src/prober.cpp | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.cspell.json b/.cspell.json index 9d47d9b..51af953 100644 --- a/.cspell.json +++ b/.cspell.json @@ -2,6 +2,7 @@ "words": [ "anyhttp", "ASIO", + "capy", "cares", "corosio", "Eisenlohr", diff --git a/src/prober.cpp b/src/prober.cpp index 12935c7..ffb337c 100644 --- a/src/prober.cpp +++ b/src/prober.cpp @@ -19,7 +19,6 @@ #include #include #include -#include namespace http = boost::beast::http; @@ -102,7 +101,7 @@ void prefer_ipv6(std::vector& addresses) Prober::Prober(capy::any_executor executor, Config config, Observer observer) : executor_(executor), config_(std::move(config)), observer_(std::move(observer)), - resolver_(std::move(executor)) + resolver_(executor) { if (config_.hosts.empty()) throw std::invalid_argument("nothing to do: no host names given"); From 321cf0e5801d2503947c627976b9d90e8b792928 Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Tue, 3 Nov 2026 19:46:58 +0000 Subject: [PATCH 21/27] fix: add --h3, to connect only to hosts with a record that offers HTTP/3 It filters hosts and not protocols: a host with such a record is treated as it would be without the option, so it can still go over TCP or be out of reach. The others are left out after their HTTPS lookup, printed only with --all and counted as "without h3". A failed lookup is not taken for having no HTTP/3. Co-Authored-By: Claude Opus 5.5 --- README.md | 6 +++ src/main.cpp | 7 ++- src/output.cpp | 8 +++- src/prober.cpp | 7 +++ src/prober.hpp | 5 +++ src/routes.cpp | 8 ++++ src/routes.hpp | 3 ++ test/output_test.cpp | 13 ++++++ test/prober_test.cpp | 104 +++++++++++++++++++++++++++++++++++++++++++ test/routes_test.cpp | 26 +++++++++++ 10 files changed, 183 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8bce29e..297cb4b 100644 --- a/README.md +++ b/README.md @@ -36,12 +36,18 @@ look like this, on stderr: | has a record that says the service is not available | out of reach, not connected to | | does not exist | not found, not connected to | | gets no answer from DNS, or has no address | lookup failed, so there is no telling, not connected to | +| has no record that offers `h3`, with `--h3` | without h3, not connected to | Aliases are followed, a record has to be compatible, and the lowest priority goes first. Within a record, HTTP/3 goes first, and then TCP on the same port: HTTP/2 if the record has `h2`, otherwise HTTP/1.1, which is in every record that does not say `no-default-alpn`. The route in the output is what the server went for, which may be HTTP/1.1 where HTTP/2 was offered. +**With `--h3`,** only hosts with a compatible record that has `h3` in its `alpn` go ahead, on whichever +target, and they are treated as above, so that one can still end up out of reach, or going over TCP +when HTTP/3 can not be connected to. The others are left out after their HTTPS lookup, and printed +only with `--all`. The counts at the end then add how many there were, as `5 without h3`. + **One request means one.** The next route is for when connecting failed. Once a request has gone out, the host has been asked, so that a timeout or a response cut short ends it there, as a failure. diff --git a/src/main.cpp b/src/main.cpp index b66bbd4..6079c01 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -100,7 +100,10 @@ po::options_description describe_options(Arguments& arguments) config.timeout = std::chrono::milliseconds(not_negative("--timeout-ms", value)); }), "how long one request may take"); - opts("all", po::bool_switch(&arguments.all), "also print the hosts that do not exist"); + opts("h3", po::bool_switch(&config.h3_only), + "connect only to the hosts with an HTTPS record that offers HTTP/3"); + opts("all", po::bool_switch(&arguments.all), + "also print the hosts that do not exist, and those that --h3 leaves out"); opts("color", po::bool_switch(&arguments.color), "colour the output even where it does not go to a terminal, e.g. into 'less -R'"); opts("verbose,v", po::value>()->zero_tokens()->composing(), @@ -200,7 +203,7 @@ int main(int argc, char* argv[]) ++counts[result.outcome]; if (!result.records.empty() || result.aliased) ++with_records; - if (all || result.outcome != Outcome::not_found) + if (all || (result.outcome != Outcome::not_found && result.outcome != Outcome::no_h3)) { std::print("{}", http_probe::format_result(result, color_results)); std::fflush(stdout); // a run can take a while, and it is worth watching diff --git a/src/output.cpp b/src/output.cpp index ddc595a..d332696 100644 --- a/src/output.cpp +++ b/src/output.cpp @@ -114,6 +114,9 @@ std::string format_result(const Result& result, bool color) case Outcome::lookup_failed: text = std::format("{}: {}\n", host, colored("lookup failed", color_of::bad, color)); break; + case Outcome::no_h3: + text = std::format("{}: no h3 record\n", host); + break; } for (const auto& record : result.records) @@ -134,10 +137,11 @@ std::string format_summary(Counts counts, bool color) return counts[outcome] ? colored(line, code, color) : line; }; return std::format( - "{}, {}, {}, {} not found, {}", count(Outcome::answered, "answered", color_of::good), + "{}, {}, {}, {} not found, {}{}", count(Outcome::answered, "answered", color_of::good), count(Outcome::failed, "failed", color_of::bad), count(Outcome::out_of_reach, "out of reach", color_of::moved), counts[Outcome::not_found], - count(Outcome::lookup_failed, "lookups failed", color_of::bad)); + count(Outcome::lookup_failed, "lookups failed", color_of::bad), + counts[Outcome::no_h3] ? std::format(", {} without h3", counts[Outcome::no_h3]) : ""); } // ================================================================================================= diff --git a/src/prober.cpp b/src/prober.cpp index ffb337c..cb77447 100644 --- a/src/prober.cpp +++ b/src/prober.cpp @@ -224,6 +224,13 @@ capy::task Prober::probe(const std::string& host) for (const auto& rejected : plan.rejected) spdlog::debug("{}: rejected: {}", host, rejected); + if (config_.h3_only && !found.lookup_failed && !found.not_found && !has_h3_record(found)) + { + spdlog::debug("{}: left out, no record offers HTTP/3", host); + result.outcome = Result::Outcome::no_h3; + co_return result; + } + if (plan.routes.empty()) { result.outcome = found.lookup_failed ? Result::Outcome::lookup_failed diff --git a/src/prober.hpp b/src/prober.hpp index 0b44f49..1495d2d 100644 --- a/src/prober.hpp +++ b/src/prober.hpp @@ -52,6 +52,10 @@ struct Config /// Where to go for HTTPS when the records don't say. Only tests have a use for anything else. std::uint16_t default_port = 443; + + /// Connect only to the hosts with a record that offers HTTP/3, see has_h3_record(). Those are + /// treated as usual, which includes going over TCP if HTTP/3 can not be connected to. + bool h3_only = false; }; /// Whether \p host is a name and nothing but a name: no URL, port, path or wildcard, and not an @@ -78,6 +82,7 @@ struct Result out_of_reach, ///< not connected to, its records say so or lead elsewhere, see plan_routes() not_found, ///< not connected to, the name does not exist lookup_failed, ///< not connected to, DNS gave no answer, so there is no telling + no_h3, ///< not connected to, it has no record that offers HTTP/3, see Config::h3_only }; std::string host; diff --git a/src/routes.cpp b/src/routes.cpp index f8c39cb..1fc85a1 100644 --- a/src/routes.cpp +++ b/src/routes.cpp @@ -210,6 +210,14 @@ Plan plan_routes(std::string_view host, const ServiceRecords& found, std::uint16 return plan; } +bool has_h3_record(const ServiceRecords& found) +{ + return std::ranges::any_of(found.records, [](const c_ares_async::https_record& record) { + const auto service = parse_service(record); + return !service.incompatible && service.offers("h3"); + }); +} + // ================================================================================================= } // namespace http_probe diff --git a/src/routes.hpp b/src/routes.hpp index 9d989dc..c854bfd 100644 --- a/src/routes.hpp +++ b/src/routes.hpp @@ -72,6 +72,9 @@ struct Plan Plan plan_routes(std::string_view host, const ServiceRecords& found, std::uint16_t default_port = 443); +/// Whether a record that has to be taken seriously (section 8) offers HTTP/3, wherever it points. +bool has_h3_record(const ServiceRecords& found); + // ================================================================================================= } // namespace http_probe diff --git a/test/output_test.cpp b/test/output_test.cpp index eba10ba..b8a4a35 100644 --- a/test/output_test.cpp +++ b/test/output_test.cpp @@ -75,6 +75,19 @@ TEST(Output, SaysWhyThereIsNoResponse) "example.com: lookup failed\n"); } +TEST(Output, SaysThatAHostWasLeftOut) +{ + EXPECT_EQ(format_result({.host = "example.com", .outcome = Outcome::no_h3}, false), + "example.com: no h3 record\n"); +} + +TEST(Output, CountsTheHostsLeftOutOnlyIfThereAreAny) +{ + EXPECT_EQ(format_summary({{Outcome::answered, 3}, {Outcome::no_h3, 5}}, false), + "3 answered, 0 failed, 0 out of reach, 0 not found, 0 lookups failed, 5 without h3"); + EXPECT_THAT(format_summary({{Outcome::answered, 3}}, false), Not(HasSubstr("h3"))); +} + TEST(Output, ColoursTheWayCAresAsyncDoes) { const auto text = format_result(answered(), true); diff --git a/test/prober_test.cpp b/test/prober_test.cpp index c9da083..b27e9ee 100644 --- a/test/prober_test.cpp +++ b/test/prober_test.cpp @@ -898,3 +898,107 @@ TEST_F(Probe, ReportsAFailedLookup) EXPECT_THAT(notes(results[0]), HasSubstr("HTTPS lookup")); EXPECT_THAT(site.requests(), IsEmpty()); } + +// ================================================================================================= +// --h3 +// ================================================================================================= + +TEST_F(Probe, H3OnlyRequestsAHostWithARecordThatOffersIt) +{ + site.dns().add_https("localhost", 1, ".", {alpn_h3, port_param(site.port())}); + site.page("/", {.body = "hello"}); + + auto configuration = config(); + configuration.h3_only = true; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::answered) << notes(results[0]); + EXPECT_EQ(results[0].route, std::format("h3 localhost:{}", site.port())); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, H3OnlyLeavesOutAHostWithoutARecord) +{ + site.dns().add_address("localhost", "127.0.0.1"); + site.page("/", {.body = "hello"}); + + auto configuration = config(); + configuration.h3_only = true; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::no_h3); + EXPECT_THAT(results[0].address, IsEmpty()); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(Probe, H3OnlyLeavesOutAHostWhoseRecordsDoNotOfferIt) +{ + site.dns().add_https("localhost", 1, ".", h2_at(site.port())); + site.page("/", {.body = "hello"}); + + auto configuration = config(); + configuration.h3_only = true; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::no_h3); + EXPECT_EQ(results[0].records.size(), 1u) << "for --all to say why"; + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(Probe, H3OnlyStillFallsBackToTcpWhenAConnectionFails) +{ + auto silent = silent_udp_socket(); + + site.dns().add_https("localhost", 1, ".", h3_at(silent.local_endpoint().port())); + site.dns().add_https("localhost", 2, ".", h2_at(site.port())); + site.page("/", {.body = "hello"}); + + auto configuration = config(); + configuration.timeout = 200ms; + configuration.h3_only = true; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 200) << notes(results[0]); + EXPECT_EQ(results[0].route, std::format("h2 localhost:{}", site.port())); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, H3OnlyStillSaysThatARecordLeadsElsewhere) +{ + site.dns().add_https("localhost", 1, "elsewhere.example.test", h3_at(site.port())); + + auto configuration = config(); + configuration.h3_only = true; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::out_of_reach); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(Probe, H3OnlyStillSaysThatANameDoesNotExist) +{ + auto configuration = config({"nowhere.example.test"}); + configuration.h3_only = true; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::not_found); +} + +TEST_F(Probe, H3OnlyDoesNotTakeAFailedLookupForNoHttp3) +{ + site.dns().set_rcode(ARES_RCODE_SERVFAIL); + + auto configuration = config(); + configuration.h3_only = true; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::lookup_failed); + EXPECT_THAT(site.requests(), IsEmpty()); +} diff --git a/test/routes_test.cpp b/test/routes_test.cpp index c029f43..068d9ea 100644 --- a/test/routes_test.cpp +++ b/test/routes_test.cpp @@ -289,6 +289,32 @@ TEST(Plan, TreatsARecordWithAnInvalidPortAsNoRecord) EXPECT_THAT(routes(plan), ElementsAre("h2 example.com:443")); } +TEST(HasH3Record, IsTrueForARecordThatOffersIt) +{ + EXPECT_TRUE(has_h3_record(found({record(1, ".", {"alpn=h3"})}))); + EXPECT_TRUE(has_h3_record(found({record(1, ".", {"alpn=h2,h3"})}))); + EXPECT_TRUE(has_h3_record(found({record(1, ".", {"alpn=h2"}), record(2, ".", {"alpn=h3"})}))); +} + +TEST(HasH3Record, DoesNotCareWhereTheRecordPoints) +{ + EXPECT_TRUE(has_h3_record(found({record(1, "other.example", {"alpn=h3"})}))); +} + +TEST(HasH3Record, IsFalseForAnythingElse) +{ + EXPECT_FALSE(has_h3_record(found({}))); + EXPECT_FALSE(has_h3_record(found({record(1, ".")}))); + EXPECT_FALSE(has_h3_record(found({record(1, ".", {"alpn=h2"})}))); + EXPECT_FALSE(has_h3_record(found({record(1, ".", {"alpn=h3-29,h2"})}))); +} + +TEST(HasH3Record, IgnoresARecordThatHasToBeIgnored) +{ + EXPECT_FALSE(has_h3_record(found({record(1, ".", {"alpn=h3", "mandatory=ech", "ech=dead"})}))); + EXPECT_FALSE(has_h3_record(found({record(1, ".", {"alpn=h3", "port=none"})}))); +} + TEST(Plan, PassesOnWhatWentWrongInTheLookup) { auto records = found({}); From d14af6f546bfba7b4d1db77a057877da9a515f4a Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Tue, 3 Nov 2026 20:00:01 +0000 Subject: [PATCH 22/27] fix: print the help in the width of the terminal The same way as the server of anyhttp, except for looking at stdout as well as stderr, as --help goes to one and a usage error to the other. Without a terminal it is the 80 columns of Boost.ProgramOptions, in place of a fixed 100. Boost only wraps the table of options, so the paragraphs around it are wrapped here. Co-Authored-By: Claude Opus 5.5 --- src/main.cpp | 70 +++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 11 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 6079c01..e3c94ae 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -22,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -32,18 +34,18 @@ namespace po = boost::program_options; namespace { -constexpr std::string_view about = R"(usage: http-probe [options] HOST... +constexpr std::string_view usage = R"(usage: http-probe [options] HOST... http-probe [options] --file LIST - -Looks up the HTTPS records of each host (RFC 9460), and requests https://HOST/ once, the way they -say or over TCP on port 443 if they don't, printing the status and header fields of the response. -Redirects are reported and not followed, and nothing else on the host is requested. )"; -constexpr std::string_view exit_code = R"( -The exit code is 0 if every host that was connected to answered, whatever the status, and every -lookup got an answer. -)"; +constexpr std::string_view about = + "Looks up the HTTPS records of each host (RFC 9460), and requests https://HOST/ once, the way " + "they say or over TCP on port 443 if they don't, printing the status and header fields of the " + "response. Redirects are reported and not followed, and nothing else on the host is requested."; + +constexpr std::string_view exit_code = + "The exit code is 0 if every host that was connected to answered, whatever the status, and " + "every lookup got an answer."; struct Arguments { @@ -66,6 +68,47 @@ int not_negative(std::string_view option, int value) return value; } +// +// How wide to print the help. Either stream will do, as they are on the same terminal, and with +// 'http-probe --help | less' only stderr still is one. +// +unsigned help_columns() +{ + for (const int fd : {STDOUT_FILENO, STDERR_FILENO}) + { + winsize ws{}; + if (::ioctl(fd, TIOCGWINSZ, &ws) == 0 && ws.ws_col >= 40) + return ws.ws_col; + } + return po::options_description::m_default_line_length; +} + +// +// A paragraph broken at its spaces into lines that fit. +// +std::string wrap(std::string_view paragraph, unsigned columns) +{ + std::string wrapped; + std::size_t line = 0; + for (const auto range : paragraph | std::views::split(' ')) + { + const std::string_view word{range}; + if (line > 0 && line + 1 + word.size() > columns) + { + wrapped += '\n'; + line = 0; + } + else if (line > 0) + { + wrapped += ' '; + ++line; + } + wrapped += word; + line += word.size(); + } + return wrapped; +} + // // The options, stored into 'arguments', which has to outlive parsing them. // @@ -73,7 +116,8 @@ po::options_description describe_options(Arguments& arguments) { auto& config = arguments.config; - po::options_description desc("Options", 100, 50); + const auto columns = help_columns(); + po::options_description desc("Options", columns, columns / 2); auto opts = desc.add_options(); opts("help,h", po::bool_switch(&arguments.help), "produce this help message"); opts("host", po::value(&config.hosts)->value_name("HOST"), @@ -115,7 +159,11 @@ po::options_description describe_options(Arguments& arguments) void print_usage(std::ostream& out, const po::options_description& options) { - out << about << '\n' << options << exit_code; + const auto columns = help_columns(); + out << usage << '\n' + << wrap(about, columns) << "\n\n" + << options << '\n' + << wrap(exit_code, columns) << '\n'; } void parse_arguments(int argc, char* argv[], const po::options_description& options, From 5f0d62b8a1111c83dd17170dcc2fe55a9c6a3696 Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Tue, 3 Nov 2026 20:00:07 +0000 Subject: [PATCH 23/27] fix: add --dot, to ask the DNS servers over TLS Along with --tls-hostname and --tls-no-verify, and host names in --dns-server, as in the resolve example of c-ares-async. The transport is the library's. The connection for DNS over TLS stays open and keeps the event loop going, so the resolver now lives for the duration of run(). Co-Authored-By: Claude Opus 5.5 --- README.md | 13 ++- src/main.cpp | 21 +++- src/prober.cpp | 150 ++++++++++++++++++++++-- src/prober.hpp | 22 +++- test/CMakeLists.txt | 11 +- test/prober_test.cpp | 263 ++++++++++++++++++++++++++++++++++++++++++- test/site.cpp | 6 + test/site.hpp | 3 + 8 files changed, 469 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 297cb4b..761ac36 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Requests hosts the way their HTTPS records say, on [anyhttp](https://github.com/ ```sh http-probe --dns-server 1.1.1.1 cloudflare.com http-probe --dns-server 1.1.1.1 --file hostnames.txt +http-probe --dot --dns-server one.one.one.one cloudflare.com ``` It looks up the HTTPS records of each host in DNS ([RFC 9460](https://www.rfc-editor.org/rfc/rfc9460)), @@ -63,7 +64,17 @@ log, so `--concurrency 1` helps. Colours are for terminals only unless there is anyhttp puts them into some of its log messages regardless. **Use `--dns-server`.** The resolver in the devcontainer answers with no data for HTTPS records, even -where they exist, which looks just like no host having any. +where they exist, which looks just like no host having any. It takes addresses and host names, with a +port behind either. A name is looked up in the clear with the servers of the system, whatever the +queries proper go out as, and only its first address becomes a server. + +**`--dot` asks over DNS over TLS** ([RFC 7858](https://www.rfc-editor.org/rfc/rfc7858)), on port 853 +unless the server says otherwise, for the records and the addresses alike. The certificate of the +server needs a name to be checked against. That is the host name in `--dns-server`, and with an +address, or several names, `--tls-hostname` has to give it. `--tls-no-verify` skips the check, which +is for testing and has nothing to do with the certificates of the hosts. A failed handshake makes the +hosts lookup failures, and says why in the log. `--ca-file` is for the hosts as well: the certificate +of the DNS server goes by the trust store of OpenSSL, which `SSL_CERT_FILE` can change. **A response goes to memory as a whole,** as anyhttp has no way of getting at it piece by piece with a GET yet. So only `--timeout-ms` limits how large one can get. diff --git a/src/main.cpp b/src/main.cpp index e3c94ae..8e6e58e 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -124,11 +124,24 @@ po::options_description describe_options(Arguments& arguments) "a host to request, the same as giving HOST by itself"); opts("file", po::value(&arguments.files)->value_name("LIST"), "read the hosts from a file, one per line, as names or as URLs"); - opts("dns-server", po::value(&config.dns_server)->value_name("ADDR"), - "the DNS server to ask, like 192.0.2.1 or 192.0.2.1:5353; default: the system's, which may " - "well not answer for HTTPS records"); + opts("dns-server", po::value(&config.dns_server)->value_name("SERVER"), + "the DNS servers to ask, comma separated, by address or by name, like 192.0.2.1, " + "192.0.2.1:5353 or dns.example.com; default: the system's, which may well not answer for " + "HTTPS records"); + opts("dot", po::bool_switch(&config.dns_tls.enabled), + "ask the DNS servers over TLS (RFC 7858), which moves them to port 853 unless they say " + "otherwise"); + opts("tls-hostname", po::value(&config.dns_tls.hostname)->value_name("NAME"), + "with --dot, the name to use for SNI and to verify the certificate of the DNS server " + "against, like one.one.one.one; default: the host name in --dns-server"); + opts("tls-no-verify", po::value()->zero_tokens()->notifier([&config](bool) { + config.dns_tls.verify = false; + }), + "with --dot, accept any certificate from the DNS server (testing only); does not apply to " + "the hosts"); opts("ca-file", po::value(&config.tls_ca_file)->value_name("PEM"), - "the CA certificates to verify servers with; default: the system's"); + "the CA certificates to verify the hosts with; default: the system's. Not for --dot, which " + "goes by the trust store of OpenSSL, as SSL_CERT_FILE can say"); opts("concurrency", po::value() ->value_name("N") diff --git a/src/prober.cpp b/src/prober.cpp index cb77447..aa0d311 100644 --- a/src/prober.cpp +++ b/src/prober.cpp @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include #include #include @@ -54,6 +56,85 @@ void log_to_spdlog(c_ares_async::log_level level, std::string_view message) } } +// ------------------------------------------------------------------------------------------------- + +// +// The entries of a list of DNS servers. +// +std::vector split_servers(std::string_view list) +{ + std::vector servers; + for (const auto part : std::views::split(list, ',')) + { + const std::string_view server(part); + if (const auto first = server.find_first_not_of(" \t"); first != server.npos) + servers.push_back(server.substr(first, server.find_last_not_of(" \t") - first + 1)); + } + return servers; +} + +// +// A server given by name, and the port behind it, colon included, if there is one. +// +struct ServerName +{ + std::string_view name; + std::string_view port; +}; + +// +// Tells the names in a list from what c-ares takes for a server, all of which is addresses: +// "192.0.2.1", "192.0.2.1:5353", "[::1]:5353", "fe80::1%eth0" or a dns:// URI. Whatever is neither +// goes to c-ares as well, for it to turn it down. +// +std::optional server_name(std::string_view server) +{ + const auto colon = server.find(':'); + const auto name = server.substr(0, colon); + const auto port = colon == server.npos ? std::string_view{} : server.substr(colon); + + const auto is_digit = [](unsigned char c) { return std::isdigit(c) != 0; }; + if (!is_host_name(name) || + (!port.empty() && (port.size() == 1 || !std::ranges::all_of(port.substr(1), is_digit)))) + return std::nullopt; + + return ServerName{name, port}; +} + +// +// A certificate needs a name to be checked against. A single name in the list of servers is that +// name, while an address is none, and of several names none is the one for all of them. Rather than +// quietly talking to whoever answers, this says so. +// +c_ares_async::tls_options complete_dns_tls(c_ares_async::tls_options tls, std::string_view servers) +{ + if (!tls.enabled) + { + if (!tls.hostname.empty() || !tls.verify) + throw std::invalid_argument("--tls-hostname and --tls-no-verify only apply with --dot"); + return tls; + } + + if (!tls.hostname.empty()) + return tls; + + std::set names; + for (const auto server : split_servers(servers)) + if (const auto named = server_name(server)) + names.insert(named->name); + + if (names.size() == 1) + tls.hostname = *names.begin(); + else if (tls.verify) + throw std::invalid_argument(std::format( + "{}, or --tls-no-verify to skip the check", + names.empty() ? "--dot needs a name to verify the certificate of the DNS " + "server against: a host name as --dns-server, or --tls-hostname" + : "--dns-server names several hosts, so --dot needs " + "--tls-hostname to verify their certificates against")); + return tls; +} + // ================================================================================================= } // namespace @@ -100,8 +181,7 @@ void prefer_ipv6(std::vector& addresses) } Prober::Prober(capy::any_executor executor, Config config, Observer observer) - : executor_(executor), config_(std::move(config)), observer_(std::move(observer)), - resolver_(executor) + : executor_(executor), config_(std::move(config)), observer_(std::move(observer)) { if (config_.hosts.empty()) throw std::invalid_argument("nothing to do: no host names given"); @@ -117,21 +197,73 @@ Prober::Prober(capy::any_executor executor, Config config, Observer observer) hosts_.push_back(std::move(name)); } - resolver_.set_logger(&log_to_spdlog); + config_.dns_tls = complete_dns_tls(std::move(config_.dns_tls), config_.dns_server); +} - if (!config_.dns_server.empty()) - if (const auto ec = resolver_.set_servers(config_.dns_server)) - throw std::invalid_argument( - std::format("invalid DNS server '{}': {}", config_.dns_server, ec.message())); +// +// c-ares takes addresses for its servers and nothing else, so a name has to be looked up before +// there is anybody to ask, the regular way and in the clear, whatever the queries proper go out as. +// +// Only the first address of a name becomes a server. c-ares gives every query its tries once per +// server, with the timeout doubling each time around, so all of them would not be failover for free +// but make a host that nobody has an answer for take much longer. +// +capy::task Prober::dns_server_addresses() +{ + std::unique_ptr resolver; // only if there is a name to look up at all + std::string addresses; + + for (const auto server : split_servers(config_.dns_server)) + { + std::string address(server); + if (const auto named = server_name(server)) + { + if (!resolver) + { + resolver = std::make_unique(executor_); + resolver->set_logger(&log_to_spdlog); + if (!config_.dns_server_lookup.empty()) + if (const auto ec = resolver->set_servers(config_.dns_server_lookup)) + throw std::invalid_argument(ec.message()); + } + + spdlog::debug("looking up DNS server '{}'", named->name); + const auto [ec, endpoints] = co_await resolver->resolve(std::string(named->name), ""); + if (ec || endpoints.empty()) + throw std::runtime_error( + std::format("DNS server '{}': {}", named->name, ec ? ec.message() : "no addresses")); + + // + // An IPv6 address only needs its brackets if there is a port behind it. + // + const auto first = endpoints.front().address(); + address = named->port.empty() ? first.to_string() + : first.is_v6() ? std::format("[{}]{}", first.to_string(), named->port) + : std::format("{}{}", first.to_string(), named->port); + } + addresses += (addresses.empty() ? "" : ",") + address; + } + co_return addresses; } capy::task<> Prober::run() { + const auto servers = co_await dns_server_addresses(); + + resolver_ = std::make_unique(executor_, config_.dns_tls); + resolver_->set_logger(&log_to_spdlog); + if (!servers.empty()) + if (const auto ec = resolver_->set_servers(servers)) + throw std::invalid_argument( + std::format("invalid DNS server '{}': {}", config_.dns_server, ec.message())); + std::vector>> workers; for (std::size_t i = 0; i < std::min(config_.concurrency, hosts_.size()); ++i) workers.push_back(work()); std::ignore = co_await capy::when_all(std::move(workers)); + + resolver_.reset(); } capy::task> Prober::work() @@ -163,7 +295,7 @@ capy::task Prober::resolve_service(const std::string& host) // GHSA-6wfj-rwm7-3542 is about. c-ares has timeouts of its own. // spdlog::debug("{}: asking for the HTTPS records of '{}'", host, found.owner); - auto [ec, records] = co_await resolver_.resolve_https(found.owner); + auto [ec, records] = co_await resolver_->resolve_https(found.owner); if (ec) { if (ec == c_ares_async::make_ares_error(ARES_ENOTFOUND)) @@ -245,7 +377,7 @@ capy::task Prober::probe(const std::string& host) // host, and no record says that it has to have one. // spdlog::debug("{}: asking for the addresses of '{}'", host, host); - auto [lookup_ec, addresses] = co_await resolver_.resolve(host, ""); + auto [lookup_ec, addresses] = co_await resolver_->resolve(host, ""); if (lookup_ec || addresses.empty()) { result.outcome = Result::Outcome::lookup_failed; diff --git a/src/prober.hpp b/src/prober.hpp index 1495d2d..187f500 100644 --- a/src/prober.hpp +++ b/src/prober.hpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -44,9 +45,19 @@ struct Config /// including any addresses of the host that could not be connected to. std::chrono::milliseconds timeout{10000}; - /// The DNS servers to ask, as resolver::set_servers() takes them. Empty for those of the system. + /// The DNS servers to ask, comma separated: what resolver::set_servers() takes, or a host name + /// with or without a port. Empty for those of the system. std::string dns_server; + /// Whether to ask them over TLS (RFC 7858), and what to check their certificates against, in the + /// trust store of the TLS library, which tls_ca_file has no say in. A single host name in + /// dns_server goes without saying, as far as the name in here goes. + c_ares_async::tls_options dns_tls; + + /// Who to look up the host names in dns_server with, always in the clear. Empty for the servers + /// of the system, and only tests have a use for anything else. + std::string dns_server_lookup; + /// The CA certificates to verify servers against, as a PEM file. Empty for the system's. std::string tls_ca_file; @@ -119,7 +130,8 @@ class Prober using Observer = std::function; /// Throws \c std::invalid_argument for a configuration that is not good to go, and does nothing - /// else until run(). + /// else until run(), which has more to say about the DNS servers: the same for a list that can't + /// be used, and \c std::runtime_error for a host name in it that can't be looked up. Prober(capy::any_executor executor, Config config, Observer observer); capy::task<> run(); @@ -134,6 +146,7 @@ class Prober std::chrono::milliseconds duration{}; }; + capy::task dns_server_addresses(); capy::task> work(); capy::task resolve_service(const std::string& host); capy::task probe(const std::string& host); @@ -149,7 +162,10 @@ class Prober Observer observer_; std::vector hosts_; std::size_t next_host_ = 0; - c_ares_async::resolver resolver_; + + /// For the duration of run(), as a connection for DNS over TLS stays open for as long as this is + /// around, and with that the event loop busy. By pointer, as there is no std::optional of it. + std::unique_ptr resolver_; }; // ================================================================================================= diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7c800d6..81ac6a4 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -78,4 +78,13 @@ set_tests_properties(http_probe_refuses_an_unknown_option PROPERTIES add_test(NAME http_probe_describes_its_options COMMAND http-probe --help) set_tests_properties(http_probe_describes_its_options PROPERTIES - PASS_REGULAR_EXPRESSION "--dns-server ADDR") + PASS_REGULAR_EXPRESSION "--dns-server SERVER") + +add_test(NAME http_probe_refuses_dot_without_a_name COMMAND http-probe --dot localhost) +set_tests_properties(http_probe_refuses_dot_without_a_name PROPERTIES + PASS_REGULAR_EXPRESSION "--dot needs a name") + +add_test(NAME http_probe_refuses_tls_options_without_dot + COMMAND http-probe --tls-no-verify localhost) +set_tests_properties(http_probe_refuses_tls_options_without_dot PROPERTIES + PASS_REGULAR_EXPRESSION "only apply with --dot") diff --git a/test/prober_test.cpp b/test/prober_test.cpp index b27e9ee..05a4e5a 100644 --- a/test/prober_test.cpp +++ b/test/prober_test.cpp @@ -19,16 +19,24 @@ #include #include +#include + #include +#include +#include #include +#include #include +#include #include #include #include #include +#include #include #include #include +#include #include #include @@ -110,7 +118,7 @@ class Probe : public testing::Test // const auto run = [&]() -> capy::task<> { co_await prober.run(); - site.stop(); + stop_sites(); }; std::exception_ptr failed; @@ -148,10 +156,12 @@ class Probe : public testing::Test // void finish() { - site.stop(); + stop_sites(); context.run(); } + virtual void stop_sites() { site.stop(); } + static int quiet() { spdlog::set_level(spdlog::level::off); // anyhttp logs through the default logger @@ -1002,3 +1012,252 @@ TEST_F(Probe, H3OnlyDoesNotTakeAFailedLookupForNoHttp3) EXPECT_EQ(results[0].outcome, Result::Outcome::lookup_failed); EXPECT_THAT(site.requests(), IsEmpty()); } + +// ================================================================================================= +// --dot +// ================================================================================================= + +namespace +{ + +const test_certificate& dns_certificate() +{ + static const auto certificate = make_test_certificate("dns.test"); + return certificate; +} + +class DnsOverTls : public Probe +{ +protected: + void SetUp() override + { + site.dns().use_tls(dns_certificate()); + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.page("/", {.body = "hello"}); + } + + void TearDown() override + { + ::unsetenv("SSL_CERT_FILE"); + if (!trusted.empty()) + std::filesystem::remove(trusted); + } + + void stop_sites() override + { + site.stop(); + names.stop(); + } + + // + // The resolver verifies against the default trust store, which is the one thing that it lets + // anybody say something about. --ca-file does not come into it. + // + void trust_the_dns_server() + { + trusted = std::filesystem::temp_directory_path() / + std::format("http-probe-test-dns-{}.pem", ::getpid()); + std::ofstream(trusted) << dns_certificate().certificate_pem; + ASSERT_EQ(::setenv("SSL_CERT_FILE", trusted.c_str(), 1), 0); + } + + Config config_over_tls(std::string hostname, bool verify = true) + { + auto configuration = config(); + configuration.dns_tls = {.enabled = true, .hostname = std::move(hostname), .verify = verify}; + return configuration; + } + + /// A server of \p site, by the name that \ref names has for it. + Config config_by_name(const Site& server) + { + names.dns().add_address("dns.test", "127.0.0.1"); + + auto configuration = config(); + const auto address = server.dns_address(); + configuration.dns_server = "dns.test" + address.substr(address.rfind(':')); + configuration.dns_server_lookup = names.dns_address(); + return configuration; + } + + Site names{context.get_executor()}; // what the system's resolver is to a host name of a server + std::filesystem::path trusted; +}; + +} // namespace + +TEST_F(DnsOverTls, AsksOverTlsAndNothingElse) +{ + trust_the_dns_server(); + probe(config_over_tls("dns.test")); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::answered) << notes(results[0]); + EXPECT_EQ(results[0].status, 200); + EXPECT_GE(site.dns_queries(fake_dns_server::transport::tls), 1u); + EXPECT_EQ(site.dns_queries(fake_dns_server::transport::udp), 0u); + EXPECT_EQ(site.dns_queries(fake_dns_server::transport::tcp), 0u); +} + +TEST_F(DnsOverTls, DoesNotConnectIfTheCertificateIsForAnotherName) +{ + trust_the_dns_server(); + probe(config_over_tls("other.test")); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::lookup_failed); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(DnsOverTls, DoesNotConnectIfTheCertificateIsNotTrusted) +{ + probe(config_over_tls("dns.test")); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::lookup_failed); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(DnsOverTls, SkipsTheCheckIfToldTo) +{ + probe(config_over_tls("", false)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::answered) << notes(results[0]); + EXPECT_GE(site.dns_queries(fake_dns_server::transport::tls), 1u); +} + +TEST_F(DnsOverTls, StillVerifiesTheHostsWhenToldToSkipTheCheck) +{ + Site other(context.get_executor()); // has a certificate of its own + + auto configuration = config_over_tls("", false); + configuration.tls_ca_file = other.ca_file().string(); + other.stop(); + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::failed); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(DnsOverTls, TakesTheNameToVerifyFromTheDnsServer) +{ + trust_the_dns_server(); + + auto configuration = config_by_name(site); + configuration.dns_tls.enabled = true; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::answered) << notes(results[0]); + EXPECT_GE(site.dns_queries(fake_dns_server::transport::tls), 1u); + EXPECT_EQ(names.dns_queries("dns.test"), names.dns_queries(fake_dns_server::transport::udp)) + << "looking up the server is in the clear, and it is all that the other one was asked"; + EXPECT_GE(names.dns_queries("dns.test"), 1u); +} + +TEST_F(DnsOverTls, PrefersTheNameItIsGivenToTheOneOfTheDnsServer) +{ + trust_the_dns_server(); + + auto configuration = config_by_name(site); + configuration.dns_tls = {.enabled = true, .hostname = "other.test", .verify = true}; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::lookup_failed); +} + +TEST_F(DnsOverTls, TakesADnsServerByNameWithoutDot) +{ + names.dns().add_https("localhost", 1, ".", h3_at(site.port())); + probe(config_by_name(names)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::answered) << notes(results[0]); + EXPECT_EQ(site.dns_queries(fake_dns_server::transport::tls), 0u); +} + +TEST_F(DnsOverTls, EndsTheRunIfADnsServerCanNotBeLookedUp) +{ + auto configuration = config(); + configuration.dns_server = "missing.test"; + configuration.dns_server_lookup = names.dns_address(); + + EXPECT_THROW(probe(std::move(configuration)), std::runtime_error); + EXPECT_THAT(results, IsEmpty()); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(DnsOverTls, RefusesAnAddressWithoutANameToVerify) +{ + EXPECT_THROW(probe(config_over_tls("")), std::invalid_argument); + finish(); +} + +TEST_F(DnsOverTls, RefusesSeveralNamesWithoutOneToVerify) +{ + auto configuration = config_over_tls(""); + configuration.dns_server = "dns1.test,dns2.test"; + EXPECT_THROW(probe(std::move(configuration)), std::invalid_argument); + finish(); +} + +TEST_F(DnsOverTls, RefusesToVerifyWithoutDot) +{ + auto configuration = config(); + configuration.dns_tls.hostname = "dns.test"; + EXPECT_THROW(probe(std::move(configuration)), std::invalid_argument); + finish(); +} + +TEST_F(Probe, RefusesADnsServerThatIsNoServer) +{ + auto configuration = config(); + configuration.dns_server = "not a server"; + EXPECT_THROW(probe(std::move(configuration)), std::invalid_argument); + finish(); +} + +// +// The connection stays open for the next query and the server has no reason to close it, so the +// program would never end if that was keeping the event loop going. This takes a context that has +// nothing else to do, unlike the one of the others, and with that a server on a thread of its own, +// which is only looked at when it is done. +// +TEST(DnsOverTlsConnection, DoesNotKeepTheProgramFromEnding) +{ + spdlog::set_level(spdlog::level::off); + + corosio::io_context site_context; + Site site(site_context.get_executor()); + site.dns().use_tls(dns_certificate()); + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.page("/", {.body = "hello"}); + + std::size_t answered = 0; + { + std::jthread serving([&] { site_context.run(); }); + + corosio::io_context context; + Prober prober( + context.get_executor(), + {.hosts = {"localhost"}, + .timeout = 1000ms, + .dns_server = site.dns_address(), + .dns_tls = {.enabled = true, .hostname = "", .verify = false}, + .tls_ca_file = site.ca_file().string(), + .default_port = site.port()}, + [&](const Result& result) { answered += result.outcome == Result::Outcome::answered; }); + capy::run_async(context.get_executor())(prober.run()); + context.run(); // what hangs if it does not end, for the timeout of the test to find + + const auto stop = [](Site& site) -> capy::task<> { + site.stop(); + co_return; + }; + capy::run_async(site_context.get_executor())(stop(site)); + } + EXPECT_EQ(answered, 1u); +} diff --git a/test/site.cpp b/test/site.cpp index 94f5aa1..9ffa9bb 100644 --- a/test/site.cpp +++ b/test/site.cpp @@ -155,6 +155,12 @@ std::size_t Site::dns_queries(std::string_view name) const }); } +std::size_t Site::dns_queries(fake_dns_server::transport via) const +{ + const auto& queries = dns_ ? dns_->queries() : queries_; + return std::ranges::count(queries, via, &fake_dns_server::query::via); +} + void Site::stop() { server_.reset(); diff --git a/test/site.hpp b/test/site.hpp index 2a4bc1d..c126cc3 100644 --- a/test/site.hpp +++ b/test/site.hpp @@ -71,6 +71,9 @@ class Site /// How often the DNS server was asked about \p name, also after stop(). std::size_t dns_queries(std::string_view name) const; + /// How many questions came in over \p via, also after stop(). + std::size_t dns_queries(fake_dns_server::transport via) const; + /// Everything the server was asked so far, in order, and all of the targets alone. const std::vector& requests() const { return *requests_; } std::vector targets() const; From bda9b2bc6620e737fd7e3ee35debe05c0fdb00f1 Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Tue, 3 Nov 2026 20:02:50 +0000 Subject: [PATCH 24/27] chore: add alpn to the dictionary Co-Authored-By: Claude Opus 5.5 --- .cspell.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.cspell.json b/.cspell.json index 51af953..b93cfff 100644 --- a/.cspell.json +++ b/.cspell.json @@ -1,5 +1,6 @@ { "words": [ + "alpn", "anyhttp", "ASIO", "capy", From 1a1490dc614d7dbf09d35a161804acaae2f7dd98 Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Tue, 3 Nov 2026 20:17:53 +0000 Subject: [PATCH 25/27] fix: update test pattern to match all test files --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index dafc6ca..d45fd54 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -38,7 +38,7 @@ "testMate.cpp.test.parallelExecutionLimit": 20, "testMate.cpp.test.advancedExecutables": [ { - "pattern": "${command:cmake.buildDirectory}/test/test_*", + "pattern": "${command:cmake.buildDirectory}/test/*_tests", "cwd": "${workspaceDirectory}", "runTask": { "before": [ From 7276c0a52a41fb4dc10113921d9253dfc78e585d Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Tue, 3 Nov 2026 20:25:26 +0000 Subject: [PATCH 26/27] fix: add "SVCB" to the spell check dictionary --- .cspell.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.cspell.json b/.cspell.json index b93cfff..43460a2 100644 --- a/.cspell.json +++ b/.cspell.json @@ -8,7 +8,8 @@ "corosio", "Eisenlohr", "psedoc", - "spdlog" + "spdlog", + "SVCB" ], "ignorePaths": [ ".devcontainer/*", From 609c18b99d981367b38b2fba858c54650c5510e1 Mon Sep 17 00:00:00 2001 From: Peter Eisenlohr Date: Tue, 3 Nov 2026 20:29:50 +0000 Subject: [PATCH 27/27] chore: move the anyhttp pin to master, which has the h2 connection credit fix Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9960b99..bd30be1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,7 +48,7 @@ set(C_ARES_ASYNC_TLS ${TLS_LIBRARY}) include(FetchContent) FetchContent_Declare(anyhttp GIT_REPOSITORY https://github.com/pgit/anyhttp.git - GIT_TAG 649443c8883cb74ecaf306db0efb77c64c6ed606 + GIT_TAG cc3faee4fb351a56bbca48db7cfcac7585e59e8b SYSTEM) FetchContent_Declare(c_ares_async GIT_REPOSITORY https://github.com/pgit/c-ares-async.git