diff --git a/.cspell.json b/.cspell.json index 9d47d9b..43460a2 100644 --- a/.cspell.json +++ b/.cspell.json @@ -1,12 +1,15 @@ { "words": [ + "alpn", "anyhttp", "ASIO", + "capy", "cares", "corosio", "Eisenlohr", "psedoc", - "spdlog" + "spdlog", + "SVCB" ], "ignorePaths": [ ".devcontainer/*", diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index defaab7..d45fd54 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -38,7 +38,7 @@ "testMate.cpp.test.parallelExecutionLimit": 20, "testMate.cpp.test.advancedExecutables": [ { - "pattern": "${command:cmake.buildDirectory}/test/test_*", + "pattern": "${command:cmake.buildDirectory}/test/*_tests", "cwd": "${workspaceDirectory}", "runTask": { "before": [ @@ -75,8 +75,6 @@ "mounts": [ "source=cpp-devcontainer-bashhistory,target=/commandhistory,type=volume", - "source=${localEnv:HOME}/src/anyhttp,target=/workspaces/anyhttp,type=bind,consistency=cached", - "source=${localEnv:HOME}/src/c-ares-async,target=/workspaces/c-ares-async,type=bind,consistency=cached", "source=${localEnv:HOME}/.claude,target=/home/vscode/.claude,type=bind,consistency=cached", "source=${localEnv:HOME}/.claude.json,target=/home/vscode/.claude.json,type=bind,consistency=cached" ], diff --git a/CLAUDE.md b/CLAUDE.md index d800e22..1404155 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,29 +1,42 @@ -# Working on the crawler +# Working on http-probe -A web crawler on two libraries of the same author, each in a repository of its own: anyhttp for HTTP -and c-ares-async for DNS. [README.md](README.md) has the presets and the build commands. +It requests `https:///` once per host and reports what came back, connecting the way the +host's HTTPS records say. It is built on two libraries of the same author, each in a repository of +its own: anyhttp for HTTP and c-ares-async for DNS. +[README.md](README.md) has the build commands. ## Safety rules These hold for all code in this repository, including tests and experiments. -- **Never open, fetch or connect to anything in c-ares-async's `url_list.txt`, and never use it as - input.** It is for DNS lookups only: 1000 mixed sites, some adult or spam-looking. The crawler is - the first program here that could turn those names into connections. -- **Tests crawl local servers on loopback only.** -- **Once the program crawls, it refuses to run without an explicit host allowlist.** robots.txt and - per-host rate limits are built in from the start, not added later. +- **One request per host, and nothing more.** c-ares-async's lists of hosts are fine as input here, + which is what makes this rule matter: they have 1000 mixed sites, some adult or spam-looking. So + nothing is followed: no links, no redirects and no Alt-Svc. Another route is for a connection that + failed: once a request has gone out, that was it for the host, however it ended. +- **Tests connect to local servers on loopback only.** +- **If the program ever crawls, it refuses to run without an explicit host allowlist,** with + robots.txt and per-host rate limits built in from the start, not added later. - **Run tests and programs under `timeout -s KILL`**, 5 seconds per test. Never let a suite sit on a hang. +## The code + +[routes.cpp](src/routes.cpp) says what the HTTPS records of a name mean for how to connect, without +any I/O, and [prober.cpp](src/prober.cpp) does the rest. The tests in [test/](test) run against +[Site](test/site.hpp), a real anyhttp server and a DNS server with a zone of its own, on loopback. + ## The libraries - Changes to a library belong in that library's repository and session, with a test in its suite. - Do not work around a library bug in crawler code: get it fixed there, then move the pin. -- Moving a pin in [CMakeLists.txt](CMakeLists.txt) is a commit of its own. CI builds from the pins - only, so a crawler change that needs a library change has to wait for that change to be pushed. -- The `dev` preset builds the sibling checkouts as they are, on whatever branch they are on, and can - differ from the pins. Check with the `pinned` preset before committing. + Do not work around a library bug in this code: get it fixed there, then move the pin. +- Ask for a library change with an issue in that library's repository, using `gh`. Peter reviews the + full text first, and the same goes for comments on issues and for closing them. The repositories + are public, so reproduce on loopback and name no hosts from the lists. +- Moving a pin in [CMakeLists.txt](CMakeLists.txt) is a commit of its own. Everything builds from + the pins, so a change here that needs a library change has to wait for that change to be pushed. +- The libraries come from GitHub, as for any other user, and there are no checkouts of them to build + against. Other agents working on them do not interfere with this repository: nothing they do + arrives here until a pin moves. To read a library at its pin, look in `build/_deps/`. - Everything in one process has to agree on the standard library, the TLS library and the I/O runtime. [CMakeLists.txt](CMakeLists.txt) decides these for both libraries, so change them there. Do not configure a tree by hand: `cc` and `c++` are GCC on this host, and a tree without clang @@ -35,5 +48,4 @@ As in anyhttp: clang-format with 3-space indent and 100 columns, members `foo_`, `snake_case`, types `CamelCase`. Commit subjects are prefixed `fix:`, `refactor:`, `docs:`, `chore:` or `test:`. -Commit or push only when asked. `gh`'s token is read-only, so push over SSH to git@github.com; Peter -opens the pull requests. +Commit or push only when asked. Push over SSH to git@github.com; Peter opens the pull requests. diff --git a/CMakeLists.txt b/CMakeLists.txt index d77f2b6..bd30be1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.30) -project(crawler VERSION 0.0.1 DESCRIPTION "A web crawler on anyhttp and c-ares-async" LANGUAGES C CXX) +project(http_probe VERSION 0.0.1 DESCRIPTION "Requests hosts the way their HTTPS records say, on anyhttp and c-ares-async" LANGUAGES C CXX) set(CMAKE_CXX_STANDARD 23) set(CMAKE_CXX_STANDARD_REQUIRED On) @@ -20,10 +20,16 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") add_compile_options($<$:-stdlib=libc++>) add_link_options(-stdlib=libc++) list(PREPEND CMAKE_PREFIX_PATH /opt/libc++) + # As in anyhttp: the shared ones share their names with the libstdc++ ones in /usr/local/lib. + set(Boost_USE_STATIC_LIBS ON) endif() -# I/O runtime: ASIO or COROSIO. COROSIO would have to declare the capy and corosio pins here, too. -set(ANYHTTP_API ASIO) +# +# I/O runtime: Capy and Corosio, and only those. The libraries take either, which is their business: +# this is a program, and it gets to decide. These shadow the cache entries of the libraries, which +# go on saying ASIO in CMakeCache.txt. +# +set(ANYHTTP_API COROSIO) set(C_ARES_ASYNC_API ${ANYHTTP_API}) # @@ -36,28 +42,55 @@ set(C_ARES_ASYNC_TLS ${TLS_LIBRARY}) # -------------------------------------------------------------------------------------------------- # -# Pinned to a commit, and moving a pin is a commit of its own. To build against a checkout instead -# there is the 'dev' preset, which sets FETCHCONTENT_SOURCE_DIR_*. SYSTEM, so that the warnings of -# this project do not apply to the headers of the libraries. -# -# anyhttp: the head of branch 'packaging', move this to master once that is merged +# Pinned to a commit, and moving a pin is a commit of its own. SYSTEM, so that the warnings of this +# project do not apply to the headers of the libraries. # include(FetchContent) FetchContent_Declare(anyhttp GIT_REPOSITORY https://github.com/pgit/anyhttp.git - GIT_TAG af39b5c71490978d545c1b47ee8e149f9a7082c9 + GIT_TAG cc3faee4fb351a56bbca48db7cfcac7585e59e8b SYSTEM) FetchContent_Declare(c_ares_async GIT_REPOSITORY https://github.com/pgit/c-ares-async.git - GIT_TAG 2fb55f687f210d3da816e24aab9cefb61d6b4673 + GIT_TAG 6d46c6c8e65231f23a250bcec5dd00b3a1d9ba34 + SYSTEM) + +# +# Both libraries pin these as well, to the same commits for now. The first declaration wins, so these +# are the ones both of them get, and here they are SYSTEM, too. +# +FetchContent_Declare(capy + GIT_REPOSITORY https://github.com/cppalliance/capy.git + GIT_TAG 24c54af2cf967ccbf8d3809af4bcd9c8f61f553b + EXCLUDE_FROM_ALL SYSTEM) +FetchContent_Declare(corosio + GIT_REPOSITORY https://github.com/cppalliance/corosio.git + GIT_TAG 41d0de18b6e4fa1b397c38a866f2daf52c6c36b5 + EXCLUDE_FROM_ALL + SYSTEM) + FetchContent_MakeAvailable(anyhttp c_ares_async) +# Only for the command line. Neither library needs this component, so neither asks for it here. +find_package(Boost 1.88 REQUIRED COMPONENTS program_options) + # -------------------------------------------------------------------------------------------------- -add_executable(crawler src/main.cpp) -target_link_libraries(crawler PRIVATE anyhttp::anyhttp c_ares_async::c_ares_async) -target_compile_options(crawler PRIVATE -Wall -Wextra -Werror) +# +# The program in a library of its own, for the tests to link. The warnings are not for the sources +# from c-ares-async that the tests borrow, see test/CMakeLists.txt. +# +set(http_probe_warnings -Wall -Wextra -Werror -Wno-missing-designated-field-initializers) + +add_library(http_probe_core STATIC src/output.cpp src/prober.cpp src/routes.cpp) +target_include_directories(http_probe_core PUBLIC src) +target_link_libraries(http_probe_core PUBLIC anyhttp::anyhttp c_ares_async::c_ares_async) +target_compile_options(http_probe_core PRIVATE ${http_probe_warnings}) + +add_executable(http-probe src/main.cpp) +target_link_libraries(http-probe PRIVATE http_probe_core Boost::program_options) +target_compile_options(http-probe PRIVATE ${http_probe_warnings}) enable_testing() -add_test(NAME crawler_starts COMMAND crawler) +add_subdirectory(test) diff --git a/CMakePresets.json b/CMakePresets.json index d859f73..7d4fddc 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -18,30 +18,15 @@ "CMAKE_EXPORT_COMPILE_COMMANDS": "ON" } }, - { - "name": "dev", - "displayName": "Debug, against sibling checkouts", - "description": "What the crawler is developed with: the two libraries as they are in their checkouts next to this one, whatever the pins say. Keep this tree in build/, clangd reads its compile_commands.json", - "inherits": "base", - "binaryDir": "${sourceDir}/build", - "cacheVariables": { - "FETCHCONTENT_SOURCE_DIR_ANYHTTP": "${sourceDir}/../anyhttp", - "FETCHCONTENT_SOURCE_DIR_C_ARES_ASYNC": "${sourceDir}/../c-ares-async" - } - }, { "name": "pinned", "displayName": "Debug, from the pins", - "description": "The libraries at the commits CMakeLists.txt pins, fetched from GitHub: what CI builds", + "description": "The libraries at the commits CMakeLists.txt pins, fetched from GitHub. Keep this tree in build/, clangd reads its compile_commands.json", "inherits": "base", - "binaryDir": "${sourceDir}/build-pinned" + "binaryDir": "${sourceDir}/build" } ], "buildPresets": [ - { - "name": "dev", - "configurePreset": "dev" - }, { "name": "pinned", "configurePreset": "pinned" @@ -60,11 +45,6 @@ "noTestsAction": "error" } }, - { - "name": "dev", - "configurePreset": "dev", - "inherits": "base" - }, { "name": "pinned", "configurePreset": "pinned", diff --git a/README.md b/README.md index c35cf33..761ac36 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,97 @@ -# crawler +# http-probe -A simple web crawler on [anyhttp](https://github.com/pgit/anyhttp) (HTTP/1.1, HTTP/2 and HTTP/3) and +Requests hosts the way their HTTPS records say, on [anyhttp](https://github.com/pgit/anyhttp) (HTTP) and [c-ares-async](https://github.com/pgit/c-ares-async) (DNS). -**It does not crawl anything yet.** The `crawler` program sets up both libraries on one event loop -and exits, which shows that they build and link into one process. - > [!WARNING] > This repository is purely experimental. It is not meant for production use, and anything in it > can change or break at any time, without notice. +## What it does + +```sh +http-probe --dns-server 1.1.1.1 cloudflare.com +http-probe --dns-server 1.1.1.1 --file hostnames.txt +http-probe --dot --dns-server one.one.one.one cloudflare.com +``` + +It looks up the HTTPS records of each host in DNS ([RFC 9460](https://www.rfc-editor.org/rfc/rfc9460)), +and requests `https:///` once, over TLS, in the way that the records say. It prints the status, +the route, the IP address it connected to, the time it took and the header fields of the response, +along with the host's records. A redirect is reported and not followed, and nothing else on the host +is requested. + +There is output for every host but those that do not exist, which `--all` adds. The counts at the end +look like this, on stderr: + +``` +190 answered, 1 failed, 3 out of reach, 12 not found, 2 lookups failed +``` + +| The host | Outcome | +|-----------------------------------------------|---------------------------------------------| +| has a record on its own name | answered or failed, by the protocols and the port of the record | +| has no record, or none that is compatible | answered or failed, over TCP on port 443, HTTP/2 or HTTP/1.1 as the server likes | +| has records for other targets only, or an alias to another name | out of reach, not connected to: anyhttp can't connect to one name and verify another | +| has a record that says the service is not available | out of reach, not connected to | +| does not exist | not found, not connected to | +| gets no answer from DNS, or has no address | lookup failed, so there is no telling, not connected to | +| has no record that offers `h3`, with `--h3` | without h3, not connected to | + +Aliases are followed, a record has to be compatible, and the lowest priority goes first. Within a +record, HTTP/3 goes first, and then TCP on the same port: HTTP/2 if the record has `h2`, otherwise +HTTP/1.1, which is in every record that does not say `no-default-alpn`. The route in the output is +what the server went for, which may be HTTP/1.1 where HTTP/2 was offered. + +**With `--h3`,** only hosts with a compatible record that has `h3` in its `alpn` go ahead, on whichever +target, and they are treated as above, so that one can still end up out of reach, or going over TCP +when HTTP/3 can not be connected to. The others are left out after their HTTPS lookup, and printed +only with `--all`. The counts at the end then add how many there were, as `5 without h3`. + +**One request means one.** The next route is for when connecting failed. Once a request has gone out, +the host has been asked, so that a timeout or a response cut short ends it there, as a failure. + +The address comes from c-ares, so `--dns-server` applies to it, and `/etc/hosts` as well. The address +hints in a record do not. Addresses are tried one at a time, IPv6 first and otherwise in the order +c-ares sorts them, within the timeout of the route, until one of them can be connected to. So an IPv6 +address that does not answer at all takes the whole timeout of the route, and IPv4 does not get a turn. +A failure says which address it was on. Certificates are always verified, against the host's name. + +**To see what is going on,** `-v` logs to stderr: the requests and responses from anyhttp, the queries +from c-ares and the steps in between. `-vv` adds all of QUIC. Hosts run at the same time mix in the +log, so `--concurrency 1` helps. Colours are for terminals only unless there is `--color`, except that +anyhttp puts them into some of its log messages regardless. + +**Use `--dns-server`.** The resolver in the devcontainer answers with no data for HTTPS records, even +where they exist, which looks just like no host having any. It takes addresses and host names, with a +port behind either. A name is looked up in the clear with the servers of the system, whatever the +queries proper go out as, and only its first address becomes a server. + +**`--dot` asks over DNS over TLS** ([RFC 7858](https://www.rfc-editor.org/rfc/rfc7858)), on port 853 +unless the server says otherwise, for the records and the addresses alike. The certificate of the +server needs a name to be checked against. That is the host name in `--dns-server`, and with an +address, or several names, `--tls-hostname` has to give it. `--tls-no-verify` skips the check, which +is for testing and has nothing to do with the certificates of the hosts. A failed handshake makes the +hosts lookup failures, and says why in the log. `--ca-file` is for the hosts as well: the certificate +of the DNS server goes by the trust store of OpenSSL, which `SSL_CERT_FILE` can change. + +**A response goes to memory as a whole,** as anyhttp has no way of getting at it piece by piece with a +GET yet. So only `--timeout-ms` limits how large one can get. + +To see it work without going anywhere, `demo_site` runs a site and a DNS server on loopback, and +prints the command to request it with: + +```sh +./build/test/demo_site +``` + + ## Build C++23 and CMake 3.30 or later. The devcontainer in [.devcontainer](.devcontainer) has the compilers -and every dependency that the two libraries need. There are two presets: - -| Preset | Tree | Libraries come from | -|----------|-----------------|-----------------------------------------------------------------| -| `dev` | `build/` | the checkouts next to this one, as they are | -| `pinned` | `build-pinned/` | GitHub, at the commits pinned in [CMakeLists.txt](CMakeLists.txt); what CI builds | +and every dependency that the two libraries need. The libraries themselves are fetched from GitHub, +at the commits pinned in [CMakeLists.txt](CMakeLists.txt): ```sh cmake --preset pinned @@ -27,4 +99,4 @@ cmake --build --preset pinned ctest --preset pinned ``` -Both presets use clang, libc++, OpenSSL and the Asio runtime. +The tree is `build/`, with clang, libc++, OpenSSL and Capy and Corosio as the runtime. diff --git a/example/hostnames.txt b/example/hostnames.txt new file mode 100644 index 0000000..80422eb --- /dev/null +++ b/example/hostnames.txt @@ -0,0 +1,999 @@ +fonts.googleapis.com +facebook.com +twitter.com +google.com +youtube.com +s.w.org +instagram.com +googletagmanager.com +linkedin.com +ajax.googleapis.com +plus.google.com +gmpg.org +pinterest.com +fonts.gstatic.com +wordpress.org +en.wikipedia.org +youtu.be +maps.google.com +itunes.apple.com +github.com +bit.ly +play.google.com +goo.gl +docs.google.com +cdnjs.cloudflare.com +vimeo.com +support.google.com +google-analytics.com +maps.googleapis.com +flickr.com +vk.com +t.co +reddit.com +amazon.com +medium.com +sites.google.com +drive.google.com +creativecommons.org +microsoft.com +developers.google.com +adobe.com +soundcloud.com +theguardian.com +apis.google.com +ec.europa.eu +lh3.googleusercontent.com +chrome.google.com +cloudflare.com +nytimes.com +maxcdn.bootstrapcdn.com +support.microsoft.com +blogger.com +forbes.com +s3.amazonaws.com +code.jquery.com +dropbox.com +translate.google.com +paypal.com +apps.apple.com +tinyurl.com +etsy.com +theatlantic.com +m.facebook.com +archive.org +amzn.to +cnn.com +policies.google.com +commons.wikimedia.org +issuu.com +i.imgur.com +wordpress.com +wp.me +businessinsider.com +yelp.com +mail.google.com +support.apple.com +t.me +apple.com +washingtonpost.com +bbc.com +gstatic.com +imgur.com +amazon.de +bbc.co.uk +googleads.g.doubleclick.net +mozilla.org +eventbrite.com +slideshare.net +w3.org +forms.gle +platform.twitter.com +accounts.google.com +telegraph.co.uk +messenger.com +web.archive.org +secure.gravatar.com +usatoday.com +huffingtonpost.com +stackoverflow.com +fb.com +npr.org +techcrunch.com +wired.com +eepurl.com +reuters.com +arxiv.org +i0.wp.com +finance.yahoo.com +player.vimeo.com +amazon.fr +developer.mozilla.org +tumblr.com +cnbc.com +booking.com +zoom.us +tools.google.com +wa.me +paypal.me +slack.com +adwords.google.com +independent.co.uk +discord.gg +open.spotify.com +tripadvisor.com +gsuite.google.com +behance.net +thinkwithgoogle.com +who.int +cloud.google.com +calendar.google.com +ibm.com +analytics.google.com +lh5.googleusercontent.com +docs.microsoft.com +groups.google.com +mixcloud.com +google.co.in +latimes.com +gist.github.com +download.macromedia.com +dailymail.co.uk +myaccount.google.com +line.me +unsplash.com +addons.mozilla.org +msdn.microsoft.com +storage.googleapis.com +pagead2.googlesyndication.com +cdn.jsdelivr.net +g.co +meetup.com +bloomberg.com +books.google.com +vox.com +imdb.com +arstechnica.com +support.mozilla.org +amazon.co.uk +statista.com +google.ca +nbcnews.com +steemit.com +upload.wikimedia.org +aws.amazon.com +amazon.co.jp +w3schools.com +weibo.com +twitch.tv +moz.com +drupal.org +nypost.com +fr.wikipedia.org +ebay.com +sourceforge.net +godaddy.com +ncbi.nlm.nih.gov +wsj.com +thenextweb.com +cdc.gov +photos.google.com +trends.google.com +opera.com +ftc.gov +j.mp +about.me +gov.uk +i.ytimg.com +squareup.com +google.co.uk +buzzfeed.com +developer.apple.com +pixabay.com +quora.com +cnet.com +google.com.au +myspace.com +theglobeandmail.com +shopify.com +whitehouse.gov +marketingplatform.google.com +kickstarter.com +freelancer.com +evernote.com +gmail.com +static.wixstatic.com +fortune.com +wix.com +dailymotion.com +goodreads.com +podcasts.google.com +google.co.jp +support.cloudflare.com +surveymonkey.com +cbsnews.com +entrepreneur.com +bing.com +engadget.com +lh4.googleusercontent.com +tandfonline.com +cargocollective.com +join.slack.com +api.whatsapp.com +shutterstock.com +pbs.twimg.com +blog.google +ads.google.com +time.com +lifehacker.com +patreon.com +scribd.com +lh6.googleusercontent.com +feeds.feedburner.com +trello.com +ted.com +helpx.adobe.com +researchgate.net +fda.gov +instructables.com +google.de +digitalocean.com +dribbble.com +ssl.gstatic.com +variety.com +indiegogo.com +salesforce.com +uk.linkedin.com +nasa.gov +codepen.io +newyorker.com +abcnews.go.com +snapchat.com +googleadservices.com +sciencedirect.com +ft.com +nature.com +gofundme.com +500px.com +qz.com +dl.dropboxusercontent.com +store.steampowered.com +webmasters.googleblog.com +podcasts.apple.com +oracle.com +m.youtube.com +gartner.com +web.facebook.com +blogs.msdn.com +mailchi.mp +digg.com +skype.com +periscope.tv +google.com.br +googleblog.blogspot.com +de.wikipedia.org +search.google.com +npmjs.com +motherboard.vice.com +aboutads.info +pexels.com +sxsw.com +fiverr.com +code.google.com +cbc.ca +venturebeat.com +hangouts.google.com +mashable.com +upwork.com +bitly.com +music.apple.com +loc.gov +news.google.com +spotify.com +netflix.com +hbr.org +pbs.org +stumbleupon.com +walmart.com +foursquare.com +amazon.ca +tiny.cc +services.google.com +xkcd.com +target.com +developers.facebook.com +nginx.com +canva.com +s7.addthis.com +elmundo.es +i2.wp.com +gizmodo.com +expedia.com +picasaweb.google.com +money.cnn.com +timesofindia.indiatimes.com +yandex.ru +vice.com +de-de.facebook.com +pastebin.com +samsung.com +rt.com +ifttt.com +dev.mysql.com +giphy.com +use.fontawesome.com +houzz.com +developer.android.com +wikipedia.org +speakerdeck.com +productforums.google.com +digitaltrends.com +steamcommunity.com +canada.ca +networkadvertising.org +linktr.ee +calendly.com +lg.com +yahoo.com +last.fm +amazon.es +businesswire.com +deezer.com +dx.doi.org +udemy.com +edition.cnn.com +1.bp.blogspot.com +l.facebook.com +dl.dropbox.com +store.google.com +gumroad.com +azure.microsoft.com +i1.wp.com +ubuntu.com +google.ru +makeuseof.com +raw.githubusercontent.com +airbnb.com +bluehost.com +deviantart.com +pewresearch.org +google.cn +profiles.google.com +login.microsoftonline.com +google.nl +google.es +fastcompany.com +is.gd +telegram.me +mediafire.com +themeforest.net +openstreetmap.org +pewinternet.org +photos.app.goo.gl +buff.ly +events.google.com +link.springer.com +bizjournals.com +residentadvisor.net +golang.org +vogue.com +s3-eu-west-1.amazonaws.com +boardgamegeek.com +un.org +i.redd.it +ad.doubleclick.net +ow.ly +get.google.com +pt.slideshare.net +buffer.com +mozilla.com +chromium.org +patheos.com +ancestry.com +youtube-nocookie.com +barnesandnoble.com +amzn.com +hulu.com +greenpeace.org +spiegel.de +apps.facebook.com +census.gov +app.box.com +windows.microsoft.com +yandex.com +sketchfab.com +avvo.com +searchengineland.com +shareasale.com +abc.net.au +trustpilot.com +plaza.rakuten.co.jp +marriott.com +amazon.it +ca.linkedin.com +uber.com +coursera.org +adage.com +foxnews.com +flic.kr +developer.chrome.com +pl.wikipedia.org +change.org +spreaker.com +filezilla-project.org +flipboard.com +pcworld.com +news.yahoo.com +pitchfork.com +weforum.org +istockphoto.com +9to5mac.com +1drv.ms +uspto.gov +irs.gov +copyblogger.com +example.com +maps.google.co.jp +fonts.google.com +siteground.com +sony.net +ja.wikipedia.org +creativemarket.com +smithsonianmag.com +mobile.twitter.com +acm.org +accenture.com +git-scm.com +bbb.org +cdn.shopify.com +schema.org +psychologytoday.com +redcross.org +nodejs.org +europa.eu +ameblo.jp +gitlab.com +sciencemag.org +hollywoodreporter.com +freshbooks.com +heise.de +chicagotribune.com +epa.gov +anchor.fm +telegram.org +ikea.com +bloglovin.com +eventbrite.co.uk +googlewebmastercentral.blogspot.com +huffpost.com +weebly.com +producthunt.com +google.fr +click.linksynergy.com +vanityfair.com +diigo.com +about.fb.com +ouest-france.fr +namecheap.com +codecanyon.net +appstore.com +tools.ietf.org +rockpapershotgun.com +columbia.edu +constantcontact.com +webmd.com +eur-lex.europa.eu +aliexpress.com +business.facebook.com +material.io +neilpatel.com +discordapp.com +m.me +design.google +francetvinfo.fr +slashgear.com +4shared.com +python.org +amazon.com.au +storify.com +elegantthemes.com +europe1.fr +pwc.com +activecampaign.com +freepik.com +ravelry.com +apnews.com +propublica.org +smugmug.com +inc.com +google.it +zazzle.com +dol.gov +scoop.it +bit.do +yadi.sk +eclipse.org +economist.com +xing.com +hbo.com +nationalgeographic.com +jetbrains.com +iconfinder.com +cafepress.com +strava.com +es.wikipedia.org +kiva.org +doi.org +boredpanda.com +justgiving.com +pinterest.ca +stats.wp.com +zdnet.com +sproutsocial.com +rebrand.ly +ranker.com +a.co +waze.com +nydailynews.com +marketwatch.com +mailchimp.com +livescience.com +flaticon.com +ilpost.it +connect.facebook.net +4.bp.blogspot.com +maps.google.co.nz +s0.wp.com +amazon.com.br +ustream.tv +lulu.com +socialmediatoday.com +market.android.com +rollingstone.com +journals.sagepub.com +zillow.com +xinhuanet.com +affiliate-program.amazon.com +politico.com +pnas.org +laughingsquid.com +access.redhat.com +s.ytimg.com +it.linkedin.com +dashlane.com +copyright.gov +news.bbc.co.uk +mentalfloss.com +zeit.de +google.cz +artsandculture.google.com +startnext.com +launchpad.net +lemonde.fr +sciencedaily.com +edx.org +smashwords.com +philips.co.uk +smile.amazon.com +intel.com +livestream.com +brookings.edu +thehill.com +popsci.com +blog.us.playstation.com +fb.me +academic.oup.com +php.net +wetransfer.com +blog.livedoor.jp +amazon.in +families.google.com +obsproject.com +t.qq.com +android.com +blogs.windows.com +hostgator.com +google.gr +theverge.com +aub.edu.lb +digiday.com +gimp.org +treasury.gov +geocities.com +healthline.com +3.bp.blogspot.com +coinbase.com +gnu.org +abc.com +aljazeera.com +getresponse.com +dev.to +thesun.co.uk +lynda.com +sellfy.com +prnt.sc +help.ubuntu.com +whatsapp.com +www8.hp.com +notion.so +vmware.com +kraken.com +vine.co +marthastewart.com +discord.me +dailycaller.com +mega.nz +media.giphy.com +flavors.me +blip.tv +prntscr.com +journals.plos.org +gravatar.com +discogs.com +europarl.europa.eu +redhat.com +bugs.chromium.org +vanmiubeauty.com +metro.co.uk +google.ie +relapse.com +earth.google.com +bandcamp.com +br.linkedin.com +gopro.com +t.ly +get.adobe.com +news.mit.edu +billboard.com +britannica.com +onlinelibrary.wiley.com +techsmith.com +sfgate.com +redbubble.com +kstatic.googleusercontent.com +repubblica.it +disqus.com +khanacademy.org +skfb.ly +androidauthority.com +accessify.com +tensorflow.org +chrisjdavis.org +box.net +poetryfoundation.org +law.cornell.edu +penguinrandomhouse.com +refinery29.com +starwars.com +fr.linkedin.com +video.google.com +prnewswire.com +springer.com +zalo.me +reverbnation.com +getpocket.com +webdesign.about.com +goo.gle +note.mu +thetimes.co.uk +fao.org +bol.com +yoursite.com +sophos.com +automattic.com +podbean.com +sublimetext.com +archives.gov +vizio.com +thelancet.com +guardian.co.uk +news.nationalgeographic.com +tunein.com +stitcher.com +coinmarketcap.com +amnestyusa.org +docs.wixstatic.com +cambridge.org +validator.w3.org +asus.com +problogger.net +worldbank.org +ticketportal.cz +dw.com +static.googleusercontent.com +www-01.ibm.com +business.google.com +es.linkedin.com +songkick.com +adweek.com +lifehack.org +apple.co +sutterhealth.org +globalnews.ca +breitbart.com +thumbtack.com +eff.org +snopes.com +crowdrise.com +hkrsa.asia +metmuseum.org +pixlr.com +institutvajrayogini.fr +rhtlbn.mosgorcredit.ru +itchyfeetonthecheap.com +reacts.ru +db.tt +udacity.com +we.tl +blogtalkradio.com +audacityteam.org +youcaring.com +hp.com +0.gravatar.com +health.harvard.edu +meta.wikimedia.org +google.ch +nhs.uk +mixi.jp +symfony.com +business.linkedin.com +mtv.com +1.gravatar.com +cyber.law.harvard.edu +bitcointalk.org +esa.int +slate.com +oecd.org +raspberrypi.org +sfexaminer.com +warriorforum.com +g.page +bhphotovideo.com +axios.com +pinterest.co.uk +unesco.org +msn.com +code.visualstudio.com +netbeans.org +blockchain.info +untappd.com +espn.com +cbs.com +michigan.gov +oprah.com +ok.ru +mhlw.go.jp +newegg.com +2.bp.blogspot.com +pipes.yahoo.com +themarthablog.com +yummly.com +bitpay.com +buymeacoffee.com +deepl.com +blogs.scientificamerican.com +audible.com +bitbucket.org +it.wikipedia.org +in.linkedin.com +cse.google.com +use.typekit.net +ru.wikipedia.org +faz.net +adf.ly +cancerresearchuk.org +moma.org +bigthink.com +bandsintown.com +addthis.com +support.office.com +upi.com +zen.yandex.ru +collegehumor.com +lefigaro.fr +gplus.to +themify.me +rtve.es +nba.com +myfitnesspal.com +chris.pirillo.com +eonline.com +science.sciencemag.org +ietf.org +ko-fi.com +otto.de +tkqlhce.com +unity3d.com +blog.naver.com +scratch.mit.edu +freewebs.com +drift.com +postmates.com +geek.com +geocities.jp +beian.gov.cn +meet.google.com +flattr.com +webroot.com +thoughtcatalog.com +google.pl +seroundtable.com +checkpoint.com +form.jotform.com +xbox.com +infusionsoft.com +g1.globo.com +capterra.com +ea.com +hawaii.edu +stock.adobe.com +osha.gov +buzzfeednews.com +jstor.org +adssettings.google.com +iheart.com +en-gb.facebook.com +foodnetwork.com +eventim.de +economictimes.indiatimes.com +presseportal.de +google.se +skillshare.com +puu.sh +examiner.com +stripe.com +firstdata.com +en.advertisercommunity.com +download.microsoft.com +s-media-cache-ak0.pinimg.com +ctt.ec +cia.gov +docker.com +homedepot.com +humblebundle.com +chiark.greenend.org.uk +pandora.com +google.co.za +payhip.com +thingiverse.com +inkscape.org +adobe.ly +css-tricks.com +ja-jp.facebook.com +jamanetwork.com +maps.gstatic.com +wattpad.com +chase.com +cisco.com +bmj.com +louvre.fr +aclu.org +squarespace.com +gum.co +idealo.de +imore.com +rakuten.com +fbi.gov +united.com +google.pt +event.on24.com +nejm.org +colorado.edu +franchising.com +logitech.com +redbull.com +tf1.fr +keep.google.com +hostinger.com +nbc.com +stats.g.doubleclick.net +ign.com +blog.hubspot.com +1.envato.market +accessdata.fda.gov +google.be +help.apple.com +technorati.com +pond5.com +ucl.ac.uk +a2hosting.com +amzn.asia +nicovideo.jp +indiewire.com +mp.weixin.qq.com +codeproject.com +1.usa.gov +gleam.io +space.com +gitter.im +lh5.ggpht.com +leparisien.fr +kotaku.com +img.youtube.com +huffingtonpost.co.uk +agoda.com +flow.microsoft.com +chronicle.com +createspace.com +lh3.ggpht.com +verizon.com +teespring.com +data.worldbank.org +blog.goo.ne.jp +kobo.com +globalsign.com +money.yandex.ru +animoto.com +google.co.nz +privacy.microsoft.com +online.wsj.com +patft.uspto.gov +rottentomatoes.com +cell.com +de.linkedin.com +health.com +geni.us +news.harvard.edu +lesechos.fr +fas.org +smashingmagazine.com +7-zip.org +realvnc.com +news.discovery.com +mlb.com +autodesk.com +popularmechanics.com +forms.office.com +rdio.com +sendspace.com +synology.com +angelfire.com +france24.com +us.battle.net +stjude.org +denverpost.com +japantimes.co.jp +stocktwits.com +blogs.adobe.com +envato.com +airtable.com +ticketmaster.com +ocw.mit.edu +whc.unesco.org +neh.gov +purl.org +nfl.com +monster.com +codex.wordpress.org +dafont.com +tesla.com +lmgtfy.com +olympic.org +technet.microsoft.com +baidu.com +desktop.github.com +overcast.fm +allmusic.com +cdbaby.com +vsco.co +nvidia.com +stadt-bremerhaven.de +faa.gov +blogs.yahoo.co.jp +bild.de +lenovo.com +funnyordie.com +ssl.google-analytics.com +abebooks.com +spectrum.ieee.org +thinkgeek.com +snip.ly +pixiv.net +buzzsprout.com +patents.google.com +orcid.org +cdn.ampproject.org +blog.feedspot.com +vr.google.com +opinionator.blogs.nytimes.com +ericsson.com diff --git a/src/main.cpp b/src/main.cpp index 50f2f71..8e6e58e 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,20 +1,309 @@ // -// Does not crawl anything yet. It sets up both libraries on one event loop, without connecting to -// or looking up anything, to show that they build and link into one process. +// http-probe [options] HOST... // -#include -#include +#include "output.hpp" +#include "prober.hpp" -#include +#include +#include +#include +#include #include -int main() +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace po = boost::program_options; + +namespace +{ + +constexpr std::string_view usage = R"(usage: http-probe [options] HOST... + http-probe [options] --file LIST +)"; + +constexpr std::string_view about = + "Looks up the HTTPS records of each host (RFC 9460), and requests https://HOST/ once, the way " + "they say or over TCP on port 443 if they don't, printing the status and header fields of the " + "response. Redirects are reported and not followed, and nothing else on the host is requested."; + +constexpr std::string_view exit_code = + "The exit code is 0 if every host that was connected to answered, whatever the status, and " + "every lookup got an answer."; + +struct Arguments { - spdlog::set_level(spdlog::level::warn); // anyhttp logs through the default logger, info is noisy + http_probe::Config config; + std::vector files; + bool help = false; + bool all = false; + bool color = false; + int verbosity = 0; +}; + +// +// The numbers are read as signed ones, because lexical_cast takes -1 for the largest value of an +// unsigned type, which would pass as a choice. +// +int not_negative(std::string_view option, int value) +{ + if (value < 0) + throw std::invalid_argument(std::format("{} can't be negative", option)); + return value; +} + +// +// How wide to print the help. Either stream will do, as they are on the same terminal, and with +// 'http-probe --help | less' only stderr still is one. +// +unsigned help_columns() +{ + for (const int fd : {STDOUT_FILENO, STDERR_FILENO}) + { + winsize ws{}; + if (::ioctl(fd, TIOCGWINSZ, &ws) == 0 && ws.ws_col >= 40) + return ws.ws_col; + } + return po::options_description::m_default_line_length; +} + +// +// A paragraph broken at its spaces into lines that fit. +// +std::string wrap(std::string_view paragraph, unsigned columns) +{ + std::string wrapped; + std::size_t line = 0; + for (const auto range : paragraph | std::views::split(' ')) + { + const std::string_view word{range}; + if (line > 0 && line + 1 + word.size() > columns) + { + wrapped += '\n'; + line = 0; + } + else if (line > 0) + { + wrapped += ' '; + ++line; + } + wrapped += word; + line += word.size(); + } + return wrapped; +} + +// +// The options, stored into 'arguments', which has to outlive parsing them. +// +po::options_description describe_options(Arguments& arguments) +{ + auto& config = arguments.config; + + const auto columns = help_columns(); + po::options_description desc("Options", columns, columns / 2); + auto opts = desc.add_options(); + opts("help,h", po::bool_switch(&arguments.help), "produce this help message"); + opts("host", po::value(&config.hosts)->value_name("HOST"), + "a host to request, the same as giving HOST by itself"); + opts("file", po::value(&arguments.files)->value_name("LIST"), + "read the hosts from a file, one per line, as names or as URLs"); + opts("dns-server", po::value(&config.dns_server)->value_name("SERVER"), + "the DNS servers to ask, comma separated, by address or by name, like 192.0.2.1, " + "192.0.2.1:5353 or dns.example.com; default: the system's, which may well not answer for " + "HTTPS records"); + opts("dot", po::bool_switch(&config.dns_tls.enabled), + "ask the DNS servers over TLS (RFC 7858), which moves them to port 853 unless they say " + "otherwise"); + opts("tls-hostname", po::value(&config.dns_tls.hostname)->value_name("NAME"), + "with --dot, the name to use for SNI and to verify the certificate of the DNS server " + "against, like one.one.one.one; default: the host name in --dns-server"); + opts("tls-no-verify", po::value()->zero_tokens()->notifier([&config](bool) { + config.dns_tls.verify = false; + }), + "with --dot, accept any certificate from the DNS server (testing only); does not apply to " + "the hosts"); + opts("ca-file", po::value(&config.tls_ca_file)->value_name("PEM"), + "the CA certificates to verify the hosts with; default: the system's. Not for --dot, which " + "goes by the trust store of OpenSSL, as SSL_CERT_FILE can say"); + opts("concurrency", + po::value() + ->value_name("N") + ->default_value(static_cast(config.concurrency)) + ->notifier( + [&config](int value) { config.concurrency = not_negative("--concurrency", value); }), + "how many hosts to work on at the same time"); + opts("timeout-ms", + po::value() + ->value_name("N") + ->default_value(static_cast(config.timeout.count())) + ->notifier([&config](int value) { + config.timeout = std::chrono::milliseconds(not_negative("--timeout-ms", value)); + }), + "how long one request may take"); + opts("h3", po::bool_switch(&config.h3_only), + "connect only to the hosts with an HTTPS record that offers HTTP/3"); + opts("all", po::bool_switch(&arguments.all), + "also print the hosts that do not exist, and those that --h3 leaves out"); + opts("color", po::bool_switch(&arguments.color), + "colour the output even where it does not go to a terminal, e.g. into 'less -R'"); + opts("verbose,v", po::value>()->zero_tokens()->composing(), + "log to stderr what is going on, the requests and responses of anyhttp and the queries of " + "c-ares among it; -vv for all of QUIC as well, which is a lot. To make sense of it, use " + "--concurrency 1."); + return desc; +} + +void print_usage(std::ostream& out, const po::options_description& options) +{ + const auto columns = help_columns(); + out << usage << '\n' + << wrap(about, columns) << "\n\n" + << options << '\n' + << wrap(exit_code, columns) << '\n'; +} + +void parse_arguments(int argc, char* argv[], const po::options_description& options, + Arguments& arguments) +{ + try + { + po::positional_options_description positional; + positional.add("host", -1); + + const auto parsed = + po::command_line_parser(argc, argv).options(options).positional(positional).run(); + + po::variables_map vm; + po::store(parsed, vm); + po::notify(vm); + + // 'verbose' takes no argument, so its parsed value is always empty: count occurrences + arguments.verbosity = static_cast(std::ranges::count_if( + parsed.options, [](const po::option& option) { return option.string_key == "verbose"; })); + } + catch (const po::error& ex) + { + throw std::invalid_argument(ex.what()); + } + + if (arguments.help) + return; + + for (const auto& file : arguments.files) + { + std::ifstream list{file}; + if (!list) + throw std::invalid_argument(std::format("can't read '{}'", file)); + std::vector skipped; + arguments.config.hosts.append_range(http_probe::read_hosts(list, skipped)); + for (const auto& line : skipped) + std::println(stderr, "http-probe: skipping '{}': not a host name", line); + } +} + +// +// Everywhere that anyhttp and c-ares log to, as both use the default logger. On stderr, so that the +// results can go somewhere by themselves. +// +void set_up_logging(int verbosity, bool color) +{ + spdlog::set_default_logger(spdlog::stderr_color_mt( + "http-probe", color ? spdlog::color_mode::always : spdlog::color_mode::automatic)); + spdlog::set_pattern("%H:%M:%S.%e %^%l%$ %v"); + spdlog::set_level(verbosity >= 2 ? spdlog::level::trace + : verbosity == 1 ? spdlog::level::debug + : spdlog::level::err); +} + +} // namespace + +int main(int argc, char* argv[]) +{ + using Outcome = http_probe::Result::Outcome; + + http_probe::Counts counts; + std::size_t with_records = 0; + + Arguments arguments; + const auto options = describe_options(arguments); + + try + { + parse_arguments(argc, argv, options, arguments); + if (arguments.help) + { + print_usage(std::cout, options); + return 0; + } + + auto& [config, files, help, all, color, verbosity] = arguments; + set_up_logging(verbosity, color); + + const bool own_dns_server = !config.dns_server.empty(); + const bool color_results = color || isatty(STDOUT_FILENO); + const bool color_summary = color || isatty(STDERR_FILENO); + + const auto report = [&](const http_probe::Result& result) { + ++counts[result.outcome]; + if (!result.records.empty() || result.aliased) + ++with_records; + if (all || (result.outcome != Outcome::not_found && result.outcome != Outcome::no_h3)) + { + std::print("{}", http_probe::format_result(result, color_results)); + std::fflush(stdout); // a run can take a while, and it is worth watching + } + }; + + boost::corosio::io_context context; + http_probe::Prober prober(context.get_executor(), std::move(config), report); + std::exception_ptr failed; + + // + // What a handler throws ends the process, so this one leaves it for after run(). + // + boost::capy::run_async( + context.get_executor(), []() noexcept {}, + [&](std::exception_ptr ex) noexcept { + failed = ex; + context.stop(); + })(prober.run()); + context.run(); + if (failed) + std::rethrow_exception(failed); + + std::println(stderr, "{}", http_probe::format_summary(counts, color_summary)); + if (with_records == 0 && !own_dns_server) + std::println(stderr, "http-probe: none of the hosts has an HTTPS record. Not every " + "resolver answers for them, so try --dns-server."); + } + catch (const std::invalid_argument& ex) + { + std::println(stderr, "http-probe: {}\n", ex.what()); + print_usage(std::cerr, options); + return 2; + } + catch (const std::exception& ex) + { + std::println(stderr, "http-probe: {}", ex.what()); + return 1; + } - boost::asio::io_context context; - anyhttp::client::Client client(context.get_executor(), anyhttp::client::Config{}); - c_ares_async::resolver resolver(context.get_executor()); - context.run(); + return counts[Outcome::failed] == 0 && counts[Outcome::lookup_failed] == 0 ? 0 : 1; } diff --git a/src/output.cpp b/src/output.cpp new file mode 100644 index 0000000..d332696 --- /dev/null +++ b/src/output.cpp @@ -0,0 +1,149 @@ +#include "output.hpp" + +#include +#include +#include + +namespace http_probe +{ + +namespace +{ + +// ================================================================================================= + +namespace color_of +{ +constexpr std::string_view name = "33"; +constexpr std::string_view v4 = "35"; +constexpr std::string_view v6 = "34"; +constexpr std::string_view good = "32"; +constexpr std::string_view moved = "36"; +constexpr std::string_view bad = "31"; +constexpr std::string_view field = "1;34"; // as anyhttp logs header fields +constexpr std::string_view note = "2"; +} // namespace color_of + +std::string colored(std::string_view text, std::string_view code, bool color) +{ + return color ? std::format("\x1b[{}m{}\x1b[0m", code, text) : std::string(text); +} + +std::string status_text(int status, bool color) +{ + const auto code = status >= 200 && status < 300 ? color_of::good + : status >= 300 && status < 400 ? color_of::moved + : color_of::bad; + return colored(std::to_string(status), code, color); +} + +std::string address_text(std::string_view address, bool color) +{ + const bool v6 = address.find(':') != address.npos; + return colored(address, v6 ? color_of::v6 : color_of::v4, color); +} + +// +// The value of an address hint: the addresses have a colour, the commas between them do not. +// +std::string address_list_text(std::string_view list, bool color) +{ + std::string text; + for (const auto address : std::views::split(list, ',')) + { + if (!text.empty()) + text += ','; + text += address_text(std::string_view(address), color); + } + return text; +} + +// +// to_text() with colours: the target, and the values of the address hints. +// +std::string record_text(const c_ares_async::https_record& record, bool color) +{ + auto text = std::format("HTTPS {} {}", record.priority, + record.target.empty() || record.target == "." + ? "." + : colored(record.target, color_of::name, color)); + + for (std::string_view param : record.params) + { + if (param.starts_with("ipv4hint=") || param.starts_with("ipv6hint=")) + { + const auto value = param.find('=') + 1; + text += std::format(" {}{}", param.substr(0, value), + address_list_text(param.substr(value), color)); + } + else + text += std::format(" {}", param); + } + return text; +} + +// ================================================================================================= + +} // namespace + +std::string format_result(const Result& result, bool color) +{ + using Outcome = Result::Outcome; + + const auto host = colored(result.host, color_of::name, color); + std::string text; + switch (result.outcome) + { + case Outcome::answered: + text = std::format("{}: {} [{}] at {}, {} bytes in {}\n", host, + status_text(result.status, color), result.route, + address_text(result.address, color), result.body_size, result.duration); + break; + case Outcome::failed: + text = std::format( + "{}: {}{}: {}\n", host, colored("failed", color_of::bad, color), + result.address.empty() ? "" : std::format(" at {}", address_text(result.address, color)), + result.error); + break; + case Outcome::out_of_reach: + text = std::format("{}: {}\n", host, colored("out of reach", color_of::moved, color)); + break; + case Outcome::not_found: + text = std::format("{}: not found\n", host); + break; + case Outcome::lookup_failed: + text = std::format("{}: {}\n", host, colored("lookup failed", color_of::bad, color)); + break; + case Outcome::no_h3: + text = std::format("{}: no h3 record\n", host); + break; + } + + for (const auto& record : result.records) + text += std::format(" record: {}\n", record_text(record, color)); + for (const auto& note : result.notes) + text += colored(std::format(" note: {}", note), color_of::note, color) + "\n"; + for (const auto& [name, value] : result.fields) + text += std::format(" {}: {}\n", colored(name, color_of::field, color), value); + return text; +} + +std::string format_summary(Counts counts, bool color) +{ + using Outcome = Result::Outcome; + + const auto count = [&](Outcome outcome, std::string_view text, std::string_view code) { + const auto line = std::format("{} {}", counts[outcome], text); + return counts[outcome] ? colored(line, code, color) : line; + }; + return std::format( + "{}, {}, {}, {} not found, {}{}", count(Outcome::answered, "answered", color_of::good), + count(Outcome::failed, "failed", color_of::bad), + count(Outcome::out_of_reach, "out of reach", color_of::moved), counts[Outcome::not_found], + count(Outcome::lookup_failed, "lookups failed", color_of::bad), + counts[Outcome::no_h3] ? std::format(", {} without h3", counts[Outcome::no_h3]) : ""); +} + +// ================================================================================================= + +} // namespace http_probe diff --git a/src/output.hpp b/src/output.hpp new file mode 100644 index 0000000..851fb48 --- /dev/null +++ b/src/output.hpp @@ -0,0 +1,29 @@ +// +// What the program prints, in the colours of c-ares-async's example where it can: host names are +// yellow, and addresses come the way "ip -color address" gives them, IPv4 in magenta and IPv6 in +// blue. Without colour, the text is the same but for the escape sequences. +// +#pragma once + +#include "prober.hpp" + +#include +#include +#include + +namespace http_probe +{ + +// ================================================================================================= + +using Counts = std::map; + +/// All there is to say about one host, a line each and with a line break at the end. +std::string format_result(const Result& result, bool color); + +/// How it went overall, without a line break. +std::string format_summary(Counts counts, bool color); + +// ================================================================================================= + +} // namespace http_probe diff --git a/src/prober.cpp b/src/prober.cpp new file mode 100644 index 0000000..aa0d311 --- /dev/null +++ b/src/prober.cpp @@ -0,0 +1,521 @@ +#include "prober.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace http = boost::beast::http; + +namespace http_probe +{ + +namespace +{ + +// ================================================================================================= + +constexpr std::string_view user_agent = "http-probe/0.0.1 (+https://github.com/pgit/http-probe)"; + +constexpr int max_alias_chain = 8; + +// +// The resolver does not write anywhere by itself. Its messages go where those of anyhttp go, which +// is the default logger of spdlog. +// +void log_to_spdlog(c_ares_async::log_level level, std::string_view message) +{ + using enum c_ares_async::log_level; + switch (level) + { + case debug: + return spdlog::debug("{}", message); + case info: + return spdlog::info("{}", message); + case warning: + return spdlog::warn("{}", message); + case error: + return spdlog::error("{}", message); + } +} + +// ------------------------------------------------------------------------------------------------- + +// +// The entries of a list of DNS servers. +// +std::vector split_servers(std::string_view list) +{ + std::vector servers; + for (const auto part : std::views::split(list, ',')) + { + const std::string_view server(part); + if (const auto first = server.find_first_not_of(" \t"); first != server.npos) + servers.push_back(server.substr(first, server.find_last_not_of(" \t") - first + 1)); + } + return servers; +} + +// +// A server given by name, and the port behind it, colon included, if there is one. +// +struct ServerName +{ + std::string_view name; + std::string_view port; +}; + +// +// Tells the names in a list from what c-ares takes for a server, all of which is addresses: +// "192.0.2.1", "192.0.2.1:5353", "[::1]:5353", "fe80::1%eth0" or a dns:// URI. Whatever is neither +// goes to c-ares as well, for it to turn it down. +// +std::optional server_name(std::string_view server) +{ + const auto colon = server.find(':'); + const auto name = server.substr(0, colon); + const auto port = colon == server.npos ? std::string_view{} : server.substr(colon); + + const auto is_digit = [](unsigned char c) { return std::isdigit(c) != 0; }; + if (!is_host_name(name) || + (!port.empty() && (port.size() == 1 || !std::ranges::all_of(port.substr(1), is_digit)))) + return std::nullopt; + + return ServerName{name, port}; +} + +// +// A certificate needs a name to be checked against. A single name in the list of servers is that +// name, while an address is none, and of several names none is the one for all of them. Rather than +// quietly talking to whoever answers, this says so. +// +c_ares_async::tls_options complete_dns_tls(c_ares_async::tls_options tls, std::string_view servers) +{ + if (!tls.enabled) + { + if (!tls.hostname.empty() || !tls.verify) + throw std::invalid_argument("--tls-hostname and --tls-no-verify only apply with --dot"); + return tls; + } + + if (!tls.hostname.empty()) + return tls; + + std::set names; + for (const auto server : split_servers(servers)) + if (const auto named = server_name(server)) + names.insert(named->name); + + if (names.size() == 1) + tls.hostname = *names.begin(); + else if (tls.verify) + throw std::invalid_argument(std::format( + "{}, or --tls-no-verify to skip the check", + names.empty() ? "--dot needs a name to verify the certificate of the DNS " + "server against: a host name as --dns-server, or --tls-hostname" + : "--dns-server names several hosts, so --dot needs " + "--tls-hostname to verify their certificates against")); + return tls; +} + +// ================================================================================================= + +} // namespace + +bool is_host_name(std::string_view host) +{ + const auto is_name_char = [](unsigned char c) { + return std::isalnum(c) != 0 || c == '-' || c == '.' || c == '_'; + }; + const auto [not_an_address, address] = corosio::make_ip_address(host); + return !host.empty() && host.size() <= 253 && std::ranges::all_of(host, is_name_char) && + not_an_address; +} + +std::vector read_hosts(std::istream& list, std::vector& skipped) +{ + std::vector hosts; + for (std::string line; std::getline(list, line);) + { + std::string_view text = line; + while (!text.empty() && std::isspace(static_cast(text.front()))) + text.remove_prefix(1); + while (!text.empty() && std::isspace(static_cast(text.back()))) + text.remove_suffix(1); + if (text.empty() || text.starts_with('#')) + continue; + + std::string host(text); + if (text.find("://") != text.npos) + if (const auto url = boost::urls::parse_uri(text)) + host = url->host_address(); + + if (is_host_name(host)) + hosts.push_back(std::move(host)); + else + skipped.emplace_back(text); + } + return hosts; +} + +void prefer_ipv6(std::vector& addresses) +{ + std::ranges::stable_partition(addresses, &corosio::endpoint::is_v6); +} + +Prober::Prober(capy::any_executor executor, Config config, Observer observer) + : executor_(executor), config_(std::move(config)), observer_(std::move(observer)) +{ + if (config_.hosts.empty()) + throw std::invalid_argument("nothing to do: no host names given"); + if (config_.concurrency == 0) + throw std::invalid_argument("a concurrency of 0 would never get anywhere"); + + std::set seen; + for (const auto& host : config_.hosts) + { + if (!is_host_name(host)) + throw std::invalid_argument(std::format("'{}' is not a host name", host)); + if (auto name = normalize_host(host); seen.insert(name).second) + hosts_.push_back(std::move(name)); + } + + config_.dns_tls = complete_dns_tls(std::move(config_.dns_tls), config_.dns_server); +} + +// +// c-ares takes addresses for its servers and nothing else, so a name has to be looked up before +// there is anybody to ask, the regular way and in the clear, whatever the queries proper go out as. +// +// Only the first address of a name becomes a server. c-ares gives every query its tries once per +// server, with the timeout doubling each time around, so all of them would not be failover for free +// but make a host that nobody has an answer for take much longer. +// +capy::task Prober::dns_server_addresses() +{ + std::unique_ptr resolver; // only if there is a name to look up at all + std::string addresses; + + for (const auto server : split_servers(config_.dns_server)) + { + std::string address(server); + if (const auto named = server_name(server)) + { + if (!resolver) + { + resolver = std::make_unique(executor_); + resolver->set_logger(&log_to_spdlog); + if (!config_.dns_server_lookup.empty()) + if (const auto ec = resolver->set_servers(config_.dns_server_lookup)) + throw std::invalid_argument(ec.message()); + } + + spdlog::debug("looking up DNS server '{}'", named->name); + const auto [ec, endpoints] = co_await resolver->resolve(std::string(named->name), ""); + if (ec || endpoints.empty()) + throw std::runtime_error( + std::format("DNS server '{}': {}", named->name, ec ? ec.message() : "no addresses")); + + // + // An IPv6 address only needs its brackets if there is a port behind it. + // + const auto first = endpoints.front().address(); + address = named->port.empty() ? first.to_string() + : first.is_v6() ? std::format("[{}]{}", first.to_string(), named->port) + : std::format("{}{}", first.to_string(), named->port); + } + addresses += (addresses.empty() ? "" : ",") + address; + } + co_return addresses; +} + +capy::task<> Prober::run() +{ + const auto servers = co_await dns_server_addresses(); + + resolver_ = std::make_unique(executor_, config_.dns_tls); + resolver_->set_logger(&log_to_spdlog); + if (!servers.empty()) + if (const auto ec = resolver_->set_servers(servers)) + throw std::invalid_argument( + std::format("invalid DNS server '{}': {}", config_.dns_server, ec.message())); + + std::vector>> workers; + for (std::size_t i = 0; i < std::min(config_.concurrency, hosts_.size()); ++i) + workers.push_back(work()); + + std::ignore = co_await capy::when_all(std::move(workers)); + + resolver_.reset(); +} + +capy::task> Prober::work() +{ + while (next_host_ < hosts_.size()) + { + const auto& host = hosts_[next_host_++]; + observer_(co_await probe(host)); + } + co_return capy::results<>{}; +} + +// ------------------------------------------------------------------------------------------------- + +// +// SVCB resolution, section 3 of RFC 9460, up to the point where it would take addresses. A lookup +// that fails is not fatal: DNS is not protected here, and this is the choice section 3.1 leaves +// for that. A name without a record is the common case and not even worth a note. +// +capy::task Prober::resolve_service(const std::string& host) +{ + ServiceRecords found{.owner = host}; + std::set visited{found.owner}; + + for (int hop = 0; hop < max_alias_chain; ++hop) + { + // + // Outside of any timeout: the image has c-ares 1.34.5, and cancelling a lookup is what + // GHSA-6wfj-rwm7-3542 is about. c-ares has timeouts of its own. + // + spdlog::debug("{}: asking for the HTTPS records of '{}'", host, found.owner); + auto [ec, records] = co_await resolver_->resolve_https(found.owner); + if (ec) + { + if (ec == c_ares_async::make_ares_error(ARES_ENOTFOUND)) + found.not_found = !found.aliased; // the target of an alias is another matter + else if (ec != c_ares_async::make_ares_error(ARES_ENODATA)) + { + found.lookup_failed = true; + found.notes.push_back( + std::format("HTTPS lookup of '{}': {}", found.owner, ec.message())); + } + co_return found; + } + + // + // A record in AliasMode makes the rest of the set irrelevant (section 2.4.1). + // + const auto alias = std::ranges::find(records, 0, &c_ares_async::https_record::priority); + if (alias == records.end()) + { + found.records = std::move(records); + co_return found; + } + + if (alias->target.empty() || alias->target == ".") + { + // + // "The service is not available" (section 2.5.1). This is advisory, but one who does not + // have to request anything can as well take the host's word for it. + // + found.notes.push_back(std::format("'{}' says the service is not available", found.owner)); + found.unavailable = true; + co_return found; + } + + found.aliased = true; + found.owner = normalize_host(alias->target); + if (!visited.insert(found.owner).second) + break; + } + + // + // Section 3.1: as if the name had no record at all. + // + co_return ServiceRecords{ + .owner = host, .notes = {std::format("alias chain of '{}' is too long or has a loop", host)}}; +} + +capy::task Prober::probe(const std::string& host) +{ + Result result{.host = host}; + + const auto found = co_await resolve_service(host); + result.aliased = found.aliased; + result.records = found.records; + + const auto plan = plan_routes(host, found, config_.default_port); + result.notes = plan.rejected; + for (const auto& rejected : plan.rejected) + spdlog::debug("{}: rejected: {}", host, rejected); + + if (config_.h3_only && !found.lookup_failed && !found.not_found && !has_h3_record(found)) + { + spdlog::debug("{}: left out, no record offers HTTP/3", host); + result.outcome = Result::Outcome::no_h3; + co_return result; + } + + if (plan.routes.empty()) + { + result.outcome = found.lookup_failed ? Result::Outcome::lookup_failed + : found.not_found ? Result::Outcome::not_found + : Result::Outcome::out_of_reach; + co_return result; + } + + // + // Routes stay on the host's own name, see plan_routes(), so that this is good for all of them. + // Outside of any timeout, as with the records. Not having an address says nothing about the + // host, and no record says that it has to have one. + // + spdlog::debug("{}: asking for the addresses of '{}'", host, host); + auto [lookup_ec, addresses] = co_await resolver_->resolve(host, ""); + if (lookup_ec || addresses.empty()) + { + result.outcome = Result::Outcome::lookup_failed; + result.notes.push_back(std::format("address lookup of '{}': {}", host, + lookup_ec ? lookup_ec.message() : "no addresses")); + co_return result; + } + + // + // One after the other, and not at the same time as with Happy Eyeballs. c-ares goes by RFC 6724, + // which puts IPv4 first wherever the source address is not a global one. + // + prefer_ipv6(addresses); + + result.outcome = Result::Outcome::failed; + for (const auto& route : plan.routes) + { + const auto deadline = std::chrono::steady_clock::now() + config_.timeout; + for (const auto& address : addresses) + { + const corosio::endpoint endpoint(address, route.port); + result.address = endpoint.address().to_string(); + spdlog::debug("{}: connecting to {} by {}, for {}", host, result.address, + route.to_string(), route.source); + auto [ec, response] = co_await attempt(route, host, endpoint, deadline); + if (ec) + { + result.error = std::format("[{}] {}", route.to_string(), response.error); + result.notes.push_back(std::format("{} at {}", result.error, result.address)); + spdlog::debug("{}: {} at {}", host, result.error, result.address); + if (response.request_sent) + co_return result; // the host may have seen it, and anything else would make it two + if (std::chrono::steady_clock::now() >= deadline) + break; // the next address gets what is left of the timeout, which is nothing + continue; + } + + result.outcome = Result::Outcome::answered; + result.error.clear(); + result.status = static_cast(response.message.result_int()); + auto spoken = route; + spoken.protocol = response.protocol; + result.route = spoken.to_string(); + for (const auto& field : response.message) + result.fields.emplace_back(field.name_string(), field.value()); + result.body_size = response.message.body().size(); + result.duration = response.duration; + co_return result; + } + } + co_return result; +} + +// +// Connects as the route says, to this one address so that there is no doubt about which it was, and +// asks for the host's root, by the deadline. After that it ends the connection in an orderly way, +// which has limits of its own and is none of the timeout's business: the response is there by then. +// +capy::task> +Prober::attempt(const Route& route, const std::string& host, const corosio::endpoint& endpoint, + std::chrono::steady_clock::time_point deadline) +{ + anyhttp::client::Config client_config; + client_config.protocol = route.protocol; + client_config.tls_ca_file = config_.tls_ca_file; + + // + // The name of the route is what anyhttp verifies the certificate against, wherever the + // connection goes. The scheme is what makes it TLS over TCP, too. + // + client_config.endpoints = {endpoint}; + client_config.url.set_scheme("https"); + client_config.url.set_host_address(route.host); + client_config.url.set_port_number(route.port); + + const auto start = std::chrono::steady_clock::now(); + anyhttp::client::Client client(executor_, std::move(client_config)); + anyhttp::Session session; + bool request_sent = false; + + // + // These stay around for as long as exchange() needs them: timeout() waits for what it has + // cancelled to finish. + // + auto [ec, response] = co_await corosio::timeout( + exchange(client, session, request_sent, boost::urls::url(std::format("https://{}/", host))), + deadline); + + response.request_sent = request_sent; + if (ec == capy::cond::timeout) + { + response.error = std::format("timed out after {}", config_.timeout); + co_return {ec, std::move(response)}; + } + + response.duration = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + + if (!ec) + if (auto [shutdown_ec] = co_await session.shutdown(); shutdown_ec) + spdlog::debug("{}: shutdown: {}", host, shutdown_ec.message()); + + co_return {ec, std::move(response)}; +} + +// +// Everything is reported rather than thrown: one host that makes anyhttp throw does not get to end +// the run for the others. +// +capy::task> Prober::exchange(anyhttp::client::Client& client, + anyhttp::Session& session, + bool& request_sent, + boost::urls::url url) +{ + try + { + auto [connect_ec, connected] = co_await client.connect(); + if (connect_ec) + co_return {connect_ec, + Response{.error = std::format("connect: {}", connect_ec.message())}}; + + session = std::move(connected); + + anyhttp::Fields headers; + headers.set(http::field::user_agent, user_agent); + request_sent = true; + auto [get_ec, message] = co_await session.get(std::move(url), std::move(headers)); + co_return {get_ec, + Response{.message = std::move(message), + .error = get_ec ? std::format("request: {}", get_ec.message()) : "", + .protocol = session.protocol()}}; + } + catch (const std::exception& ex) + { + co_return {std::make_error_code(std::errc::connection_aborted), Response{.error = ex.what()}}; + } +} + +// ================================================================================================= + +} // namespace http_probe diff --git a/src/prober.hpp b/src/prober.hpp new file mode 100644 index 0000000..187f500 --- /dev/null +++ b/src/prober.hpp @@ -0,0 +1,173 @@ +// +// One request to https:/// for each host, in the way that the HTTPS records of the host say. +// +#pragma once + +#include "routes.hpp" + +#include + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace http_probe +{ + +namespace capy = boost::capy; +namespace corosio = boost::corosio; + +// ================================================================================================= + +struct Config +{ + /// The host names to request, exactly as given: these are the only names that are connected to. + std::vector hosts; + + /// How many hosts to work on at the same time. + std::size_t concurrency = 16; + + /// How long the request may take on one route, from connecting to the end of the response, + /// including any addresses of the host that could not be connected to. + std::chrono::milliseconds timeout{10000}; + + /// The DNS servers to ask, comma separated: what resolver::set_servers() takes, or a host name + /// with or without a port. Empty for those of the system. + std::string dns_server; + + /// Whether to ask them over TLS (RFC 7858), and what to check their certificates against, in the + /// trust store of the TLS library, which tls_ca_file has no say in. A single host name in + /// dns_server goes without saying, as far as the name in here goes. + c_ares_async::tls_options dns_tls; + + /// Who to look up the host names in dns_server with, always in the clear. Empty for the servers + /// of the system, and only tests have a use for anything else. + std::string dns_server_lookup; + + /// The CA certificates to verify servers against, as a PEM file. Empty for the system's. + std::string tls_ca_file; + + /// Where to go for HTTPS when the records don't say. Only tests have a use for anything else. + std::uint16_t default_port = 443; + + /// Connect only to the hosts with a record that offers HTTP/3, see has_h3_record(). Those are + /// treated as usual, which includes going over TCP if HTTP/3 can not be connected to. + bool h3_only = false; +}; + +/// Whether \p host is a name and nothing but a name: no URL, port, path or wildcard, and not an +/// address either, which has no records to look up. +bool is_host_name(std::string_view host); + +/** + * The hosts in a list, one per line, each as a name or as a URL to take the host from. Empty lines + * and those starting with '#' are left out, and what is neither of the two goes to \p skipped, so + * that one odd line does not get in the way of the others. + */ +std::vector read_hosts(std::istream& list, std::vector& skipped); + +/// Moves the IPv6 addresses to the front, each family keeping the order it was in. +void prefer_ipv6(std::vector& addresses); + +/// What came of one host. +struct Result +{ + enum class Outcome + { + answered, ///< a response arrived, whatever its status + failed, ///< it was connected to, and no response could be had + out_of_reach, ///< not connected to, its records say so or lead elsewhere, see plan_routes() + not_found, ///< not connected to, the name does not exist + lookup_failed, ///< not connected to, DNS gave no answer, so there is no telling + no_h3, ///< not connected to, it has no record that offers HTTP/3, see Config::h3_only + }; + + std::string host; + Outcome outcome = Outcome::failed; + bool aliased = false; ///< its records are those of another name + std::vector records; ///< the records that were considered + + int status = 0; ///< with \c answered + std::string route; ///< with \c answered: what the response came over, as the server chose it + std::vector> fields; ///< with \c answered + std::size_t body_size = 0; ///< with \c answered + std::chrono::milliseconds duration{}; ///< with \c answered: of the attempt that got through + + /// The address that answered or, with \c failed, the last one contacted. Empty without one. + std::string address; + std::string error; ///< with \c failed: what went wrong on the last route + std::vector notes; ///< records that could not be used, routes that did not work +}; + +/** + * Looks up the HTTPS records and the addresses of each of Config::hosts with the resolver of + * c-ares-async, and sends one request to each host that can be connected to (see plan_routes()), + * trying its addresses one at a time on each route, IPv6 first. A response is what comes back, + * whatever its status: a redirect is reported and not followed, nothing in the body is looked at, + * and nothing else on the host is requested. The next route is for a connection that did not come + * about: once a request has gone out, that was it for this host, however it ended. + * + * Hosts are worked on Config::concurrency at a time, so results arrive as they are ready. All of + * this is on the one executor, which is also what calls the observer. + */ +class Prober +{ +public: + using Observer = std::function; + + /// Throws \c std::invalid_argument for a configuration that is not good to go, and does nothing + /// else until run(), which has more to say about the DNS servers: the same for a list that can't + /// be used, and \c std::runtime_error for a host name in it that can't be looked up. + Prober(capy::any_executor executor, Config config, Observer observer); + + capy::task<> run(); + +private: + struct Response + { + anyhttp::client::Message message; + std::string error; + anyhttp::Protocol protocol = anyhttp::Protocol::h3; ///< what the server chose to speak + bool request_sent = false; + std::chrono::milliseconds duration{}; + }; + + capy::task dns_server_addresses(); + capy::task> work(); + capy::task resolve_service(const std::string& host); + capy::task probe(const std::string& host); + capy::task> attempt(const Route& route, const std::string& host, + const corosio::endpoint& endpoint, + std::chrono::steady_clock::time_point deadline); + capy::task> exchange(anyhttp::client::Client& client, + anyhttp::Session& session, bool& request_sent, + boost::urls::url url); + + capy::any_executor executor_; + Config config_; + Observer observer_; + std::vector hosts_; + std::size_t next_host_ = 0; + + /// For the duration of run(), as a connection for DNS over TLS stays open for as long as this is + /// around, and with that the event loop busy. By pointer, as there is no std::optional of it. + std::unique_ptr resolver_; +}; + +// ================================================================================================= + +} // namespace http_probe diff --git a/src/routes.cpp b/src/routes.cpp new file mode 100644 index 0000000..1fc85a1 --- /dev/null +++ b/src/routes.cpp @@ -0,0 +1,223 @@ +#include "routes.hpp" + +#include +#include +#include +#include +#include +#include + +namespace http_probe +{ + +namespace +{ + +// ================================================================================================= + +// +// What a ServiceMode record says, once its parameters have been read. +// +struct Service +{ + std::vector alpn; + bool no_default_alpn = false; + std::optional port; + std::optional incompatible; // why this record has to be ignored, see section 8 + + bool offers(std::string_view protocol) const { return std::ranges::contains(alpn, protocol); } + + // + // What to connect with, in the order to try them, which is at most one connection per transport. + // anyhttp's h2 offers HTTP/1.1 along with it, so that a server still has the choice over TCP. + // + std::vector protocols() const + { + using enum anyhttp::Protocol; + std::vector result; + if (offers("h3")) + result.push_back(h3); + if (offers("h2")) + result.push_back(h2); + else if (offers("http/1.1")) + result.push_back(h1); + return result; + } +}; + +std::vector split_list(std::string_view list) +{ + return list | std::views::split(',') | + std::views::transform([](auto part) { return std::string(std::string_view(part)); }) | + std::ranges::to(); +} + +// +// A key that can be listed as mandatory for a client like this one: everything it either acts on +// or may leave alone, like the address hints. Not "ech", which it can not do, and not a key it has +// no name for. +// +bool can_be_mandatory(std::string_view key) +{ + static constexpr std::array keys{"mandatory", "alpn", "no-default-alpn", + "port", "ipv4hint", "ipv6hint"}; + return std::ranges::contains(keys, key); +} + +Service parse_service(const c_ares_async::https_record& record) +{ + Service service; + for (std::string_view param : record.params) + { + const auto equals = param.find('='); + const auto key = param.substr(0, equals); + const auto value = equals == param.npos ? std::string_view{} : param.substr(equals + 1); + + if (key == "alpn") + service.alpn = split_list(value); + else if (key == "no-default-alpn") + service.no_default_alpn = true; + else if (key == "port") + { + std::uint16_t port = 0; + const auto [end, ec] = std::from_chars(value.data(), value.data() + value.size(), port); + if (ec != std::errc{} || end != value.data() + value.size()) + service.incompatible = std::format("invalid port '{}'", value); + else + service.port = port; + } + else if (key == "mandatory") + for (const auto& mandatory : split_list(value)) + if (!can_be_mandatory(mandatory)) + service.incompatible = std::format("mandatory key '{}' is not supported", mandatory); + } + + if (!service.no_default_alpn) + service.alpn.emplace_back("http/1.1"); // the default set of the HTTPS record, section 9.1 + return service; +} + +bool is_own_name(std::string_view target) { return target.empty() || target == "."; } + +// ================================================================================================= + +} // namespace + +std::string normalize_host(std::string_view host) +{ + if (host.ends_with('.')) + host.remove_suffix(1); + + std::string result(host); + std::ranges::transform(result, result.begin(), [](unsigned char c) { return std::tolower(c); }); + return result; +} + +std::string to_text(const c_ares_async::https_record& record) +{ + auto text = + std::format("HTTPS {} {}", record.priority, record.target.empty() ? "." : record.target); + for (const auto& param : record.params) + text += " " + param; + return text; +} + +std::string Route::to_string() const +{ + using enum anyhttp::Protocol; + return std::format("{} {}:{}", protocol == h3 ? "h3" : protocol == h2 ? "h2" : "h1", host, port); +} + +Plan plan_routes(std::string_view host, const ServiceRecords& found, std::uint16_t default_port) +{ + Plan plan{.rejected = found.notes}; + + if (found.lookup_failed || found.not_found) + return plan; + + if (found.unavailable) + { + plan.out_of_reach = true; + return plan; + } + + auto records = found.records; + std::ranges::stable_sort(records, {}, &c_ares_async::https_record::priority); + + bool any_compatible = false; + for (const auto& record : records) + { + const auto text = to_text(record); + const auto service = parse_service(record); + if (service.incompatible) + { + plan.rejected.push_back(std::format("{}: {}", text, *service.incompatible)); + continue; + } + + const auto protocols = service.protocols(); + if (protocols.empty()) + { + plan.rejected.push_back(std::format("{}: no protocol that anyhttp speaks", text)); + continue; + } + any_compatible = true; + + const auto target = normalize_host(is_own_name(record.target) ? found.owner : record.target); + if (target != host) + { + plan.rejected.push_back(std::format( + "{}: target '{}' is not the host, and anyhttp can only verify the name it connects to", + text, target)); + continue; + } + + for (const auto protocol : protocols) + { + Route route{.protocol = protocol, + .host = target, + .port = service.port.value_or(default_port), + .source = text}; + const auto same = [&](const Route& other) { + return other.protocol == route.protocol && other.port == route.port; + }; + if (std::ranges::none_of(plan.routes, same)) // a second try would only fail the same way + plan.routes.push_back(std::move(route)); + } + } + + if (!plan.routes.empty()) + return plan; + + // + // After an alias, the name it led to is an endpoint even without any parameters (section 3). + // + if (found.aliased && !any_compatible) + plan.rejected.push_back(std::format( + "alias target '{}' is not the host, and anyhttp can only verify the name it connects to", + found.owner)); + + if (found.aliased || any_compatible) + { + plan.out_of_reach = true; + return plan; + } + + plan.routes.push_back(Route{.protocol = anyhttp::Protocol::h2, + .host = std::string(host), + .port = default_port, + .source = "no usable HTTPS record"}); + return plan; +} + +bool has_h3_record(const ServiceRecords& found) +{ + return std::ranges::any_of(found.records, [](const c_ares_async::https_record& record) { + const auto service = parse_service(record); + return !service.incompatible && service.offers("h3"); + }); +} + +// ================================================================================================= + +} // namespace http_probe diff --git a/src/routes.hpp b/src/routes.hpp new file mode 100644 index 0000000..c854bfd --- /dev/null +++ b/src/routes.hpp @@ -0,0 +1,80 @@ +// +// From the HTTPS records of a name to the ways of connecting to it (RFC 9460). No I/O in here: +// prober.cpp asks the questions, this says what the answers mean. +// +#pragma once + +#include + +#include + +#include +#include +#include +#include + +namespace http_probe +{ + +// ================================================================================================= + +/// Lower case and without the dot at the end, which is how host names are compared. +std::string normalize_host(std::string_view host); + +/// A record the way a zone file has it, without the name: "HTTPS 1 . alpn=h3 port=8443". +std::string to_text(const c_ares_async::https_record& record); + +// ------------------------------------------------------------------------------------------------- + +/// What SVCB resolution (section 3) found for a host. +struct ServiceRecords +{ + std::string owner; ///< the name the records are of, at the end of any alias chain + bool aliased = false; ///< at least one AliasMode record was followed + std::vector records; ///< the ServiceMode records of \c owner + std::vector notes; ///< what went wrong on the way, none of which is fatal + bool lookup_failed = false; ///< no answer, which is not to say that there are no records + bool not_found = false; ///< the host itself does not exist + bool unavailable = false; ///< an alias to "." says that the service is not available +}; + +/// One way of connecting, always over TLS. The TLS name and the authority stay the host's. +struct Route +{ + anyhttp::Protocol protocol = anyhttp::Protocol::h3; + std::string host; ///< the name to connect to + std::uint16_t port = 443; + std::string source; ///< the record that led to this route + + std::string to_string() const; ///< "h3 example.com:443" +}; + +/// How to reach a host. +struct Plan +{ + std::vector routes; ///< in the order to try them; empty if there is no usable way + std::vector rejected; ///< what the records offered, or we knew of, and why not + bool out_of_reach = false; ///< the records say not to connect, or only to a target we can't use +}; + +/** + * Says how to connect to https://\p host/, given what was found for it. + * + * Records go by priority, and within a record, HTTP/3 comes ahead of TCP. A name without a usable + * record, the most common thing there is, gets \p default_port over TCP, and so does a record that + * does not say which port (section 9.1 on both of them). + * + * anyhttp verifies the certificate against the host it connects to, while RFC 9460 (section 9.1) + * wants that to be the origin, so a record is usable only if it stays on the host's own name. If + * none of them does, the host is out of reach: the records do say where to go, and it is not here. + * A name that does not exist and a lookup that failed have no routes either. + */ +Plan plan_routes(std::string_view host, const ServiceRecords& found, + std::uint16_t default_port = 443); + +/// Whether a record that has to be taken seriously (section 8) offers HTTP/3, wherever it points. +bool has_h3_record(const ServiceRecords& found); + +// ================================================================================================= + +} // namespace http_probe diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt new file mode 100644 index 0000000..81ac6a4 --- /dev/null +++ b/test/CMakeLists.txt @@ -0,0 +1,90 @@ +# +# GoogleTest as in c-ares-async: from wherever it is installed, from its release archive otherwise. +# +include(FetchContent) +FetchContent_Declare( + googletest + URL https://github.com/google/googletest/releases/download/v1.18.0/googletest-1.18.0.tar.gz + URL_HASH SHA256=6e3191c1455468b3fc35a417fb565c1c5071aee1b7e7f85e30cf48a98d37d8b5 + FIND_PACKAGE_ARGS NAMES GTest) +set(INSTALL_GTEST OFF) +FetchContent_MakeAvailable(googletest) + +find_package(OpenSSL REQUIRED) + +# +# The DNS server of c-ares-async's own tests, at the commit of the pin: not an interface of that +# library, but all it takes to have a zone of our own on loopback. Moving the pin may mean adapting +# to it. Compiled apart, as the warnings of this project are not for those sources. +# +add_library( + fake_dns STATIC ${c_ares_async_SOURCE_DIR}/test/fake_dns_server.cpp + ${c_ares_async_SOURCE_DIR}/test/test_certificate.cpp) +target_include_directories(fake_dns SYSTEM PUBLIC ${c_ares_async_SOURCE_DIR}/test) +target_link_libraries(fake_dns PUBLIC c_ares_async::c_ares_async anyhttp::anyhttp OpenSSL::SSL + OpenSSL::Crypto) + +add_library(test_site STATIC site.cpp) +target_include_directories(test_site PUBLIC .) +target_link_libraries(test_site PUBLIC http_probe_core fake_dns) +target_compile_options(test_site PRIVATE ${http_probe_warnings}) + +add_executable(http_probe_tests output_test.cpp prober_test.cpp routes_test.cpp) +target_link_libraries(http_probe_tests PRIVATE test_site GTest::gmock GTest::gtest_main) +target_compile_options(http_probe_tests PRIVATE ${http_probe_warnings}) + +include(GoogleTest) +gtest_discover_tests(http_probe_tests DISCOVERY_TIMEOUT 60 PROPERTIES TIMEOUT 5) + +# +# A site on loopback to point the program at by hand, as the README shows. +# +add_executable(demo_site demo_site.cpp) +target_link_libraries(demo_site PRIVATE test_site) +target_compile_options(demo_site PRIVATE ${http_probe_warnings}) + +# +# What the program itself does ahead of any connection. Everything else is in the tests above. +# +add_test(NAME http_probe_refuses_no_hosts COMMAND http-probe) +set_tests_properties(http_probe_refuses_no_hosts PROPERTIES + PASS_REGULAR_EXPRESSION "nothing to do: no host names") + +add_test(NAME http_probe_refuses_a_url COMMAND http-probe https://localhost/) +set_tests_properties(http_probe_refuses_a_url PROPERTIES + PASS_REGULAR_EXPRESSION "is not a host name") + +add_test(NAME http_probe_refuses_a_file_it_can_not_read COMMAND http-probe --file /nonexistent/list) +set_tests_properties(http_probe_refuses_a_file_it_can_not_read PROPERTIES + PASS_REGULAR_EXPRESSION "can't read") + +add_test(NAME http_probe_refuses_a_concurrency_of_zero + COMMAND http-probe --concurrency 0 localhost) +set_tests_properties(http_probe_refuses_a_concurrency_of_zero PROPERTIES + PASS_REGULAR_EXPRESSION "concurrency of 0") + +add_test(NAME http_probe_refuses_a_negative_concurrency + COMMAND http-probe --concurrency=-1 localhost) +set_tests_properties(http_probe_refuses_a_negative_concurrency PROPERTIES + PASS_REGULAR_EXPRESSION "--concurrency can't be negative") + +add_test(NAME http_probe_refuses_a_negative_timeout COMMAND http-probe --timeout-ms -1 localhost) +set_tests_properties(http_probe_refuses_a_negative_timeout PROPERTIES + PASS_REGULAR_EXPRESSION "--timeout-ms can't be negative") + +add_test(NAME http_probe_refuses_an_unknown_option COMMAND http-probe --no-such-option localhost) +set_tests_properties(http_probe_refuses_an_unknown_option PROPERTIES + PASS_REGULAR_EXPRESSION "unrecognised option '--no-such-option'") + +add_test(NAME http_probe_describes_its_options COMMAND http-probe --help) +set_tests_properties(http_probe_describes_its_options PROPERTIES + PASS_REGULAR_EXPRESSION "--dns-server SERVER") + +add_test(NAME http_probe_refuses_dot_without_a_name COMMAND http-probe --dot localhost) +set_tests_properties(http_probe_refuses_dot_without_a_name PROPERTIES + PASS_REGULAR_EXPRESSION "--dot needs a name") + +add_test(NAME http_probe_refuses_tls_options_without_dot + COMMAND http-probe --tls-no-verify localhost) +set_tests_properties(http_probe_refuses_tls_options_without_dot PROPERTIES + PASS_REGULAR_EXPRESSION "only apply with --dot") diff --git a/test/demo_site.cpp b/test/demo_site.cpp new file mode 100644 index 0000000..41980fa --- /dev/null +++ b/test/demo_site.cpp @@ -0,0 +1,49 @@ +// +// A site on loopback, with an HTTPS record that sends the program to it over HTTP/3, and the +// command to request it with. Runs for as long as it takes to stop it. +// +#include "site.hpp" + +#include +#include +#include +#include + +#include + +#include +#include +#include + +namespace capy = boost::capy; +namespace corosio = boost::corosio; + +static capy::task<> stop_on_signal(corosio::signal_set& signals, http_probe::test::Site& site) +{ + std::ignore = co_await signals.wait(); + site.stop(); +} + +int main() +{ + spdlog::set_level(spdlog::level::err); + + corosio::io_context context; + http_probe::test::Site site(context.get_executor()); + + site.dns().add_https("localhost", 1, ".", + {{ARES_SVCB_PARAM_ALPN, {2, 'h', '3'}}, + {ARES_SVCB_PARAM_PORT, + {static_cast(site.port() >> 8), + static_cast(site.port() & 0xff)}}}); + + site.page("/", {.body = "

hello

"}); + + std::println("http-probe --dns-server {} --ca-file {} localhost", site.dns_address(), + site.ca_file().string()); + std::fflush(stdout); + + corosio::signal_set signals(context, SIGINT, SIGTERM); + capy::run_async(context.get_executor())(stop_on_signal(signals, site)); + context.run(); +} diff --git a/test/output_test.cpp b/test/output_test.cpp new file mode 100644 index 0000000..b8a4a35 --- /dev/null +++ b/test/output_test.cpp @@ -0,0 +1,157 @@ +// +// What the program prints: with colour, and with nothing of it left without. +// +#include "output.hpp" + +#include +#include + +using namespace http_probe; +using namespace std::chrono_literals; +using Outcome = Result::Outcome; +using testing::HasSubstr; +using testing::Not; + +namespace +{ + +Result answered(int status = 200) +{ + return {.host = "example.com", + .outcome = Outcome::answered, + .records = {{.priority = 1, + .target = ".", + .params = {"alpn=h3", "ipv4hint=192.0.2.1", "ipv6hint=2001:db8::1"}}}, + .status = status, + .route = "h3 example.com:443", + .fields = {{"server", "test"}}, + .body_size = 5, + .duration = 12ms, + .address = "192.0.2.1"}; +} + +} // namespace + +TEST(Output, SaysWhatCameBack) +{ + EXPECT_EQ(format_result(answered(), false), + "example.com: 200 [h3 example.com:443] at 192.0.2.1, 5 bytes in 12ms\n" + " record: HTTPS 1 . alpn=h3 ipv4hint=192.0.2.1 ipv6hint=2001:db8::1\n" + " server: test\n"); +} + +TEST(Output, SaysWhichAddressFailed) +{ + EXPECT_EQ(format_result({.host = "example.com", + .outcome = Outcome::failed, + .address = "2001:db8::1", + .error = "[h3 example.com:443] timed out"}, + false), + "example.com: failed at 2001:db8::1: [h3 example.com:443] timed out\n"); +} + +TEST(Output, ColoursAnAddressByItsFamily) +{ + auto result = answered(); + EXPECT_THAT(format_result(result, true), HasSubstr("at \x1b[35m192.0.2.1\x1b[0m,")); + + result.address = "2001:db8::1"; + EXPECT_THAT(format_result(result, true), HasSubstr("at \x1b[34m2001:db8::1\x1b[0m,")); +} + +TEST(Output, SaysWhyThereIsNoResponse) +{ + EXPECT_EQ(format_result({.host = "example.com", + .outcome = Outcome::failed, + .error = "[h3 example.com:443] timed out", + .notes = {"some note"}}, + false), + "example.com: failed: [h3 example.com:443] timed out\n note: some note\n"); + EXPECT_EQ(format_result({.host = "example.com", .outcome = Outcome::out_of_reach}, false), + "example.com: out of reach\n"); + EXPECT_EQ(format_result({.host = "example.com", .outcome = Outcome::not_found}, false), + "example.com: not found\n"); + EXPECT_EQ(format_result({.host = "example.com", .outcome = Outcome::lookup_failed}, false), + "example.com: lookup failed\n"); +} + +TEST(Output, SaysThatAHostWasLeftOut) +{ + EXPECT_EQ(format_result({.host = "example.com", .outcome = Outcome::no_h3}, false), + "example.com: no h3 record\n"); +} + +TEST(Output, CountsTheHostsLeftOutOnlyIfThereAreAny) +{ + EXPECT_EQ(format_summary({{Outcome::answered, 3}, {Outcome::no_h3, 5}}, false), + "3 answered, 0 failed, 0 out of reach, 0 not found, 0 lookups failed, 5 without h3"); + EXPECT_THAT(format_summary({{Outcome::answered, 3}}, false), Not(HasSubstr("h3"))); +} + +TEST(Output, ColoursTheWayCAresAsyncDoes) +{ + const auto text = format_result(answered(), true); + + EXPECT_THAT(text, HasSubstr("\x1b[33mexample.com\x1b[0m:")) << "host names are yellow"; + EXPECT_THAT(text, HasSubstr("ipv4hint=\x1b[35m192.0.2.1\x1b[0m")) << "IPv4 is magenta"; + EXPECT_THAT(text, HasSubstr("ipv6hint=\x1b[34m2001:db8::1\x1b[0m")) << "IPv6 is blue"; +} + +TEST(Output, LeavesTheCommasOfAnAddressListWithoutColour) +{ + auto result = answered(); + result.records[0].params = {"ipv4hint=192.0.2.1,192.0.2.2", "ipv6hint=2001:db8::1,2001:db8::2"}; + + const auto text = format_result(result, true); + EXPECT_THAT(text, HasSubstr("ipv4hint=\x1b[35m192.0.2.1\x1b[0m,\x1b[35m192.0.2.2\x1b[0m")); + EXPECT_THAT(text, HasSubstr("ipv6hint=\x1b[34m2001:db8::1\x1b[0m,\x1b[34m2001:db8::2\x1b[0m")); + + EXPECT_THAT(format_result(result, false), + HasSubstr("ipv4hint=192.0.2.1,192.0.2.2 ipv6hint=2001:db8::1,2001:db8::2")); +} + +TEST(Output, ColoursATargetAsAHostName) +{ + auto result = answered(); + result.records[0].target = "svc.example.net"; + + EXPECT_THAT(format_result(result, true), HasSubstr("\x1b[33msvc.example.net\x1b[0m")); + EXPECT_THAT(format_result(result, false), HasSubstr("HTTPS 1 svc.example.net alpn=h3")); +} + +TEST(Output, ColoursTheStatusByItsClass) +{ + EXPECT_THAT(format_result(answered(204), true), HasSubstr("\x1b[32m204\x1b[0m")); + EXPECT_THAT(format_result(answered(301), true), HasSubstr("\x1b[36m301\x1b[0m")); + EXPECT_THAT(format_result(answered(404), true), HasSubstr("\x1b[31m404\x1b[0m")); + EXPECT_THAT(format_result(answered(503), true), HasSubstr("\x1b[31m503\x1b[0m")); +} + +TEST(Output, HasNoColourWithoutColour) +{ + for (const auto& result : + {answered(), Result{.host = "h", .outcome = Outcome::failed, .notes = {"n"}}, + Result{.host = "h", .outcome = Outcome::lookup_failed}}) + EXPECT_THAT(format_result(result, false), Not(HasSubstr("\x1b"))); + + const Counts counts{{Outcome::answered, 1}, {Outcome::failed, 1}}; + EXPECT_THAT(format_summary(counts, false), Not(HasSubstr("\x1b"))); +} + +TEST(Output, CountsTheOutcomes) +{ + EXPECT_EQ(format_summary({{Outcome::answered, 3}, + {Outcome::failed, 1}, + {Outcome::not_found, 900}, + {Outcome::lookup_failed, 17}}, + false), + "3 answered, 1 failed, 0 out of reach, 900 not found, 17 lookups failed"); +} + +TEST(Output, DrawsAttentionOnlyToWhatHappened) +{ + const auto text = format_summary({{Outcome::answered, 1}}, true); + + EXPECT_THAT(text, HasSubstr("\x1b[32m1 answered\x1b[0m")); + EXPECT_THAT(text, HasSubstr(", 0 failed,")) << "nothing to see, no colour"; +} diff --git a/test/prober_test.cpp b/test/prober_test.cpp new file mode 100644 index 0000000..05a4e5a --- /dev/null +++ b/test/prober_test.cpp @@ -0,0 +1,1263 @@ +// +// The whole program but its command line, against a server and a DNS server of its own on loopback. +// Nothing in here may ever reach for anything else. +// +#include "prober.hpp" +#include "site.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace http_probe; +using namespace std::chrono_literals; +using http_probe::test::Site; +using testing::Contains; +using testing::ElementsAre; +using testing::HasSubstr; +using testing::IsEmpty; +using testing::Pair; + +namespace boost::corosio +{ + +/// What a failed expectation shows of an endpoint: Corosio has no output operator for one. +inline void PrintTo(const endpoint& endpoint, std::ostream* out) +{ + *out << endpoint.address() << ':' << endpoint.port(); +} + +} // namespace boost::corosio + +namespace +{ + +corosio::ip_address make_address(std::string_view text) +{ + auto [ec, address] = corosio::make_ip_address(text); + if (ec) + throw std::system_error(ec, std::string(text)); + return address; +} + +fake_dns_server::svc_param port_param(std::uint16_t port) +{ + return {ARES_SVCB_PARAM_PORT, + {static_cast(port >> 8), static_cast(port & 0xff)}}; +} + +const fake_dns_server::svc_param alpn_h3{ARES_SVCB_PARAM_ALPN, {2, 'h', '3'}}; +const fake_dns_server::svc_param alpn_h2{ARES_SVCB_PARAM_ALPN, {2, 'h', '2'}}; +const fake_dns_server::svc_param alpn_h1{ARES_SVCB_PARAM_ALPN, + {8, 'h', 't', 't', 'p', '/', '1', '.', '1'}}; +const fake_dns_server::svc_param no_default_alpn{ARES_SVCB_PARAM_NO_DEFAULT_ALPN, {}}; + +// +// One protocol and no other, for a test to know which one it is getting. A record with nothing but +// an "alpn" also has the default set, and that means trying HTTP/1.1 as well. +// +std::vector only(fake_dns_server::svc_param alpn, std::uint16_t port) +{ + return {std::move(alpn), no_default_alpn, port_param(port)}; +} + +std::vector h3_at(std::uint16_t port) { return only(alpn_h3, port); } +std::vector h2_at(std::uint16_t port) { return only(alpn_h2, port); } +std::vector h1_at(std::uint16_t port) { return only(alpn_h1, port); } + +class Probe : public testing::Test +{ +protected: + Config config(std::vector hosts = {"localhost"}) + { + return {.hosts = std::move(hosts), + .timeout = 1000ms, + .dns_server = site.dns_address(), + .tls_ca_file = site.ca_file().string(), + .default_port = site.port()}; + } + + void probe(Config config) + { + Prober prober(context.get_executor(), std::move(config), + [this](const Result& result) { results.push_back(result); }); + + // + // In a variable: the coroutine refers to what the lambda captured, for as long as it runs. + // + const auto run = [&]() -> capy::task<> { + co_await prober.run(); + stop_sites(); + }; + + std::exception_ptr failed; + capy::run_async( + context.get_executor(), []() noexcept {}, + [&](std::exception_ptr ex) noexcept { + failed = ex; + context.stop(); + })(run()); + context.run(); + if (failed) + std::rethrow_exception(failed); + } + + // + // A port where nothing is ever going to answer, on any address that "localhost" may be. + // + corosio::udp_socket silent_udp_socket() + { + corosio::udp_socket socket(context); + if (const auto ec = socket.open(corosio::family::v6)) + throw std::system_error(ec, "open"); + if (const auto ec = socket.bind({corosio::ipv6_address::any(), 0})) + throw std::system_error(ec, "bind"); + return socket; + } + + corosio::tcp_acceptor silent_tcp_acceptor() + { + return corosio::tcp_acceptor(context, {corosio::ipv6_address::any(), 0}); + } + + // + // What the tests that do not get as far as run() still have to do. + // + void finish() + { + stop_sites(); + context.run(); + } + + virtual void stop_sites() { site.stop(); } + + static int quiet() + { + spdlog::set_level(spdlog::level::off); // anyhttp logs through the default logger + return 0; + } + + static std::string notes(const Result& result) + { + std::string text; + for (const auto& note : result.notes) + text += note + "\n"; + return text; + } + + int quiet_ = quiet(); // ahead of the server, which already has something to say when it starts + corosio::io_context context; + Site site{context.get_executor()}; + std::vector results; +}; + +} // namespace + +// ================================================================================================= +// What it refuses +// ================================================================================================= + +TEST_F(Probe, RefusesToRunWithoutHosts) +{ + EXPECT_THROW(probe(config({})), std::invalid_argument); + finish(); +} + +TEST_F(Probe, RefusesAConcurrencyOfZero) +{ + auto configuration = config(); + configuration.concurrency = 0; + EXPECT_THROW(probe(std::move(configuration)), std::invalid_argument); + finish(); +} + +TEST_F(Probe, RefusesAnythingButAHostName) +{ + for (const auto* host : {"https://localhost/", "localhost/x", "localhost:443", "*.localhost", + "user@localhost", "", "127.0.0.1", "::1"}) + EXPECT_THROW(probe(config({host})), std::invalid_argument) << host; + finish(); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST(ReadHosts, TakesNamesAndTheHostsOfUrls) +{ + std::istringstream list("example.com\n" + " www.example.com \r\n" + "\n" + "# a comment\n" + "https://www.example.net/some/path?query\n" + "http://example.org:8080/\n"); + std::vector skipped; + + EXPECT_THAT(read_hosts(list, skipped), + ElementsAre("example.com", "www.example.com", "www.example.net", "example.org")); + EXPECT_THAT(skipped, IsEmpty()); +} + +TEST(ReadHosts, SkipsWhatIsNeitherAndSaysSo) +{ + std::istringstream list("example.com\nnot a host\nhttps://127.0.0.1/\nexample.net\n"); + std::vector skipped; + + EXPECT_THAT(read_hosts(list, skipped), ElementsAre("example.com", "example.net")); + EXPECT_THAT(skipped, ElementsAre("not a host", "https://127.0.0.1/")); +} + +// ================================================================================================= +// One request, and what is reported of it +// ================================================================================================= + +TEST_F(Probe, RequestsTheHostAtThePortOfItsRecord) +{ + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + const auto& result = results[0]; + EXPECT_EQ(result.outcome, Result::Outcome::answered) << notes(result); + EXPECT_EQ(result.host, "localhost"); + EXPECT_EQ(result.status, 200); + EXPECT_EQ(result.route, std::format("h3 localhost:{}", site.port())); + EXPECT_EQ(result.body_size, 5u); + EXPECT_THAT(result.fields, Contains(Pair("content-type", "text/html"))); + ASSERT_EQ(result.records.size(), 1u); + EXPECT_THAT(to_text(result.records[0]), HasSubstr("alpn=h3")); + + ASSERT_EQ(site.requests().size(), 1u); + EXPECT_EQ(site.requests()[0].authority, "localhost") << "the host, not where it went"; + EXPECT_THAT(site.requests()[0].user_agent, HasSubstr("http-probe/")); +} + +TEST_F(Probe, RequestsNothingButTheRoot) +{ + // + // No robots.txt either: that is about crawling, and this does not. + // + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.page("/", {.body = R"(a b)"}); + site.page("/a", {.body = "not asked for"}); + site.page("/b", {.body = "not asked for"}); + + probe(config()); + + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, ReportsARedirectAndDoesNotFollowIt) +{ + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.page("/", {.status = 302, .location = "/moved"}); + site.page("/moved", {.body = "not asked for"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 302); + EXPECT_THAT(results[0].fields, Contains(Pair("location", "/moved"))); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, ReportsAnErrorStatusAsAResponse) +{ + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.page("/", {.status = 503}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 503); + EXPECT_EQ(results[0].outcome, Result::Outcome::answered); +} + +TEST_F(Probe, RequestsEachHostOnce) +{ + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.page("/", {.body = "hello"}); + + probe(config({"localhost", "LOCALHOST", "localhost."})); + + EXPECT_EQ(results.size(), 1u); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, GoesOnAfterAHostThatFails) +{ + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.page("/", {.body = "hello"}); + + probe(config({"nowhere.example.test", "localhost"})); + + ASSERT_EQ(results.size(), 2u); + EXPECT_EQ(results[0].host, "nowhere.example.test"); + EXPECT_EQ(results[0].outcome, Result::Outcome::not_found); + EXPECT_EQ(results[1].host, "localhost"); + EXPECT_EQ(results[1].status, 200); +} + +// ================================================================================================= +// A list of hosts +// ================================================================================================= + +TEST_F(Probe, DoesNotConnectToANameThatDoesNotExist) +{ + // + // Which would also take the lookup of its address to the resolver of the system, and away from + // loopback, for every test that has a name like this. + // + site.dns().add_address("localhost", "127.0.0.1"); + site.page("/", {.body = "hello"}); + + auto configuration = config({"nowhere.example.test", "localhost"}); + configuration.concurrency = 1; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 2u); + EXPECT_EQ(results[0].outcome, Result::Outcome::not_found); + EXPECT_EQ(results[1].outcome, Result::Outcome::answered) << notes(results[1]); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, ReportsEachHostOnceHoweverManyThereAre) +{ + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.page("/", {.body = "hello"}); + + std::vector hosts{"localhost"}; + for (int i = 0; i < 100; ++i) + hosts.push_back(std::format("host{}.example.test", i)); + + auto configuration = config(hosts); + configuration.concurrency = 7; + probe(std::move(configuration)); + + std::set reported; + for (const auto& result : results) + reported.insert(result.host); + EXPECT_EQ(results.size(), hosts.size()); + EXPECT_EQ(reported.size(), hosts.size()); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, DoesNotLetASlowHostHoldUpTheOthers) +{ + auto silent = silent_udp_socket(); + site.dns().add_https("localhost", 1, ".", h3_at(silent.local_endpoint().port())); + + auto configuration = config({"localhost", "nowhere.example.test"}); + configuration.timeout = 200ms; + configuration.concurrency = 2; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 2u); + EXPECT_EQ(results[0].host, "nowhere.example.test"); + EXPECT_EQ(results[1].host, "localhost"); +} + +TEST_F(Probe, WorksOnOneHostAtATimeIfToldTo) +{ + auto silent = silent_udp_socket(); + site.dns().add_https("localhost", 1, ".", h3_at(silent.local_endpoint().port())); + + auto configuration = config({"localhost", "nowhere.example.test"}); + configuration.timeout = 200ms; + configuration.concurrency = 1; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 2u); + EXPECT_EQ(results[0].host, "localhost"); + EXPECT_EQ(results[1].host, "nowhere.example.test"); +} + +// ================================================================================================= +// How the HTTPS records decide about the connection +// ================================================================================================= + +TEST_F(Probe, SpeaksHttp2OverTlsWhereTheRecordSaysSo) +{ + site.dns().add_https("localhost", 1, ".", h2_at(site.port())); + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 200) << notes(results[0]); + EXPECT_EQ(results[0].route, std::format("h2 localhost:{}", site.port())); + EXPECT_EQ(results[0].body_size, 5u); + ASSERT_EQ(site.requests().size(), 1u); + EXPECT_EQ(site.requests()[0].authority, "localhost"); + EXPECT_THAT(site.requests()[0].user_agent, HasSubstr("http-probe/")); +} + +TEST_F(Probe, SpeaksHttp1OverTlsWhereTheRecordSaysSo) +{ + site.dns().add_https("localhost", 1, ".", h1_at(site.port())); + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 200) << notes(results[0]); + EXPECT_EQ(results[0].route, std::format("h1 localhost:{}", site.port())); + EXPECT_EQ(results[0].body_size, 5u); + ASSERT_EQ(site.requests().size(), 1u); + EXPECT_THAT(site.requests()[0].authority, HasSubstr("localhost")); +} + +TEST_F(Probe, PrefersHttp3OfWhatARecordOffers) +{ + site.dns().add_https("localhost", 1, ".", {alpn_h3, port_param(site.port())}); + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].route, std::format("h3 localhost:{}", site.port())) << notes(results[0]); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, GoesOverTcpToTheDefaultPortWithoutARecord) +{ + site.dns().add_address("localhost", "127.0.0.1"); + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::answered) << notes(results[0]); + EXPECT_EQ(results[0].route, std::format("h2 localhost:{}", site.port())); + EXPECT_THAT(results[0].records, IsEmpty()); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, DoesNotFollowARedirectOverHttp2) +{ + site.dns().add_https("localhost", 1, ".", h2_at(site.port())); + site.page("/", {.status = 302, .location = "/moved"}); + site.page("/moved", {.body = "not asked for"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 302) << notes(results[0]); + EXPECT_THAT(results[0].fields, Contains(Pair("location", "/moved"))); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, DoesNotFollowARedirectOverHttp1) +{ + site.dns().add_https("localhost", 1, ".", h1_at(site.port())); + site.page("/", {.status = 302, .location = "/moved"}); + site.page("/moved", {.body = "not asked for"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 302) << notes(results[0]); + // As it came, and only HTTP/2 and HTTP/3 have field names in lower case on the wire. + EXPECT_THAT(results[0].fields, Contains(Pair("Location", "/moved"))); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, FallsBackToTheNextRecordWhenAConnectionFails) +{ + // + // Nothing answers on this port. The socket is ours for the length of the test, so that nothing + // else will either. + // + auto silent = silent_udp_socket(); + + site.dns().add_https("localhost", 1, ".", h3_at(silent.local_endpoint().port())); + site.dns().add_https("localhost", 2, ".", h3_at(site.port())); + site.page("/", {.body = "hello"}); + + auto configuration = config(); + configuration.timeout = 200ms; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 200) << notes(results[0]); + EXPECT_EQ(results[0].route, std::format("h3 localhost:{}", site.port())); + EXPECT_THAT(notes(results[0]), HasSubstr("timed out")); +} + +TEST_F(Probe, FallsBackFromHttp3ToTcpWhenAConnectionFails) +{ + auto silent = silent_udp_socket(); + + site.dns().add_https("localhost", 1, ".", h3_at(silent.local_endpoint().port())); + site.dns().add_https("localhost", 2, ".", h2_at(site.port())); + site.page("/", {.body = "hello"}); + + auto configuration = config(); + configuration.timeout = 200ms; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 200) << notes(results[0]); + EXPECT_EQ(results[0].route, std::format("h2 localhost:{}", site.port())); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, TriesNoOtherRouteOnceARequestHasGoneOut) +{ + // + // The server has it, it is just not done with it. Everything after this would be a second + // request to the same host: over HTTP/1.1 on the same record, and over HTTP/2 on the next. + // + site.dns().add_https("localhost", 1, ".", {alpn_h3, port_param(site.port())}); + site.dns().add_https("localhost", 2, ".", h2_at(site.port())); + site.page("/", {.body = "too late", .delay = 400ms}); + + auto configuration = config(); + configuration.timeout = 100ms; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::failed); + EXPECT_THAT(results[0].error, HasSubstr("timed out")); + EXPECT_THAT(results[0].error, HasSubstr("h3")); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, TriesNoOtherRouteOnceARequestHasGoneOutOverTcp) +{ + site.dns().add_https("localhost", 1, ".", h2_at(site.port())); + site.dns().add_https("localhost", 2, ".", h1_at(site.port())); + site.page("/", {.body = "too late", .delay = 400ms}); + + auto configuration = config(); + configuration.timeout = 100ms; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::failed); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, DoesNotConnectToATargetOtherThanTheHost) +{ + // + // The server would answer: it is the certificate that could not be right for this name. + // + site.dns().add_https("localhost", 1, "elsewhere.example.test", h3_at(site.port())); + site.dns().add_address("elsewhere.example.test", "127.0.0.1"); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::out_of_reach); + EXPECT_THAT(notes(results[0]), HasSubstr("elsewhere.example.test")); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(Probe, SaysWhichAddressAnswered) +{ + site.dns().add_https("localhost", 1, ".", {alpn_h3, port_param(site.port())}); + site.dns().add_https("localhost", 2, ".", h2_at(site.port())); + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + ASSERT_EQ(results[0].outcome, Result::Outcome::answered) << notes(results[0]); + EXPECT_TRUE(make_address(results[0].address).is_loopback()) << results[0].address; +} + +TEST_F(Probe, SaysWhichAddressAnsweredOverTcp) +{ + site.dns().add_https("localhost", 1, ".", h2_at(site.port())); + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + ASSERT_EQ(results[0].outcome, Result::Outcome::answered) << notes(results[0]); + EXPECT_TRUE(make_address(results[0].address).is_loopback()) << results[0].address; +} + +// +// No other resolver knows this name, so that getting as far as the certificate, which is not for +// this name, is what it takes to know that the addresses came from the DNS server. +// +TEST_F(Probe, ConnectsToTheAddressInDnsAndSaysWhichOne) +{ + site.dns().add_https("site.example.test", 1, ".", h2_at(site.port())); + site.dns().add_address("site.example.test", "127.0.0.1"); + + probe(config({"site.example.test"})); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::failed); + EXPECT_EQ(results[0].address, "127.0.0.1"); + EXPECT_THAT(results[0].error, HasSubstr("connect")); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(Probe, ConnectsToAnIPv6AddressInDns) +{ + site.dns().add_https("site.example.test", 1, ".", h2_at(site.port())); + site.dns().add_address("site.example.test", "::1"); + + probe(config({"site.example.test"})); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::failed); + EXPECT_EQ(results[0].address, "::1"); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(Probe, TriesTheNextAddressWhenAConnectionFails) +{ + site.dns().add_https("site.example.test", 1, ".", h2_at(site.port())); + site.dns().add_address("site.example.test", "127.0.0.1"); + site.dns().add_address("site.example.test", "::1"); + + probe(config({"site.example.test"})); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::failed); + EXPECT_THAT(notes(results[0]), HasSubstr("at 127.0.0.1")); + EXPECT_THAT(notes(results[0]), HasSubstr("at ::1")); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +// +// This one would hold without prefer_ipv6(), as c-ares puts ::1 ahead of 127.0.0.1 by itself. An +// address that it would put behind IPv4 is not on loopback, so the order is tested on its own +// below. +// +TEST_F(Probe, TriesIPv6First) +{ + site.dns().add_https("site.example.test", 1, ".", h2_at(site.port())); + site.dns().add_address("site.example.test", "127.0.0.1"); + site.dns().add_address("site.example.test", "::1"); + + probe(config({"site.example.test"})); + + ASSERT_EQ(results.size(), 1u); + const auto text = notes(results[0]); + ASSERT_THAT(text, HasSubstr("at ::1")); + ASSERT_THAT(text, HasSubstr("at 127.0.0.1")); + EXPECT_LT(text.find("at ::1"), text.find("at 127.0.0.1")); + EXPECT_EQ(results[0].address, "127.0.0.1"); // the last one tried +} + +corosio::endpoint at(const char* address) { return {make_address(address), 443}; } + +TEST(PreferIpv6, MovesIPv6ToTheFrontAndKeepsTheOrderWithinEachFamily) +{ + std::vector addresses{at("192.0.2.1"), at("2001:db8::1"), at("192.0.2.2"), at("2001:db8::2")}; + + prefer_ipv6(addresses); + + EXPECT_THAT(addresses, + ElementsAre(at("2001:db8::1"), at("2001:db8::2"), at("192.0.2.1"), at("192.0.2.2"))); +} + +TEST(PreferIpv6, LeavesASingleFamilyAlone) +{ + std::vector addresses{at("192.0.2.2"), at("192.0.2.1")}; + + prefer_ipv6(addresses); + + EXPECT_THAT(addresses, ElementsAre(at("192.0.2.2"), at("192.0.2.1"))); +} + +TEST_F(Probe, DoesNotConnectWithoutAnAddress) +{ + site.dns().add_https("site.example.test", 1, ".", h2_at(site.port())); + + probe(config({"site.example.test"})); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::lookup_failed); + EXPECT_THAT(results[0].address, IsEmpty()); + EXPECT_THAT(notes(results[0]), HasSubstr("address lookup")); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(Probe, RefusesACertificateItDoesNotTrust) +{ + Site other(context.get_executor()); // has a certificate of its own + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.page("/", {.body = "hello"}); + + auto configuration = config(); + configuration.tls_ca_file = other.ca_file().string(); + other.stop(); + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::failed); + EXPECT_THAT(results[0].error, HasSubstr("connect")); + EXPECT_THAT(site.requests(), IsEmpty()) << "no request survives the handshake"; +} + +TEST_F(Probe, RefusesACertificateItDoesNotTrustOverTcp) +{ + Site other(context.get_executor()); + site.dns().add_https("localhost", 1, ".", h2_at(site.port())); + site.page("/", {.body = "hello"}); + + auto configuration = config(); + configuration.tls_ca_file = other.ca_file().string(); + other.stop(); + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::failed); + EXPECT_THAT(results[0].error, HasSubstr("connect")); + EXPECT_THAT(site.requests(), IsEmpty()) << "no request survives the handshake"; +} + +TEST_F(Probe, FollowsAnAliasToItsRecords) +{ + // + // At the end of the chain it is the name of the alias's target that counts. So this is a route + // only for an alias that leads back to the host, by way of another name. + // + site.dns().add_https("localhost", 0, "alias.example.test"); + site.dns().add_https("alias.example.test", 1, "localhost", h3_at(site.port())); + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 200) << notes(results[0]); + EXPECT_EQ(site.dns_queries("alias.example.test"), 1u); +} + +TEST_F(Probe, RecognizesAnAliasLoop) +{ + site.dns().add_https("localhost", 0, "alias.example.test"); + site.dns().add_https("alias.example.test", 0, "localhost"); + site.page("/", {.body = "hello"}); + + probe(config()); + + // + // As if it had no record at all (section 3.1), which leaves the default port. + // + ASSERT_EQ(results.size(), 1u); + EXPECT_THAT(notes(results[0]), HasSubstr("too long or has a loop")); + EXPECT_EQ(results[0].outcome, Result::Outcome::answered) << notes(results[0]); + EXPECT_EQ(results[0].route, std::format("h2 localhost:{}", site.port())); +} + +TEST_F(Probe, IgnoresServiceRecordsNextToAnAlias) +{ + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.dns().add_https("localhost", 0, "alias.example.test"); + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::out_of_reach) + << "the alias has no records of its own"; + EXPECT_TRUE(results[0].aliased); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(Probe, TakesAHostsWordForItThatTheServiceIsNotAvailable) +{ + site.dns().add_https("localhost", 0, "."); + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::out_of_reach); + EXPECT_THAT(notes(results[0]), HasSubstr("not available")); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +// ================================================================================================= +// What can be seen of it +// ================================================================================================= + +TEST_F(Probe, LogsTheHttpAndTheLookupsAtDebugLevel) +{ + // + // What -v is for. The wording is up to the libraries, so this is only about it being there. + // + std::ostringstream log; + const auto previous = spdlog::default_logger(); + spdlog::set_default_logger(std::make_shared( + "test", std::make_shared(log))); + spdlog::set_level(spdlog::level::debug); + + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.page("/", {.body = "hello"}); + probe(config()); + + spdlog::set_default_logger(previous); + quiet(); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 200); + + EXPECT_THAT(log.str(), HasSubstr("asking for the HTTPS records of 'localhost'")) << "our own"; + EXPECT_THAT(log.str(), HasSubstr("GET https://localhost/")) << "anyhttp's request"; + EXPECT_THAT(log.str(), HasSubstr("content-type")) << "anyhttp's response"; +} + +// ================================================================================================= +// What can go wrong +// ================================================================================================= + +TEST_F(Probe, GivesUpOnAServerThatDoesNotAnswer) +{ + auto silent = silent_udp_socket(); + site.dns().add_https("localhost", 1, ".", h3_at(silent.local_endpoint().port())); + + auto configuration = config(); + configuration.timeout = 100ms; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::failed); + EXPECT_THAT(results[0].error, HasSubstr("timed out")); +} + +TEST_F(Probe, GivesUpOnATcpServerThatDoesNotAnswer) +{ + // + // It takes connections, as the kernel does that for it, and leaves the handshake hanging. + // + auto silent = silent_tcp_acceptor(); + site.dns().add_https("localhost", 1, ".", h2_at(silent.local_endpoint().port())); + + auto configuration = config(); + configuration.timeout = 100ms; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::failed); + EXPECT_THAT(results[0].error, HasSubstr("timed out")); +} + +TEST_F(Probe, ReportsAPortThatIsClosed) +{ + std::uint16_t closed = 0; + { + auto acceptor = silent_tcp_acceptor(); + closed = acceptor.local_endpoint().port(); + } + site.dns().add_https("localhost", 1, ".", h2_at(closed)); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::failed); + EXPECT_THAT(results[0].error, HasSubstr("connect")); +} + +TEST_F(Probe, ReportsANameThatDoesNotExist) +{ + probe(config({"nowhere.example.test"})); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::not_found); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(Probe, ReportsAFailedLookup) +{ + site.dns().set_rcode(ARES_RCODE_SERVFAIL); + site.page("/", {.body = "hello"}); + + probe(config()); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::lookup_failed) << "not the same as no HTTP/3"; + EXPECT_THAT(notes(results[0]), HasSubstr("HTTPS lookup")); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +// ================================================================================================= +// --h3 +// ================================================================================================= + +TEST_F(Probe, H3OnlyRequestsAHostWithARecordThatOffersIt) +{ + site.dns().add_https("localhost", 1, ".", {alpn_h3, port_param(site.port())}); + site.page("/", {.body = "hello"}); + + auto configuration = config(); + configuration.h3_only = true; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::answered) << notes(results[0]); + EXPECT_EQ(results[0].route, std::format("h3 localhost:{}", site.port())); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, H3OnlyLeavesOutAHostWithoutARecord) +{ + site.dns().add_address("localhost", "127.0.0.1"); + site.page("/", {.body = "hello"}); + + auto configuration = config(); + configuration.h3_only = true; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::no_h3); + EXPECT_THAT(results[0].address, IsEmpty()); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(Probe, H3OnlyLeavesOutAHostWhoseRecordsDoNotOfferIt) +{ + site.dns().add_https("localhost", 1, ".", h2_at(site.port())); + site.page("/", {.body = "hello"}); + + auto configuration = config(); + configuration.h3_only = true; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::no_h3); + EXPECT_EQ(results[0].records.size(), 1u) << "for --all to say why"; + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(Probe, H3OnlyStillFallsBackToTcpWhenAConnectionFails) +{ + auto silent = silent_udp_socket(); + + site.dns().add_https("localhost", 1, ".", h3_at(silent.local_endpoint().port())); + site.dns().add_https("localhost", 2, ".", h2_at(site.port())); + site.page("/", {.body = "hello"}); + + auto configuration = config(); + configuration.timeout = 200ms; + configuration.h3_only = true; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].status, 200) << notes(results[0]); + EXPECT_EQ(results[0].route, std::format("h2 localhost:{}", site.port())); + EXPECT_THAT(site.targets(), ElementsAre("/")); +} + +TEST_F(Probe, H3OnlyStillSaysThatARecordLeadsElsewhere) +{ + site.dns().add_https("localhost", 1, "elsewhere.example.test", h3_at(site.port())); + + auto configuration = config(); + configuration.h3_only = true; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::out_of_reach); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(Probe, H3OnlyStillSaysThatANameDoesNotExist) +{ + auto configuration = config({"nowhere.example.test"}); + configuration.h3_only = true; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::not_found); +} + +TEST_F(Probe, H3OnlyDoesNotTakeAFailedLookupForNoHttp3) +{ + site.dns().set_rcode(ARES_RCODE_SERVFAIL); + + auto configuration = config(); + configuration.h3_only = true; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::lookup_failed); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +// ================================================================================================= +// --dot +// ================================================================================================= + +namespace +{ + +const test_certificate& dns_certificate() +{ + static const auto certificate = make_test_certificate("dns.test"); + return certificate; +} + +class DnsOverTls : public Probe +{ +protected: + void SetUp() override + { + site.dns().use_tls(dns_certificate()); + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.page("/", {.body = "hello"}); + } + + void TearDown() override + { + ::unsetenv("SSL_CERT_FILE"); + if (!trusted.empty()) + std::filesystem::remove(trusted); + } + + void stop_sites() override + { + site.stop(); + names.stop(); + } + + // + // The resolver verifies against the default trust store, which is the one thing that it lets + // anybody say something about. --ca-file does not come into it. + // + void trust_the_dns_server() + { + trusted = std::filesystem::temp_directory_path() / + std::format("http-probe-test-dns-{}.pem", ::getpid()); + std::ofstream(trusted) << dns_certificate().certificate_pem; + ASSERT_EQ(::setenv("SSL_CERT_FILE", trusted.c_str(), 1), 0); + } + + Config config_over_tls(std::string hostname, bool verify = true) + { + auto configuration = config(); + configuration.dns_tls = {.enabled = true, .hostname = std::move(hostname), .verify = verify}; + return configuration; + } + + /// A server of \p site, by the name that \ref names has for it. + Config config_by_name(const Site& server) + { + names.dns().add_address("dns.test", "127.0.0.1"); + + auto configuration = config(); + const auto address = server.dns_address(); + configuration.dns_server = "dns.test" + address.substr(address.rfind(':')); + configuration.dns_server_lookup = names.dns_address(); + return configuration; + } + + Site names{context.get_executor()}; // what the system's resolver is to a host name of a server + std::filesystem::path trusted; +}; + +} // namespace + +TEST_F(DnsOverTls, AsksOverTlsAndNothingElse) +{ + trust_the_dns_server(); + probe(config_over_tls("dns.test")); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::answered) << notes(results[0]); + EXPECT_EQ(results[0].status, 200); + EXPECT_GE(site.dns_queries(fake_dns_server::transport::tls), 1u); + EXPECT_EQ(site.dns_queries(fake_dns_server::transport::udp), 0u); + EXPECT_EQ(site.dns_queries(fake_dns_server::transport::tcp), 0u); +} + +TEST_F(DnsOverTls, DoesNotConnectIfTheCertificateIsForAnotherName) +{ + trust_the_dns_server(); + probe(config_over_tls("other.test")); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::lookup_failed); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(DnsOverTls, DoesNotConnectIfTheCertificateIsNotTrusted) +{ + probe(config_over_tls("dns.test")); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::lookup_failed); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(DnsOverTls, SkipsTheCheckIfToldTo) +{ + probe(config_over_tls("", false)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::answered) << notes(results[0]); + EXPECT_GE(site.dns_queries(fake_dns_server::transport::tls), 1u); +} + +TEST_F(DnsOverTls, StillVerifiesTheHostsWhenToldToSkipTheCheck) +{ + Site other(context.get_executor()); // has a certificate of its own + + auto configuration = config_over_tls("", false); + configuration.tls_ca_file = other.ca_file().string(); + other.stop(); + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::failed); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(DnsOverTls, TakesTheNameToVerifyFromTheDnsServer) +{ + trust_the_dns_server(); + + auto configuration = config_by_name(site); + configuration.dns_tls.enabled = true; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::answered) << notes(results[0]); + EXPECT_GE(site.dns_queries(fake_dns_server::transport::tls), 1u); + EXPECT_EQ(names.dns_queries("dns.test"), names.dns_queries(fake_dns_server::transport::udp)) + << "looking up the server is in the clear, and it is all that the other one was asked"; + EXPECT_GE(names.dns_queries("dns.test"), 1u); +} + +TEST_F(DnsOverTls, PrefersTheNameItIsGivenToTheOneOfTheDnsServer) +{ + trust_the_dns_server(); + + auto configuration = config_by_name(site); + configuration.dns_tls = {.enabled = true, .hostname = "other.test", .verify = true}; + probe(std::move(configuration)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::lookup_failed); +} + +TEST_F(DnsOverTls, TakesADnsServerByNameWithoutDot) +{ + names.dns().add_https("localhost", 1, ".", h3_at(site.port())); + probe(config_by_name(names)); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].outcome, Result::Outcome::answered) << notes(results[0]); + EXPECT_EQ(site.dns_queries(fake_dns_server::transport::tls), 0u); +} + +TEST_F(DnsOverTls, EndsTheRunIfADnsServerCanNotBeLookedUp) +{ + auto configuration = config(); + configuration.dns_server = "missing.test"; + configuration.dns_server_lookup = names.dns_address(); + + EXPECT_THROW(probe(std::move(configuration)), std::runtime_error); + EXPECT_THAT(results, IsEmpty()); + EXPECT_THAT(site.requests(), IsEmpty()); +} + +TEST_F(DnsOverTls, RefusesAnAddressWithoutANameToVerify) +{ + EXPECT_THROW(probe(config_over_tls("")), std::invalid_argument); + finish(); +} + +TEST_F(DnsOverTls, RefusesSeveralNamesWithoutOneToVerify) +{ + auto configuration = config_over_tls(""); + configuration.dns_server = "dns1.test,dns2.test"; + EXPECT_THROW(probe(std::move(configuration)), std::invalid_argument); + finish(); +} + +TEST_F(DnsOverTls, RefusesToVerifyWithoutDot) +{ + auto configuration = config(); + configuration.dns_tls.hostname = "dns.test"; + EXPECT_THROW(probe(std::move(configuration)), std::invalid_argument); + finish(); +} + +TEST_F(Probe, RefusesADnsServerThatIsNoServer) +{ + auto configuration = config(); + configuration.dns_server = "not a server"; + EXPECT_THROW(probe(std::move(configuration)), std::invalid_argument); + finish(); +} + +// +// The connection stays open for the next query and the server has no reason to close it, so the +// program would never end if that was keeping the event loop going. This takes a context that has +// nothing else to do, unlike the one of the others, and with that a server on a thread of its own, +// which is only looked at when it is done. +// +TEST(DnsOverTlsConnection, DoesNotKeepTheProgramFromEnding) +{ + spdlog::set_level(spdlog::level::off); + + corosio::io_context site_context; + Site site(site_context.get_executor()); + site.dns().use_tls(dns_certificate()); + site.dns().add_https("localhost", 1, ".", h3_at(site.port())); + site.page("/", {.body = "hello"}); + + std::size_t answered = 0; + { + std::jthread serving([&] { site_context.run(); }); + + corosio::io_context context; + Prober prober( + context.get_executor(), + {.hosts = {"localhost"}, + .timeout = 1000ms, + .dns_server = site.dns_address(), + .dns_tls = {.enabled = true, .hostname = "", .verify = false}, + .tls_ca_file = site.ca_file().string(), + .default_port = site.port()}, + [&](const Result& result) { answered += result.outcome == Result::Outcome::answered; }); + capy::run_async(context.get_executor())(prober.run()); + context.run(); // what hangs if it does not end, for the timeout of the test to find + + const auto stop = [](Site& site) -> capy::task<> { + site.stop(); + co_return; + }; + capy::run_async(site_context.get_executor())(stop(site)); + } + EXPECT_EQ(answered, 1u); +} diff --git a/test/routes_test.cpp b/test/routes_test.cpp new file mode 100644 index 0000000..068d9ea --- /dev/null +++ b/test/routes_test.cpp @@ -0,0 +1,324 @@ +// +// What the records of a name mean for how to connect, without any I/O. +// +#include "routes.hpp" + +#include +#include + +using namespace http_probe; +using testing::ElementsAre; +using testing::HasSubstr; +using testing::IsEmpty; + +namespace +{ + +c_ares_async::https_record record(unsigned short priority, std::string target, + std::vector params = {}) +{ + return {.priority = priority, .target = std::move(target), .params = std::move(params)}; +} + +ServiceRecords found(std::vector records, + std::string owner = "example.com", bool aliased = false) +{ + return {.owner = std::move(owner), .aliased = aliased, .records = std::move(records)}; +} + +std::vector routes(const Plan& plan) +{ + std::vector result; + for (const auto& route : plan.routes) + result.push_back(route.to_string()); + return result; +} + +} // namespace + +// ================================================================================================= + +TEST(NormalizeHost, ComparesHostsTheWayDnsDoes) +{ + EXPECT_EQ(normalize_host("EXAMPLE.com."), "example.com"); +} + +TEST(ToText, ReadsLikeAZoneFile) +{ + EXPECT_EQ(to_text(record(1, ".", {"alpn=h3", "port=8443"})), "HTTPS 1 . alpn=h3 port=8443"); + EXPECT_EQ(to_text(record(0, "other.example")), "HTTPS 0 other.example"); + EXPECT_EQ(to_text(record(1, "")), "HTTPS 1 ."); +} + +// ------------------------------------------------------------------------------------------------- + +TEST(Plan, ConnectsOverHttp3FirstAndThenOverTcp) +{ + const auto plan = plan_routes("example.com", found({record(1, ".", {"alpn=h3,h2"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443", "h2 example.com:443")); +} + +TEST(Plan, TakesTheDefaultSetIntoAccount) +{ + // + // "alpn=h3" alone does not mean HTTP/3 alone: the default set is still there (section 7.1.2). + // + const auto plan = plan_routes("example.com", found({record(1, ".", {"alpn=h3"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443", "h1 example.com:443")); +} + +TEST(Plan, HasNoTcpWithNoDefaultAlpn) +{ + const auto plan = + plan_routes("example.com", found({record(1, ".", {"alpn=h3", "no-default-alpn"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443")); +} + +TEST(Plan, ConnectsOverTcpWhereTheRecordOffersNoHttp3) +{ + EXPECT_THAT(routes(plan_routes("example.com", found({record(1, ".", {"alpn=h2"})}))), + ElementsAre("h2 example.com:443")); + EXPECT_THAT(routes(plan_routes("example.com", found({record(1, ".", {"alpn=http/1.1"})}))), + ElementsAre("h1 example.com:443")); + EXPECT_THAT(routes(plan_routes("example.com", found({record(1, ".")}))), + ElementsAre("h1 example.com:443")); +} + +TEST(Plan, GoesToThePortOfTheRecord) +{ + const auto plan = plan_routes("example.com", found({record(1, ".", {"alpn=h3", "port=8443"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:8443", "h1 example.com:8443")); +} + +TEST(Plan, TriesTheLowestPriorityFirst) +{ + const auto plan = + plan_routes("example.com", found({record(3, ".", {"alpn=h3", "no-default-alpn", "port=3"}), + record(1, ".", {"alpn=h3", "no-default-alpn", "port=1"}), + record(2, ".", {"alpn=h2", "no-default-alpn", "port=2"})})); + + EXPECT_THAT(routes(plan), + ElementsAre("h3 example.com:1", "h2 example.com:2", "h3 example.com:3")); +} + +TEST(Plan, DoesNotTryTheSameThingTwice) +{ + // + // Records that differ only in what is of no use here, like their address hints. + // + const auto plan = + plan_routes("example.com", found({record(1, ".", {"alpn=h3", "ipv4hint=192.0.2.1"}), + record(2, ".", {"alpn=h3", "ipv4hint=192.0.2.2"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443", "h1 example.com:443")); +} + +TEST(Plan, GoesOverTcpToTheDefaultPortWithoutARecord) +{ + EXPECT_THAT(routes(plan_routes("example.com", found({}))), ElementsAre("h2 example.com:443")); + EXPECT_THAT(plan_routes("example.com", found({})).rejected, IsEmpty()) << "nothing odd about it"; + EXPECT_THAT(routes(plan_routes("example.com", found({}), 8443)), + ElementsAre("h2 example.com:8443")); +} + +TEST(Plan, RecordsThatDoNotSayGoToTheDefaultPort) +{ + EXPECT_THAT(routes(plan_routes("example.com", found({record(1, ".", {"alpn=h2"})}), 8443)), + ElementsAre("h2 example.com:8443")); +} + +TEST(Plan, HasNoRouteForANameThatDoesNotExist) +{ + auto records = found({}); + records.not_found = true; + + const auto plan = plan_routes("example.com", records); + + EXPECT_THAT(plan.routes, IsEmpty()); + EXPECT_FALSE(plan.out_of_reach); +} + +TEST(Plan, HasNoRouteAfterALookupThatFailed) +{ + auto records = found({}); + records.lookup_failed = true; + + const auto plan = plan_routes("example.com", records); + + EXPECT_THAT(plan.routes, IsEmpty()); + EXPECT_FALSE(plan.out_of_reach); +} + +TEST(Plan, TakesAHostsWordForItThatTheServiceIsNotAvailable) +{ + auto records = found({}); + records.unavailable = true; + + const auto plan = plan_routes("example.com", records); + + EXPECT_THAT(plan.routes, IsEmpty()); + EXPECT_TRUE(plan.out_of_reach); +} + +TEST(Plan, RejectsATargetOtherThanTheHost) +{ + // + // The certificate has to be valid for the origin (section 9.1), and anyhttp would check it + // against what it connects to. + // + for (const auto* alpn : {"alpn=h3", "alpn=h2"}) + { + const auto plan = plan_routes("example.com", found({record(1, "cdn.example.net", {alpn})})); + + EXPECT_THAT(plan.routes, IsEmpty()); + ASSERT_EQ(plan.rejected.size(), 1u); + EXPECT_THAT(plan.rejected[0], HasSubstr("cdn.example.net")); + EXPECT_TRUE(plan.out_of_reach); + } +} + +TEST(Plan, UsesTheRecordItCanOfAMixedSet) +{ + // + // As facebook.com has it. + // + const auto plan = plan_routes( + "example.com", + found({record(1, ".", {"alpn=h2,h3"}), record(2, "fallback.example.net", {"alpn=h2,h3"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443", "h2 example.com:443")); + ASSERT_EQ(plan.rejected.size(), 1u); + EXPECT_THAT(plan.rejected[0], HasSubstr("fallback.example.net")); + EXPECT_FALSE(plan.out_of_reach); +} + +TEST(Plan, DoesNotTakeSomethingThatOnlyStartsWithH3) +{ + const auto plan = plan_routes("example.com", found({record(1, ".", {"alpn=h3-29,h2"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h2 example.com:443")); +} + +TEST(Plan, IgnoresWhatItDoesNotKnowUnlessItIsMandatory) +{ + // + // key13930 is real, seen on one of the hosts in c-ares-async's list. + // + const auto plan = + plan_routes("example.com", found({record(1, ".", {"alpn=h3", "key13930=01", "ech=dead"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443", "h1 example.com:443")); +} + +TEST(Plan, TakesAnExplicitTargetThatIsTheHost) +{ + const auto plan = plan_routes("example.com", found({record(1, "EXAMPLE.com.", {"alpn=h2"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h2 example.com:443")); +} + +TEST(Plan, TakesTheOwnNameOfARecordAtTheEndOfAnAliasAsThatName) +{ + // + // "." is the owner name, section 2.5.2, and that is not the host any more. + // + const auto plan = + plan_routes("example.com", found({record(1, ".", {"alpn=h3"})}, "other.example", true)); + + EXPECT_THAT(plan.routes, IsEmpty()); + EXPECT_TRUE(plan.out_of_reach); + EXPECT_THAT(plan.rejected.front(), HasSubstr("other.example")); +} + +TEST(Plan, SaysSoAboutTheTargetOfAnAlias) +{ + const auto plan = plan_routes("example.com", found({}, "other.example", true)); + + EXPECT_THAT(plan.routes, IsEmpty()); + EXPECT_TRUE(plan.out_of_reach); + ASSERT_EQ(plan.rejected.size(), 1u); + EXPECT_THAT(plan.rejected[0], HasSubstr("other.example")); +} + +TEST(Plan, TreatsARecordWithNoProtocolItSpeaksAsNoRecord) +{ + const auto plan = plan_routes("example.com", found({record(1, ".", {"no-default-alpn"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h2 example.com:443")); + EXPECT_THAT(plan.rejected.front(), HasSubstr("no protocol")); +} + +TEST(Plan, TreatsARecordWithAMandatoryKeyItDoesNotKnowAsNoRecord) +{ + // + // Section 8: what is left is the name without a record, and that goes for all that follow. + // + const auto plan = + plan_routes("example.com", + found({record(1, ".", {"mandatory=alpn,key65280", "alpn=h3", "key65280=01"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h2 example.com:443")); + EXPECT_THAT(plan.rejected.front(), HasSubstr("key65280")); +} + +TEST(Plan, TreatsARecordThatRequiresEchAsNoRecord) +{ + const auto plan = + plan_routes("example.com", found({record(1, ".", {"mandatory=ech", "alpn=h3", "ech=dead"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h2 example.com:443")); +} + +TEST(Plan, AcceptsMandatoryKeysItCanLeaveAlone) +{ + const auto plan = plan_routes( + "example.com", + found({record(1, ".", {"mandatory=alpn,ipv4hint", "alpn=h3", "ipv4hint=192.0.2.1"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h3 example.com:443", "h1 example.com:443")); +} + +TEST(Plan, TreatsARecordWithAnInvalidPortAsNoRecord) +{ + const auto plan = plan_routes("example.com", found({record(1, ".", {"alpn=h3", "port=x"})})); + + EXPECT_THAT(routes(plan), ElementsAre("h2 example.com:443")); +} + +TEST(HasH3Record, IsTrueForARecordThatOffersIt) +{ + EXPECT_TRUE(has_h3_record(found({record(1, ".", {"alpn=h3"})}))); + EXPECT_TRUE(has_h3_record(found({record(1, ".", {"alpn=h2,h3"})}))); + EXPECT_TRUE(has_h3_record(found({record(1, ".", {"alpn=h2"}), record(2, ".", {"alpn=h3"})}))); +} + +TEST(HasH3Record, DoesNotCareWhereTheRecordPoints) +{ + EXPECT_TRUE(has_h3_record(found({record(1, "other.example", {"alpn=h3"})}))); +} + +TEST(HasH3Record, IsFalseForAnythingElse) +{ + EXPECT_FALSE(has_h3_record(found({}))); + EXPECT_FALSE(has_h3_record(found({record(1, ".")}))); + EXPECT_FALSE(has_h3_record(found({record(1, ".", {"alpn=h2"})}))); + EXPECT_FALSE(has_h3_record(found({record(1, ".", {"alpn=h3-29,h2"})}))); +} + +TEST(HasH3Record, IgnoresARecordThatHasToBeIgnored) +{ + EXPECT_FALSE(has_h3_record(found({record(1, ".", {"alpn=h3", "mandatory=ech", "ech=dead"})}))); + EXPECT_FALSE(has_h3_record(found({record(1, ".", {"alpn=h3", "port=none"})}))); +} + +TEST(Plan, PassesOnWhatWentWrongInTheLookup) +{ + auto records = found({}); + records.notes.push_back("HTTPS lookup failed"); + + EXPECT_THAT(plan_routes("example.com", records).rejected.front(), "HTTPS lookup failed"); +} diff --git a/test/site.cpp b/test/site.cpp new file mode 100644 index 0000000..9ffa9bb --- /dev/null +++ b/test/site.cpp @@ -0,0 +1,175 @@ +#include "site.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include "routes.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace capy = boost::capy; +namespace corosio = boost::corosio; +namespace http = boost::beast::http; + +namespace http_probe::test +{ + +// ================================================================================================= + +Site::Site(const capy::any_executor& executor) : requests_(std::make_shared>()) +{ + auto directory = (std::filesystem::temp_directory_path() / "http-probe-test-XXXXXX").string(); + if (!mkdtemp(directory.data())) + throw std::runtime_error("mkdtemp"); + directory_ = directory; + certificate_file_ = directory_ / "certificate.pem"; + key_file_ = directory_ / "key.pem"; + + const auto certificate = make_test_certificate("localhost"); + std::ofstream(certificate_file_) << certificate.certificate_pem; + std::ofstream(key_file_) << certificate.key_pem; + + start(executor); + port_ = server_->local_endpoint().port(); + capy::run_async(executor, polling_.get_token())(poll_dns()); + + server_->on_request( + [this, requests = requests_](anyhttp::server::Request request, + anyhttp::server::Response response) -> capy::task<> { + requests->push_back(Request{ + .target = std::string(request.url().encoded_target()), + .authority = std::string(request.url().encoded_host_and_port()), + .user_agent = std::string(request.fields()[http::field::user_agent]), + .time = std::chrono::steady_clock::now(), + }); + + std::array buffer; + for (;;) + if (auto [ec, n] = co_await request.read_some(capy::make_buffer(buffer)); ec || n == 0) + break; + + const auto it = pages_.find(std::string(request.url().encoded_path())); + const auto page = it != pages_.end() ? it->second : Page{.status = 404}; + + if (page.delay.count() > 0) + std::ignore = co_await corosio::delay(page.delay); + + anyhttp::Fields fields; + fields.set(http::field::content_type, page.content_type); + if (!page.location.empty()) + fields.set(http::field::location, page.location); + + if (auto [ec] = co_await response.submit(page.status, std::move(fields)); ec) + co_return; + co_await response.write_eof(boost::asio::buffer(page.body)); // has no overload for Capy's + }); +} + +// +// Port 0 finds a free port for one protocol, and then the servers need the same number for the +// other: anyhttp's has TCP and UDP on one port, and so does the DNS server. With tests running at +// the same time, another one may have the other half by then, so this is tried a few times. +// +void Site::start(const capy::any_executor& executor) +{ + for (int attempt = 1;; ++attempt) + { + const auto try_again = [&](std::error_code ec) { + return ec == std::errc::address_in_use && attempt < 20; + }; + + try + { + // + // "::" takes IPv4 as well, and "localhost" may be either. + // + server_.emplace(executor, + anyhttp::server::Config{.listen_address = "::", + .port = 0, + .tls_certificate_chain = certificate_file_, + .tls_private_key = key_file_}); + dns_.emplace(dns_context_.get_executor()); + return; + } + catch (const std::system_error& ex) // from the server, on Corosio + { + if (!try_again(ex.code())) + throw; + } + catch (const boost::system::system_error& ex) // from the DNS server, on Asio + { + if (!try_again(ex.code())) + throw; + } + server_.reset(); + dns_.reset(); + } +} + +// +// A millisecond at most for the DNS server to get to a question, which nothing here would notice. +// Once stopped, this does not look at the site anymore, which may be gone by the time it gets here. +// +capy::task<> Site::poll_dns() +{ + const std::stop_token stopped = co_await capy::this_coro::stop_token; + while (!stopped.stop_requested()) + { + dns_context_.restart(); + dns_context_.poll(); + std::ignore = co_await corosio::delay(std::chrono::milliseconds(1)); + } +} + +Site::~Site() +{ + polling_.request_stop(); + std::error_code ignored; + std::filesystem::remove_all(directory_, ignored); +} + +std::vector Site::targets() const +{ + std::vector targets; + for (const auto& request : *requests_) + targets.push_back(request.target); + return targets; +} + +std::size_t Site::dns_queries(std::string_view name) const +{ + const auto& queries = dns_ ? dns_->queries() : queries_; + return std::ranges::count_if(queries, [&](const auto& query) { + return normalize_host(query.name) == normalize_host(name); + }); +} + +std::size_t Site::dns_queries(fake_dns_server::transport via) const +{ + const auto& queries = dns_ ? dns_->queries() : queries_; + return std::ranges::count(queries, via, &fake_dns_server::query::via); +} + +void Site::stop() +{ + server_.reset(); + if (dns_) + queries_ = dns_->queries(); + dns_.reset(); + polling_.request_stop(); +} + +// ================================================================================================= + +} // namespace http_probe::test diff --git a/test/site.hpp b/test/site.hpp new file mode 100644 index 0000000..c126cc3 --- /dev/null +++ b/test/site.hpp @@ -0,0 +1,103 @@ +// +// A web site with DNS of its own, all on loopback, for the program to be pointed at: an anyhttp +// server on one port, for every protocol, and a DNS server with whatever zone the test wants. The +// server has a certificate for "localhost". That name comes from /etc/hosts and not from the DNS +// server, which a test can only have the last word on with a name that the certificate is not for. +// +// The DNS server comes from the tests of c-ares-async, where it is on Asio whatever the runtime. So +// it has a context of its own, which a coroutine polls from the one thread that everything runs on: +// a server on a thread of its own would race whoever looks at what it was asked. +// +#pragma once + +#include + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace http_probe::test +{ + +// ================================================================================================= + +class Site +{ +public: + struct Page + { + unsigned status = 200; + std::string body; + std::string content_type = "text/html"; + std::string location; + std::chrono::milliseconds delay{}; ///< how long the server takes to get to it + }; + + /// One request that arrived. + struct Request + { + std::string target; + std::string authority; + std::string user_agent; + std::chrono::steady_clock::time_point time; + }; + + explicit Site(const boost::capy::any_executor& executor); + ~Site(); + + /// A path that has no page gets a 404. + void page(std::string path, Page page) { pages_[std::move(path)] = std::move(page); } + + std::uint16_t port() const { return port_; } + const std::filesystem::path& ca_file() const { return certificate_file_; } + + /// The zone. Good until stop(), as opposed to what was asked of it, see dns_queries(). + fake_dns_server& dns() { return *dns_; } + std::string dns_address() const { return dns_->address(); } + + /// How often the DNS server was asked about \p name, also after stop(). + std::size_t dns_queries(std::string_view name) const; + + /// How many questions came in over \p via, also after stop(). + std::size_t dns_queries(fake_dns_server::transport via) const; + + /// Everything the server was asked so far, in order, and all of the targets alone. + const std::vector& requests() const { return *requests_; } + std::vector targets() const; + + /// Ends the servers, which is what lets the context run dry. + void stop(); + +private: + void start(const boost::capy::any_executor& executor); + boost::capy::task<> poll_dns(); + + std::filesystem::path directory_; + std::filesystem::path certificate_file_; + std::filesystem::path key_file_; + std::map pages_; + std::shared_ptr> requests_; + std::optional server_; + boost::asio::io_context dns_context_; + std::optional dns_; + std::stop_source polling_; + std::vector queries_; // what the DNS server knew when it was stopped + std::uint16_t port_ = 0; +}; + +// ================================================================================================= + +} // namespace http_probe::test