From 951df7a2ce88a52cf5a34b93d32552709b203b0c Mon Sep 17 00:00:00 2001 From: Patrick Kastner <1919773+kastnerp@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:08:12 -0400 Subject: [PATCH] Agent readiness: robots.txt with AI rules and content signals, api-catalog, auth.md - robots.txt: explicit AI crawler groups, Content-Signal (search, ai-input, ai-train), and an absolute sitemap URL (it had a doubled or relative base). - /.well-known/api-catalog (RFC 9727; _config.yml include) listing the AIJ validation-data API. - /auth.md: the site needs no authentication. Co-Authored-By: Claude Sonnet 5.5 --- .well-known/api-catalog | 17 +++++++++++++++++ _config.yml | 2 +- auth.md | 18 ++++++++++++++++++ robots.txt | 28 ++++++++++++++++++++++++++-- 4 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 .well-known/api-catalog create mode 100644 auth.md diff --git a/.well-known/api-catalog b/.well-known/api-catalog new file mode 100644 index 0000000000000..14bb18e6f2998 --- /dev/null +++ b/.well-known/api-catalog @@ -0,0 +1,17 @@ +{ + "linkset": [ + { + "anchor": "https://aij.eddy3d.com/api/", + "title": "AIJ UWE benchmark API: cleaned wind-tunnel and field data, geometry and validation results (Sustainable Urban Systems Lab)", + "service-desc": [ + { "href": "https://aij.eddy3d.com/api/openapi.json", "type": "application/vnd.oai.openapi+json" } + ], + "service-doc": [ + { "href": "https://aij.eddy3d.com/api-reference/", "type": "text/html" } + ], + "describedby": [ + { "href": "https://aij.eddy3d.com/api/llms.txt", "type": "text/plain", "title": "Machine-readable summary of the API for agents" } + ] + } + ] +} diff --git a/_config.yml b/_config.yml index 0f794d74b7f34..910fd563cec13 100644 --- a/_config.yml +++ b/_config.yml @@ -234,7 +234,7 @@ kramdown: start_line: 1 # Includes & excludes -include: ["_pages"] +include: ["_pages", ".well-known"] exclude: - AGENTS.md - al-folio-upgrade-report.md diff --git a/auth.md b/auth.md new file mode 100644 index 0000000000000..201cb33c15ac2 --- /dev/null +++ b/auth.md @@ -0,0 +1,18 @@ +# auth.md: sustainableurbansystems.com + +**Audience:** AI agents and automated clients. + +**No authentication.** This is a public website (Sustainable Urban Systems Lab): research, publications, people and news. It needs no account, key or +token, and it has no API of its own. There is no registration or provisioning endpoint and no OAuth server, so there is no +protected-resource or authorization-server metadata to publish. + +## How to use it + +- Send plain `GET` requests. Honour [`/robots.txt`](https://sustainableurbansystems.com/robots.txt) and its content signals, and identify yourself + with a descriptive `User-Agent`. The sitemap is [`/sitemap.xml`](https://sustainableurbansystems.com/sitemap.xml). +- The one public data API connected to this work, the AIJ wind-benchmark data, is at and needs no + credentials either; it is listed in [`/.well-known/api-catalog`](https://sustainableurbansystems.com/.well-known/api-catalog). + +## Contact + + diff --git a/robots.txt b/robots.txt index a450fbe28faaf..3792eb9bef3fe 100644 --- a/robots.txt +++ b/robots.txt @@ -1,7 +1,31 @@ --- permalink: /robots.txt --- +# robots.txt for https://sustainableurbansystems.com/ (RFC 9309) +# Content signals: https://contentsignals.org/ + +# AI crawlers and agents: explicitly welcome. +User-agent: GPTBot +User-agent: OAI-SearchBot +User-agent: ChatGPT-User +User-agent: ClaudeBot +User-agent: Claude-Web +User-agent: Claude-User +User-agent: Claude-SearchBot +User-agent: anthropic-ai +User-agent: Google-Extended +User-agent: PerplexityBot +User-agent: Perplexity-User +User-agent: Applebot-Extended +User-agent: CCBot +User-agent: Meta-ExternalAgent +User-agent: Bytespider +Content-Signal: search=yes, ai-input=yes, ai-train=yes +Allow: / + +# Everyone else. User-agent: * -Disallow: +Content-Signal: search=yes, ai-input=yes, ai-train=yes +Allow: / -Sitemap: {{ site.baseurl | prepend: site.url }}/sitemap.xml +Sitemap: {{ '/sitemap.xml' | absolute_url }}