fix(docs): preserve indexed legacy URLs and canonical documentation

tests/test_pr_preview.py

@@ -47,14 +47,14 @@ def fake_sphinx(source: Path, output: Path, entry: dict,
                     catalog: dict, *, review_only: bool = False) -> None:
         assert review_only
         output.mkdir(parents=True)
-        (output / "index.html").write_text("Review manual")
+        (output / "index.html").write_text("<html><head></head><body>Review manual</body></html>")
         target = output / "webhooks"
         target.mkdir()
         (target / "index.html").write_text(
-            '<a href="new_event.html">Event</a>')
-        (target / "new_event.html").write_text("Native event")
+            '<html><head></head><body><a href="new_event.html">Event</a></body></html>')
+        (target / "new_event.html").write_text("<html><head></head><body>Native event</body></html>")
         (output / "api").mkdir()
-        (output / "api/openapi.html").write_text("OpenAPI")
+        (output / "api/openapi.html").write_text("<html><head></head><body>OpenAPI</body></html>")
 
     def test_future_pages_are_built_and_linked_without_filename_registration(self):
         with tempfile.TemporaryDirectory() as tmp:

tests/test_seo.py

@@ -0,0 +1,118 @@
+# SPDX-FileCopyrightText: 2026 LibreCode coop and contributors
+# SPDX-License-Identifier: CC-BY-3.0
+
+"""Regression coverage for permanent URLs and indexable documentation."""
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from versioning.build import legacy_redirects
+from versioning.seo import (
+    PUBLIC_ORIGIN, apply_seo, canonical_url, mark_preview_noindex,
+    read_legacy_routes, verify_seo,
+)
+
+
+class DocumentationSeoTests(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        self.root = Path(self.tmp.name)
+        self.site = self.root / "site"
+        self.site.mkdir()
+        self.catalog = {"versions": [{"slug": "15"}, {"slug": "main"}],
+                        "current_stable": "15"}
+
+    def page(self, name: str, body: str = "<h1>Article</h1>"):
+        path = self.site / name
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(f"<!doctype html><html><head><title>Documentation</title></head>"
+                        f"<body>{body}</body></html>", encoding="utf-8")
+        return path
+
+    def test_historic_routes_cover_all_existing_manual_pages(self):
+        root = Path(__file__).resolve().parent.parent
+        routes = read_legacy_routes(root)
+        source = {f"{manual}/{p.relative_to(root / manual).with_suffix('.html')}"
+                  for manual in ("user_manual", "admin_manual", "developer_manual")
+                  for p in (root / manual).rglob("*.rst")}
+        self.assertEqual(set(routes), source)
+        self.assertEqual(len(routes), 46)
+        self.assertEqual(routes["user_manual/functionality.html"],
+                         "main/user_manual/functionality.html")
+        self.assertEqual(routes["admin_manual/installation.html"],
+                         "main/admin_manual/installation.html")
+        self.assertEqual(routes["developer_manual/architecture.html"],
+                         "main/developer_manual/architecture.html")
+
+    def test_legacy_redirects_keep_individual_targets_across_new_stable(self):
+        self.page("15/index.html")
+        for name in ("15/user_manual/index.html",
+                     "main/user_manual/functionality.html"):
+            self.page(name)
+        (self.site / "versions.json").write_text(
+            json.dumps(self.catalog), encoding="utf-8")
+        self.page("index.html", '<div id="main-content"></div>')
+        with patch("versioning.build.version_overview", return_value=self.page("index.html").read_text()):
+            legacy_redirects(self.site, "15", routes={
+                "user_manual/index.html": "15/user_manual/index.html",
+                "user_manual/functionality.html": "main/user_manual/functionality.html",
+            })
+        redirect = (self.site / "user_manual/functionality.html").read_text()
+        self.assertIn("url=/main/user_manual/functionality.html", redirect)
+        self.assertNotIn("url=/15/user_manual/index.html", redirect)
+
+    def test_missing_redirect_destination_fails_before_deploy(self):
+        self.page("15/index.html")
+        (self.site / "versions.json").write_text(json.dumps(self.catalog))
+        with self.assertRaisesRegex(ValueError, "Missing historical destination"):
+            legacy_redirects(self.site, "15", routes={
+                "user_manual/functionality.html": "main/user_manual/missing.html"
+            })
+
+    def test_published_sitemap_and_canonical_are_exact(self):
+        self.page("index.html")
+        self.page("15/index.html")
+        self.page("15/user_manual/index.html")
+        self.page("15/user_manual/guide.html")
+        self.page("main/index.html")
+        self.page("15/user_manual/search.html")
+        self.page("15/user_manual/404.html")
+        apply_seo(self.site, self.catalog)
+        self.assertEqual(verify_seo(self.site, self.catalog), [])
+        self.assertIn(f"{PUBLIC_ORIGIN}/15/user_manual/guide.html",
+                      (self.site / "sitemap.xml").read_text())
+        self.assertNotIn("search.html", (self.site / "sitemap.xml").read_text())
+        self.assertIn('content="noindex,follow"',
+                      (self.site / "15/user_manual/search.html").read_text())
+
+    def test_preview_content_is_not_indexable(self):
+        self.page("index.html")
+        self.page("15/user_manual/guide.html")
+        self.page("_review/16/developer_manual/index.html")
+        apply_seo(self.site, self.catalog, preview=True)
+        mark_preview_noindex(self.site / "_review")
+        self.assertEqual(verify_seo(self.site, self.catalog, preview=True), [])
+        self.assertFalse((self.site / "sitemap.xml").exists())
+
+    def test_unsafe_and_generic_legacy_destinations_are_rejected(self):
+        source = self.root / "src"
+        (source / "versioned").mkdir(parents=True)
+        for bad in ("15/user_manual/index.html", "../../private.html"):
+            (source / "versioned/legacy-routes.json").write_text(json.dumps({
+                "user_manual/functionality.html": bad
+            }))
+            with self.assertRaises(ValueError):
+                read_legacy_routes(source)
+
+    def test_canonical_has_stable_directory_semantics(self):
+        self.assertEqual(canonical_url(Path("15/user_manual/index.html")),
+                         f"{PUBLIC_ORIGIN}/15/user_manual/")
+        self.assertEqual(canonical_url(Path("15/user_manual/guide.html")),
+                         f"{PUBLIC_ORIGIN}/15/user_manual/guide.html")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_versioning.py

@@ -98,7 +98,8 @@ def test_updating_15_preserves_existing_13_14_and_main(self):
             (incoming / "versions.json").write_text(json.dumps(updated))
             (incoming / "15").mkdir()
             (incoming / "15/index.html").write_text("15-updated")
-            result_catalog = assemble(incoming, result, "15", published)
+            with patch("versioning.publish.legacy_redirects"), patch("versioning.publish.apply_seo"):
+                result_catalog = assemble(incoming, result, "15", published)
             self.assertEqual((result / "13/saved.html").read_text(), "do-not-erase")
             self.assertEqual((result / "14/index.html").read_text(), "14-original")
             self.assertEqual((result / "main/index.html").read_text(), "main-original")
@@ -204,7 +205,7 @@ def test_preview_review_link_is_valid_during_initial_site_verification(self):
             for filename in ("home.css", "home-theme.js", "logo.png"):
                 (root / filename).write_text("asset")
             with patch.dict("os.environ", {"LIBRESIGN_PREVIEW_PR": "127"}):
-                legacy_redirects(root, "15")
+                legacy_redirects(root, "15", routes={})
                 self.assertEqual(inspect_links(root, root / "index.html"), [])
             self.assertTrue((root / "_review/index.html").is_file())
 
@@ -219,7 +220,7 @@ def test_production_does_not_create_review_page(self):
             (root / "15/index.html").parent.mkdir()
             (root / "15/index.html").write_text("Stable")
             with patch.dict("os.environ", {"LIBRESIGN_PREVIEW_PR": ""}):
-                legacy_redirects(root, "15")
+                legacy_redirects(root, "15", routes={})
             self.assertFalse((root / "_review/index.html").exists())
             self.assertNotIn('href="_review/"', (root / "index.html").read_text())
 

versioned/legacy-routes.json

@@ -0,0 +1,48 @@
+{
+  "admin_manual/configuration.html": "main/admin_manual/configuration.html",
+  "admin_manual/index.html": "15/admin_manual/index.html",
+  "admin_manual/installation.html": "main/admin_manual/installation.html",
+  "admin_manual/maintenance.html": "main/admin_manual/maintenance.html",
+  "admin_manual/security.html": "main/admin_manual/security.html",
+  "developer_manual/api/external-email-signers.html": "15/developer_manual/api/external-email-signers.html",
+  "developer_manual/api/guide-api.html": "main/developer_manual/api/guide-api.html",
+  "developer_manual/api/index.html": "15/developer_manual/api/index.html",
+  "developer_manual/api/openapi.html": "15/developer_manual/api/openapi.html",
+  "developer_manual/architecture.html": "main/developer_manual/architecture.html",
+  "developer_manual/development-workflow.html": "main/developer_manual/development-workflow.html",
+  "developer_manual/error-handling.html": "main/developer_manual/error-handling.html",
+  "developer_manual/getting-started/branch-policies.html": "main/developer_manual/getting-started/branch-policies.html",
+  "developer_manual/getting-started/commits.html": "main/developer_manual/getting-started/commits.html",
+  "developer_manual/getting-started/development-environment/coding-standards.html": "main/developer_manual/getting-started/development-environment/coding-standards.html",
+  "developer_manual/getting-started/development-environment/frontend.html": "main/developer_manual/getting-started/development-environment/frontend.html",
+  "developer_manual/getting-started/development-environment/index.html": "main/developer_manual/getting-started/development-environment/index.html",
+  "developer_manual/getting-started/development-environment/setup.html": "main/developer_manual/getting-started/development-environment/setup.html",
+  "developer_manual/getting-started/development-environment/updating.html": "main/developer_manual/getting-started/development-environment/updating.html",
+  "developer_manual/getting-started/index.html": "main/developer_manual/getting-started/index.html",
+  "developer_manual/getting-started/tests.html": "main/developer_manual/getting-started/tests.html",
+  "developer_manual/index.html": "15/developer_manual/index.html",
+  "developer_manual/integration/application.html": "15/developer_manual/integration/application.html",
+  "developer_manual/integration/deployment.html": "15/developer_manual/integration/deployment.html",
+  "developer_manual/integration/identity.html": "15/developer_manual/integration/identity.html",
+  "developer_manual/integration/index.html": "15/developer_manual/integration/index.html",
+  "developer_manual/integration/lifecycle.html": "15/developer_manual/integration/lifecycle.html",
+  "developer_manual/prologue/code-of-conduct.html": "main/developer_manual/prologue/code-of-conduct.html",
+  "developer_manual/prologue/community-support.html": "main/developer_manual/prologue/community-support.html",
+  "developer_manual/prologue/index.html": "main/developer_manual/prologue/index.html",
+  "developer_manual/prologue/reporting-bugs.html": "main/developer_manual/prologue/reporting-bugs.html",
+  "developer_manual/release-process.html": "main/developer_manual/release-process.html",
+  "developer_manual/release-process/configuration.html": "main/developer_manual/release-process/configuration.html",
+  "developer_manual/release-process/manual.html": "main/developer_manual/release-process/manual.html",
+  "developer_manual/release-process/milestones.html": "main/developer_manual/release-process/milestones.html",
+  "developer_manual/release-process/preparing.html": "main/developer_manual/release-process/preparing.html",
+  "developer_manual/release-process/publishing.html": "main/developer_manual/release-process/publishing.html",
+  "developer_manual/release-process/reusable-tooling.html": "main/developer_manual/release-process/reusable-tooling.html",
+  "developer_manual/release-process/versioning.html": "main/developer_manual/release-process/versioning.html",
+  "developer_manual/requesting-features.html": "main/developer_manual/requesting-features.html",
+  "developer_manual/translation.html": "main/developer_manual/translation.html",
+  "user_manual/faq.html": "main/user_manual/faq.html",
+  "user_manual/functionality.html": "main/user_manual/functionality.html",
+  "user_manual/index.html": "15/user_manual/index.html",
+  "user_manual/start.html": "main/user_manual/start.html",
+  "user_manual/suport.html": "main/user_manual/suport.html"
+}

versioning/build.py

@@ -148,7 +148,7 @@ def version_overview(catalog: dict) -> str:
     )
 
 
-def legacy_redirects(output: Path, current: str) -> None:
+def legacy_redirects(output: Path, current: str, *, routes: dict[str, str] | None = None) -> None:
     destination = output / current
     if not (destination / "index.html").is_file():
         raise ValueError("Cannot create legacy redirects without stable landing page")
@@ -168,26 +168,19 @@ def legacy_redirects(output: Path, current: str) -> None:
             "<p>Release-specific review pages are generated after validation.</p>",
             encoding="utf-8",
         )
-    # Previous URLs did not include a version prefix. Preserve page paths
-    # when the same page exists in current stable; otherwise go to that
-    # manual's landing page, never silently to a development-only page.
-    for manual in MANUALS[1:]:
-        base = output / manual
-        base.mkdir(parents=True, exist_ok=True)
-        main_source = output / "main" / manual
-        for page in main_source.rglob("*.html"):
-            relative = page.relative_to(main_source)
-            target = destination / manual / relative
-            if not target.is_file():
-                target = destination / manual / "index.html"
-            redirect = base / relative
-            redirect.parent.mkdir(parents=True, exist_ok=True)
-            redirect.write_text(
-                redirect_html("/" + str(target.relative_to(output)).replace(os.sep, "/")),
-                encoding="utf-8",
-            )
-        (base / "index.html").write_text(
-            redirect_html(f"/{current}/{manual}/"), encoding="utf-8",
+    # The previous unversioned URLs are a permanent migration interface.
+    # Do not move their destinations when a future stable line is published.
+    from versioning.seo import read_legacy_routes
+
+    for old, target in (read_legacy_routes(ROOT) if routes is None else routes).items():
+        destination_page = output / target
+        if not destination_page.is_file():
+            raise ValueError(f"Missing historical destination: {old} => {target}")
+        legacy_page = output / old
+        legacy_page.parent.mkdir(parents=True, exist_ok=True)
+        legacy_page.write_text(
+            redirect_html("/" + target),
+            encoding="utf-8",
         )
 
 

versioning/pr_preview.py

@@ -16,6 +16,7 @@
 
 from versioning.build import ROOT, run_sphinx, source_tree
 from versioning.verify import inspect_links
+from versioning.seo import mark_preview_noindex
 
 PREVIEW_MARKER = "<!-- libresign-documentation-preview-links -->"
 REVIEW_MANUALS = ("user_manual", "admin_manual", "developer_manual")
@@ -138,6 +139,7 @@ def preview_unreleased(site: Path, preview_root: str) -> list[str]:
             local /= "index.html"
         if not local.is_file():
             raise ValueError(f"Preview route is missing: {route}")
+    mark_preview_noindex(review)
     (review / "routes.txt").write_text(
         "\n".join(unique_routes) + "\n", encoding="utf-8"
     )

versioning/publish.py

@@ -3,10 +3,12 @@
 
 import argparse
 import json
+import os
 import shutil
 from pathlib import Path
 
 from versioning.build import legacy_redirects
+from versioning.seo import apply_seo
 
 ROOT = Path(__file__).resolve().parent.parent
 
@@ -62,6 +64,7 @@ def assemble(built: Path, destination: Path, selected: str,
         legacy_redirects(destination, output_catalog["current_stable"])
     (destination / "CNAME").write_text("docs.libresign.coop\n", encoding="utf-8")
     (destination / ".nojekyll").touch()
+    apply_seo(destination, output_catalog, preview=bool(os.environ.get("LIBRESIGN_PREVIEW_PR")))
     return output_catalog