🐛 Preserve snippet variants when cleaning up docs sources

pyproject.toml

@@ -90,6 +90,7 @@ tests = [
     "psycopg[binary] >=3.2.10",
     "pymysql[rsa] >=1.1.1",
     "pytest >=7.0.1",
+    "pyyaml >=5.3.1",
     "ruff >=0.16.0",
     "ty>=0.0.25",
     "typer >=0.26.1",

scripts/docs.py

@@ -271,99 +271,54 @@ def update_docs_includes_py39_to_py310() -> None:
 @app.command()
 def remove_unused_docs_src() -> None:
     """
-    Delete .py files in docs_src that are not included in any .md file under docs/.
+    Delete unused Python snippets, keeping their variants and supporting package files.
     """
     docs_src_path = Path("docs_src")
-    # Collect all docs .md content referencing docs_src
-    all_docs_content = ""
-    for md_file in docs_path.rglob("*.md"):
-        all_docs_content += md_file.read_text(encoding="utf-8")
-    # Build a set of directory-based package roots (e.g. docs_src/bigger_applications/app_py39)
-    # where at least one file is referenced in docs. All files in these directories
-    # should be kept since they may be internally imported by the referenced files.
-    used_package_dirs: set[Path] = set()
-    for py_file in docs_src_path.rglob("*.py"):
-        if py_file.name == "__init__.py":
-            continue
-        rel_path = str(py_file)
-        if rel_path in all_docs_content:
-            parts = py_file.relative_to(docs_src_path).parts
-            if len(parts) > 2 and not py_file.name.startswith("tutorial"):
-                # File is inside a package directory (e.g.
-                # docs_src/tutorial/fastapi/app_testing/tutorial001_py310/).
-                # Mark the immediate parent as a used package so sibling
-                # files (likely imported by the referenced file) are kept.
-                used_package_dirs.add(py_file.parent)
+    all_docs_content = "\n".join(
+        md_file.read_text(encoding="utf-8") for md_file in docs_path.rglob("*.md")
+    )
+    variant_suffix = re.compile(r"(?:_non_an|_an)?(?:_py\d+)?$")
+
+    def variant_key(path: Path) -> tuple[str, ...]:
+        return tuple(variant_suffix.sub("", part, count=1) for part in path.parts)
+
+    def package_key(parts: tuple[str, ...]) -> tuple[str, ...] | None:
+        # Package examples use tutorial001 directories at different topic depths.
+        for index, part in enumerate(parts):
+            if re.fullmatch(r"tutorial\d+\w*", part):
+                return parts[: index + 1]
+        return None
+
+    file_keys = {
+        py_file: variant_key(py_file.relative_to(docs_src_path).with_suffix(""))
+        for py_file in docs_src_path.rglob("*.py")
+    }
+    used_files = {
+        key
+        for py_file, key in file_keys.items()
+        if py_file.as_posix() in all_docs_content
+    }
+    # Keep entire referenced packages, including their variants and modules
+    # imported by the referenced file, without keeping unrelated tutorial packages.
+    used_package_dirs = {
+        package for key in used_files if (package := package_key(key[:-1])) is not None
+    }
     removed = 0
-    for py_file in sorted(docs_src_path.rglob("*.py")):
-        if py_file.name == "__init__.py":
+    for py_file, key in sorted(file_keys.items()):
+        if py_file.name == "__init__.py" or key in used_files:
             continue
-        # Build the relative path as it appears in includes (e.g. docs_src/first_steps/tutorial001.py)
-        rel_path = str(py_file)
-        if rel_path in all_docs_content:
-            continue
-        # If this file is inside a directory-based package where any sibling is
-        # referenced, keep it (it's likely imported internally).
-        if py_file.parent in used_package_dirs:
-            continue
-        # Check if the _an counterpart (or non-_an counterpart) is referenced.
-        # If either variant is included, keep both.
-        # Handle both file-level _an (tutorial001_an.py) and directory-level _an
-        # (app_an/main.py)
-        counterpart_found = False
-        full_path_str = str(py_file)
-        if "_an" in py_file.stem:
-            # This is an _an file, check if the non-_an version is referenced
-            counterpart = full_path_str.replace(
-                f"/{py_file.stem}", f"/{py_file.stem.replace('_an', '', 1)}"
-            )
-            if counterpart in all_docs_content:
-                counterpart_found = True
-        else:
-            # This is a non-_an file, check if there's an _an version referenced
-            # Insert _an before any version suffix or at the end of the stem
-            stem = py_file.stem
-            for suffix in ("_py39", "_py310"):
-                if suffix in stem:
-                    an_stem = stem.replace(suffix, f"_an{suffix}", 1)
-                    break
-            else:
-                an_stem = f"{stem}_an"
-            counterpart = full_path_str.replace(f"/{stem}.", f"/{an_stem}.")
-            if counterpart in all_docs_content:
-                counterpart_found = True
-        # Also check directory-level _an counterparts
-        if not counterpart_found:
-            parent_name = py_file.parent.name
-            if "_an" in parent_name:
-                counterpart_parent = parent_name.replace("_an", "", 1)
-                counterpart_dir = str(py_file).replace(
-                    f"/{parent_name}/", f"/{counterpart_parent}/"
-                )
-                if counterpart_dir in all_docs_content:
-                    counterpart_found = True
-            else:
-                # Try inserting _an into parent directory name
-                for suffix in ("_py39", "_py310"):
-                    if suffix in parent_name:
-                        an_parent = parent_name.replace(suffix, f"_an{suffix}", 1)
-                        break
-                else:
-                    an_parent = f"{parent_name}_an"
-                counterpart_dir = str(py_file).replace(
-                    f"/{parent_name}/", f"/{an_parent}/"
-                )
-                if counterpart_dir in all_docs_content:
-                    counterpart_found = True
-        if counterpart_found:
+        if package_key(key[:-1]) in used_package_dirs:
             continue
         logging.info(f"Removing unused file: {py_file}")
         py_file.unlink()
         removed += 1
-    # Clean up directories that are empty or only contain __init__.py / __pycache__
+    # Preserve init-only subpackages inside used packages, and clean up unused ones.
     for dir_path in sorted(docs_src_path.rglob("*"), reverse=True):
         if not dir_path.is_dir():
             continue
+        key = variant_key(dir_path.relative_to(docs_src_path))
+        if package_key(key) in used_package_dirs:
+            continue
         remaining = [
             f
             for f in dir_path.iterdir()

tests/test_docs_cleanup.py

@@ -0,0 +1,123 @@
+from pathlib import Path
+
+import pytest
+
+from scripts.docs import remove_unused_docs_src
+
+
+def create_files(paths: set[str]) -> None:
+    for path_str in paths:
+        path = Path(path_str)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text("# Example code\n", encoding="utf-8")
+
+
+def include_file(path: str) -> None:
+    doc = Path("docs/en/docs/tutorial.md")
+    doc.parent.mkdir(parents=True, exist_ok=True)
+    doc.write_text("{* " + path + " *}\n", encoding="utf-8")
+
+
+@pytest.mark.parametrize(
+    "names",
+    [
+        ("tutorial001", "tutorial001_non_an"),
+        ("tutorial001", "tutorial001_an"),
+        ("tutorial001_py310", "tutorial001_non_an_py310"),
+        ("tutorial001_py310", "tutorial001_an_py310"),
+        (
+            "tutorial001_py310",
+            "tutorial001_an_py310",
+            "tutorial001_py314",
+            "tutorial001_an_py314",
+        ),
+        (
+            "tutorial001_py311",
+            "tutorial001_non_an_py311",
+            "tutorial001_py314",
+            "tutorial001_non_an_py314",
+        ),
+        ("tutorial001_py311", "tutorial001_py314"),
+        ("tutorial_another", "tutorial_another_non_an"),
+    ],
+)
+@pytest.mark.parametrize("preferred", [0, 1])
+def test_keep_file_variants(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    names: tuple[str, ...],
+    preferred: int,
+) -> None:
+    monkeypatch.chdir(tmp_path)
+    kept = {f"docs_src/tutorial/topic/{name}.py" for name in names}
+    unused = {
+        "docs_src/tutorial/topic/tutorial002.py",
+        "docs_src/tutorial/topic/tutorial001_extra.py",
+    }
+    create_files(kept | unused)
+    include_file(f"docs_src/tutorial/topic/{names[preferred]}.py")
+
+    remove_unused_docs_src()
+
+    assert {path.as_posix() for path in Path("docs_src").rglob("*.py")} == kept
+
+
+@pytest.mark.parametrize(
+    "names",
+    [
+        ("tutorial001", "tutorial001_non_an"),
+        ("tutorial001", "tutorial001_an"),
+        (
+            "tutorial001_py310",
+            "tutorial001_an_py310",
+            "tutorial001_py314",
+            "tutorial001_an_py314",
+        ),
+        (
+            "tutorial001_py311",
+            "tutorial001_non_an_py311",
+            "tutorial001_py314",
+            "tutorial001_non_an_py314",
+        ),
+    ],
+)
+@pytest.mark.parametrize("preferred", [0, 1])
+@pytest.mark.parametrize("included", ["main.py", "routers/items.py", "__init__.py"])
+@pytest.mark.parametrize(
+    "topic", ["tutorial/code_structure", "tutorial/fastapi/app_testing"]
+)
+def test_keep_entire_variant_packages(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    names: tuple[str, ...],
+    preferred: int,
+    included: str,
+    topic: str,
+) -> None:
+    monkeypatch.chdir(tmp_path)
+    kept = {
+        f"docs_src/{topic}/{name}/{module}"
+        for name in names
+        for module in (
+            "__init__.py",
+            "main.py",
+            "config.py",
+            "routers/items.py",
+            "routers/utils.py",
+            "empty_package/__init__.py",
+        )
+    }
+    kept.add(f"docs_src/{topic}/{names[1]}/variant_only.py")
+    unused = {
+        f"docs_src/{topic}/tutorial002/main.py",
+        f"docs_src/{topic}/tutorial002/__init__.py",
+        f"docs_src/{topic}/tutorial002/empty_package/__init__.py",
+        f"docs_src/{topic}/tutorial002/routers/items.py",
+    }
+    create_files(kept | unused)
+    include_file(f"docs_src/{topic}/{names[preferred]}/{included}")
+
+    remove_unused_docs_src()
+
+    assert {path.as_posix() for path in Path("docs_src").rglob("*.py")} == kept
+    assert not Path(f"docs_src/{topic}/tutorial002").exists()