From 947e5a02c9f0deab6c269512d4f1ccd01aad2dd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Mon, 9 Nov 2026 16:01:12 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Preserve=20snippet=20variants=20?= =?UTF-8?q?when=20cleaning=20up=20docs=20sources?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 1 + scripts/docs.py | 119 +++++++++++------------------------ tests/test_docs_cleanup.py | 123 +++++++++++++++++++++++++++++++++++++ uv.lock | 2 + 4 files changed, 163 insertions(+), 82 deletions(-) create mode 100644 tests/test_docs_cleanup.py diff --git a/pyproject.toml b/pyproject.toml index 6e32774796..d05dd8c521 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -90,6 +90,7 @@ tests = [ "psycopg[binary] >=3.2.10", "pymysql[rsa] >=1.1.1", "pytest >=7.0.1", + "pyyaml >=5.3.1", "ruff >=0.16.0", "ty>=0.0.25", "typer >=0.26.1", diff --git a/scripts/docs.py b/scripts/docs.py index 07e2745f57..99b4a09068 100644 --- a/scripts/docs.py +++ b/scripts/docs.py @@ -271,99 +271,54 @@ def update_docs_includes_py39_to_py310() -> None: @app.command() def remove_unused_docs_src() -> None: """ - Delete .py files in docs_src that are not included in any .md file under docs/. + Delete unused Python snippets, keeping their variants and supporting package files. """ docs_src_path = Path("docs_src") - # Collect all docs .md content referencing docs_src - all_docs_content = "" - for md_file in docs_path.rglob("*.md"): - all_docs_content += md_file.read_text(encoding="utf-8") - # Build a set of directory-based package roots (e.g. docs_src/bigger_applications/app_py39) - # where at least one file is referenced in docs. All files in these directories - # should be kept since they may be internally imported by the referenced files. - used_package_dirs: set[Path] = set() - for py_file in docs_src_path.rglob("*.py"): - if py_file.name == "__init__.py": - continue - rel_path = str(py_file) - if rel_path in all_docs_content: - parts = py_file.relative_to(docs_src_path).parts - if len(parts) > 2 and not py_file.name.startswith("tutorial"): - # File is inside a package directory (e.g. - # docs_src/tutorial/fastapi/app_testing/tutorial001_py310/). - # Mark the immediate parent as a used package so sibling - # files (likely imported by the referenced file) are kept. - used_package_dirs.add(py_file.parent) + all_docs_content = "\n".join( + md_file.read_text(encoding="utf-8") for md_file in docs_path.rglob("*.md") + ) + variant_suffix = re.compile(r"(?:_non_an|_an)?(?:_py\d+)?$") + + def variant_key(path: Path) -> tuple[str, ...]: + return tuple(variant_suffix.sub("", part, count=1) for part in path.parts) + + def package_key(parts: tuple[str, ...]) -> tuple[str, ...] | None: + # Package examples use tutorial001 directories at different topic depths. + for index, part in enumerate(parts): + if re.fullmatch(r"tutorial\d+\w*", part): + return parts[: index + 1] + return None + + file_keys = { + py_file: variant_key(py_file.relative_to(docs_src_path).with_suffix("")) + for py_file in docs_src_path.rglob("*.py") + } + used_files = { + key + for py_file, key in file_keys.items() + if py_file.as_posix() in all_docs_content + } + # Keep entire referenced packages, including their variants and modules + # imported by the referenced file, without keeping unrelated tutorial packages. + used_package_dirs = { + package for key in used_files if (package := package_key(key[:-1])) is not None + } removed = 0 - for py_file in sorted(docs_src_path.rglob("*.py")): - if py_file.name == "__init__.py": + for py_file, key in sorted(file_keys.items()): + if py_file.name == "__init__.py" or key in used_files: continue - # Build the relative path as it appears in includes (e.g. docs_src/first_steps/tutorial001.py) - rel_path = str(py_file) - if rel_path in all_docs_content: - continue - # If this file is inside a directory-based package where any sibling is - # referenced, keep it (it's likely imported internally). - if py_file.parent in used_package_dirs: - continue - # Check if the _an counterpart (or non-_an counterpart) is referenced. - # If either variant is included, keep both. - # Handle both file-level _an (tutorial001_an.py) and directory-level _an - # (app_an/main.py) - counterpart_found = False - full_path_str = str(py_file) - if "_an" in py_file.stem: - # This is an _an file, check if the non-_an version is referenced - counterpart = full_path_str.replace( - f"/{py_file.stem}", f"/{py_file.stem.replace('_an', '', 1)}" - ) - if counterpart in all_docs_content: - counterpart_found = True - else: - # This is a non-_an file, check if there's an _an version referenced - # Insert _an before any version suffix or at the end of the stem - stem = py_file.stem - for suffix in ("_py39", "_py310"): - if suffix in stem: - an_stem = stem.replace(suffix, f"_an{suffix}", 1) - break - else: - an_stem = f"{stem}_an" - counterpart = full_path_str.replace(f"/{stem}.", f"/{an_stem}.") - if counterpart in all_docs_content: - counterpart_found = True - # Also check directory-level _an counterparts - if not counterpart_found: - parent_name = py_file.parent.name - if "_an" in parent_name: - counterpart_parent = parent_name.replace("_an", "", 1) - counterpart_dir = str(py_file).replace( - f"/{parent_name}/", f"/{counterpart_parent}/" - ) - if counterpart_dir in all_docs_content: - counterpart_found = True - else: - # Try inserting _an into parent directory name - for suffix in ("_py39", "_py310"): - if suffix in parent_name: - an_parent = parent_name.replace(suffix, f"_an{suffix}", 1) - break - else: - an_parent = f"{parent_name}_an" - counterpart_dir = str(py_file).replace( - f"/{parent_name}/", f"/{an_parent}/" - ) - if counterpart_dir in all_docs_content: - counterpart_found = True - if counterpart_found: + if package_key(key[:-1]) in used_package_dirs: continue logging.info(f"Removing unused file: {py_file}") py_file.unlink() removed += 1 - # Clean up directories that are empty or only contain __init__.py / __pycache__ + # Preserve init-only subpackages inside used packages, and clean up unused ones. for dir_path in sorted(docs_src_path.rglob("*"), reverse=True): if not dir_path.is_dir(): continue + key = variant_key(dir_path.relative_to(docs_src_path)) + if package_key(key) in used_package_dirs: + continue remaining = [ f for f in dir_path.iterdir() diff --git a/tests/test_docs_cleanup.py b/tests/test_docs_cleanup.py new file mode 100644 index 0000000000..28a115ce32 --- /dev/null +++ b/tests/test_docs_cleanup.py @@ -0,0 +1,123 @@ +from pathlib import Path + +import pytest + +from scripts.docs import remove_unused_docs_src + + +def create_files(paths: set[str]) -> None: + for path_str in paths: + path = Path(path_str) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("# Example code\n", encoding="utf-8") + + +def include_file(path: str) -> None: + doc = Path("docs/en/docs/tutorial.md") + doc.parent.mkdir(parents=True, exist_ok=True) + doc.write_text("{* " + path + " *}\n", encoding="utf-8") + + +@pytest.mark.parametrize( + "names", + [ + ("tutorial001", "tutorial001_non_an"), + ("tutorial001", "tutorial001_an"), + ("tutorial001_py310", "tutorial001_non_an_py310"), + ("tutorial001_py310", "tutorial001_an_py310"), + ( + "tutorial001_py310", + "tutorial001_an_py310", + "tutorial001_py314", + "tutorial001_an_py314", + ), + ( + "tutorial001_py311", + "tutorial001_non_an_py311", + "tutorial001_py314", + "tutorial001_non_an_py314", + ), + ("tutorial001_py311", "tutorial001_py314"), + ("tutorial_another", "tutorial_another_non_an"), + ], +) +@pytest.mark.parametrize("preferred", [0, 1]) +def test_keep_file_variants( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + names: tuple[str, ...], + preferred: int, +) -> None: + monkeypatch.chdir(tmp_path) + kept = {f"docs_src/tutorial/topic/{name}.py" for name in names} + unused = { + "docs_src/tutorial/topic/tutorial002.py", + "docs_src/tutorial/topic/tutorial001_extra.py", + } + create_files(kept | unused) + include_file(f"docs_src/tutorial/topic/{names[preferred]}.py") + + remove_unused_docs_src() + + assert {path.as_posix() for path in Path("docs_src").rglob("*.py")} == kept + + +@pytest.mark.parametrize( + "names", + [ + ("tutorial001", "tutorial001_non_an"), + ("tutorial001", "tutorial001_an"), + ( + "tutorial001_py310", + "tutorial001_an_py310", + "tutorial001_py314", + "tutorial001_an_py314", + ), + ( + "tutorial001_py311", + "tutorial001_non_an_py311", + "tutorial001_py314", + "tutorial001_non_an_py314", + ), + ], +) +@pytest.mark.parametrize("preferred", [0, 1]) +@pytest.mark.parametrize("included", ["main.py", "routers/items.py", "__init__.py"]) +@pytest.mark.parametrize( + "topic", ["tutorial/code_structure", "tutorial/fastapi/app_testing"] +) +def test_keep_entire_variant_packages( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + names: tuple[str, ...], + preferred: int, + included: str, + topic: str, +) -> None: + monkeypatch.chdir(tmp_path) + kept = { + f"docs_src/{topic}/{name}/{module}" + for name in names + for module in ( + "__init__.py", + "main.py", + "config.py", + "routers/items.py", + "routers/utils.py", + "empty_package/__init__.py", + ) + } + kept.add(f"docs_src/{topic}/{names[1]}/variant_only.py") + unused = { + f"docs_src/{topic}/tutorial002/main.py", + f"docs_src/{topic}/tutorial002/__init__.py", + f"docs_src/{topic}/tutorial002/empty_package/__init__.py", + f"docs_src/{topic}/tutorial002/routers/items.py", + } + create_files(kept | unused) + include_file(f"docs_src/{topic}/{names[preferred]}/{included}") + + remove_unused_docs_src() + + assert {path.as_posix() for path in Path("docs_src").rglob("*.py")} == kept + assert not Path(f"docs_src/{topic}/tutorial002").exists() diff --git a/uv.lock b/uv.lock index b7dc53e064..fee90bb7d6 100644 --- a/uv.lock +++ b/uv.lock @@ -2022,6 +2022,7 @@ tests = [ { name = "psycopg", extra = ["binary"] }, { name = "pymysql", extra = ["rsa"] }, { name = "pytest" }, + { name = "pyyaml" }, { name = "ruff" }, { name = "sqlalchemy", version = "2.0.55", source = { registry = "https://pypi.org/simple" }, extra = ["asyncio"], marker = "python_full_version < '3.11'" }, { name = "sqlalchemy", version = "2.1.6", source = { registry = "https://pypi.org/simple" }, extra = ["asyncio"], marker = "python_full_version >= '3.11'" }, @@ -2101,6 +2102,7 @@ tests = [ { name = "psycopg", extras = ["binary"], specifier = ">=3.2.10" }, { name = "pymysql", extras = ["rsa"], specifier = ">=1.1.1" }, { name = "pytest", specifier = ">=7.0.1" }, + { name = "pyyaml", specifier = ">=5.3.1" }, { name = "ruff", specifier = ">=0.16.0" }, { name = "sqlalchemy", extras = ["asyncio"] }, { name = "ty", specifier = ">=0.0.25" },