feat(generate_markdown): --optimize-only retro pass over committed PNGs

autohands/generate_markdown.py

@@ -29,6 +29,17 @@
   from script side-effects and will be reverted with them.
 - Regeneration is manual / at-release, only when a curated script changes —
   never per-commit.
+
+Extracted figures are optimized (256-colour quantize + optimized encode) as part
+of every render. That is forward-only — it never touches images committed by an
+earlier render — so pages built before the optimizer shipped keep their original
+PNGs. To bring those up to date, run the retro pass::
+
+    python ../PyAutoHands/autohands/generate_markdown.py autolens --optimize-only
+
+which walks ``markdown/**/<page>_files/`` through the same function, renders
+nothing, and reports the bytes reclaimed. It is idempotent — re-running it on an
+already-optimized workspace changes nothing.
 """
 
 import argparse
@@ -244,11 +255,15 @@ def optimize_pngs(files_dir: Path):
     corpus). Forward-only by construction: it touches the files of the render
     in progress, never previously committed images. Skips any image that does
     not get smaller.
+
+    Returns ``(bytes_before, bytes_after)`` over the PNGs it looked at, so
+    ``--optimize-only`` can report what a retro pass actually reclaimed.
     """
     if not files_dir.exists():
-        return
+        return 0, 0
     from PIL import Image
 
+    before = after = 0
     for png in sorted(files_dir.glob("*.png")):
         original_size = png.stat().st_size
         tmp = png.with_name(png.name + ".opt")
@@ -263,10 +278,57 @@ def optimize_pngs(files_dir: Path):
                 )
                 image = image.quantize(colors=256, method=method)
             image.save(tmp, format="PNG", optimize=True)
+        before += original_size
         if tmp.stat().st_size < original_size:
+            after += tmp.stat().st_size
             tmp.replace(png)
         else:
+            after += original_size
             tmp.unlink()
+    return before, after
+
+
+def optimize_existing(workspace_path: Path):
+    """
+    Retro-optimize the PNGs of pages rendered *before* ``optimize_pngs`` shipped.
+
+    ``optimize_pngs`` only ever sees the render in progress, so pages committed
+    earlier keep their unoptimized images forever. This walks every
+    ``markdown/**/<page>_files/`` directory in the workspace and puts them
+    through the same function, so a retro pass and a future re-render produce
+    identical bytes. Idempotent: already-optimized images do not shrink again
+    and are left alone.
+    """
+    markdown_path = workspace_path / MARKDOWN_DIR
+    if not markdown_path.is_dir():
+        print(f"No {MARKDOWN_DIR}/ directory in {workspace_path} — nothing to optimize.")
+        return 0, 0
+
+    total_before = total_after = 0
+    for files_dir in sorted(markdown_path.rglob("*_files")):
+        if not files_dir.is_dir():
+            continue
+        before, after = optimize_pngs(files_dir)
+        if not before:
+            continue
+        total_before += before
+        total_after += after
+        print(
+            f"  {files_dir.relative_to(workspace_path)}: "
+            f"{before / 1e6:.2f}MB -> {after / 1e6:.2f}MB "
+            f"({100 * after / before:.0f}%)"
+        )
+
+    if not total_before:
+        print(f"No PNGs found under {MARKDOWN_DIR}/.")
+    else:
+        print(
+            f"Optimized {MARKDOWN_DIR}/: {total_before / 1e6:.1f}MB -> "
+            f"{total_after / 1e6:.1f}MB "
+            f"({100 * total_after / total_before:.0f}%, "
+            f"{(total_before - total_after) / 1e6:.1f}MB reclaimed)"
+        )
+    return total_before, total_after
 
 
 def _markdown_header(script_rel: Path, md_dir: Path) -> str:
@@ -445,8 +507,22 @@ def main():
         default=None,
         help="Only render curated scripts whose path contains this substring",
     )
+    parser.add_argument(
+        "--optimize-only",
+        action="store_true",
+        help=(
+            "Skip rendering: re-encode the PNGs of pages already committed "
+            "under markdown/ through the same optimizer a fresh render uses"
+        ),
+    )
     args = parser.parse_args()
 
+    # Checked before the TEST_MODE guard on purpose: --optimize-only executes no
+    # script and runs no search, so a truncated-search build cannot corrupt it.
+    if args.optimize_only:
+        optimize_existing(Path.cwd())
+        return
+
     if os.environ.get("PYAUTO_TEST_MODE"):
         sys.exit(
             "generate_markdown.py refuses to run with PYAUTO_TEST_MODE set: a "

docs/internals.md

@@ -117,7 +117,7 @@ All scripts in `autohands/` are run from within a checked-out workspace director
 - **`run_python.py <project> <directory>`** — Executes Python scripts in a workspace folder, skipping files listed in `config/no_run.yaml`
 - **`run.py <project> <directory> [--visualise]`** — Executes Jupyter notebooks in a workspace folder, skipping files in `config/no_run.yaml`
 - **`generate.py <project>`** — Converts Python scripts in `scripts/` to `.ipynb` notebooks in `notebooks/`, run from within the workspace root
-- **`generate_markdown.py <project> [--only <substring>]`** — Renders the curated scripts listed in the workspace's `config/build/markdown_examples.yaml` to **executed** markdown pages with output images under `markdown/`, plus an index, committed so examples are readable on GitHub. Manual / at-release only, never per-commit; refuses `PYAUTO_TEST_MODE` (truncated searches make wrong images — model-fit reruns instead resume from the completed `output/` cache); never renders `features/` scripts; restores tracked files a script modifies (e.g. simulators rewriting `dataset/`). Rules and rationale in the module docstring.
+- **`generate_markdown.py <project> [--only <substring>] [--optimize-only]`** — Renders the curated scripts listed in the workspace's `config/build/markdown_examples.yaml` to **executed** markdown pages with output images under `markdown/`, plus an index, committed so examples are readable on GitHub. Manual / at-release only, never per-commit; refuses `PYAUTO_TEST_MODE` (truncated searches make wrong images — model-fit reruns instead resume from the completed `output/` cache); never renders `features/` scripts; restores tracked files a script modifies (e.g. simulators rewriting `dataset/`). Extracted figures are optimized on the way out, but only for the render in progress — `--optimize-only` renders nothing and puts already-committed `markdown/**/<page>_files/` PNGs through the same optimizer, for pages built before it shipped. Rules and rationale in the module docstring.
 - **`script_matrix.py <project1> [project2 ...]`** — Outputs a JSON matrix of `{name, directory}` pairs for GitHub Actions matrix strategy
 - **`tag_and_merge.sh --version <version>`** — Commits pending changes and tags library repos (PyAutoNerves, PyAutoFit, PyAutoArray, PyAutoGalaxy, PyAutoLens) for release
 - **`url_check`** — URL hygiene moved to PyAutoHeart (Heart owns all health checking). `autohands url_check` is now a thin shim to `pyauto-heart url_check`; the ecosystem-wide sweep runs from PyAutoHeart's central `url-check.yml` workflow (replacing the old per-repo `url_check.yml` workflows). The runnable scripts live at `PyAutoHeart/heart/checks/url_check*.{sh,py}`.

tests/test_generate_markdown.py

@@ -238,7 +238,80 @@ def test_quantizes_in_place_and_shrinks(self, tmp_path):
         assert not list(files_dir.glob("*.opt"))
 
     def test_missing_dir_is_noop(self, tmp_path):
-        generate_markdown.optimize_pngs(tmp_path / "absent")
+        assert generate_markdown.optimize_pngs(tmp_path / "absent") == (0, 0)
+
+
+def _noisy_png(path: Path, seed: int = 0) -> None:
+    """A 128x128 image that quantizes well — the shape matplotlib figures have."""
+    from PIL import Image
+    import random
+
+    random.seed(seed)
+    image = Image.new("RGB", (128, 128))
+    image.putdata(
+        [
+            (random.randrange(50, 200), random.randrange(50, 200), 30)
+            for _ in range(128 * 128)
+        ]
+    )
+    image.save(path)
+
+
+class TestOptimizeExisting:
+    def test_walks_every_files_dir_and_shrinks(self, tmp_path):
+        pages = [
+            tmp_path / "markdown" / "start_here_files",
+            tmp_path / "markdown" / "imaging" / "modeling_files",
+        ]
+        pngs = []
+        for i, files_dir in enumerate(pages):
+            files_dir.mkdir(parents=True)
+            png = files_dir / "fig_0.png"
+            _noisy_png(png, seed=i)
+            pngs.append(png)
+        before_sizes = [png.stat().st_size for png in pngs]
+
+        total_before, total_after = generate_markdown.optimize_existing(tmp_path)
+
+        assert total_after < total_before
+        assert total_before == sum(before_sizes)
+        for png in pngs:
+            assert png.stat().st_size < before_sizes[pngs.index(png)]
+        assert not list(tmp_path.rglob("*.opt"))
+
+    def test_leaves_everything_outside_markdown_alone(self, tmp_path):
+        outside = tmp_path / "dataset" / "imaging_files"
+        outside.mkdir(parents=True)
+        untouched = outside / "fig_0.png"
+        _noisy_png(untouched)
+        before = untouched.stat().st_size
+
+        files_dir = tmp_path / "markdown" / "page_files"
+        files_dir.mkdir(parents=True)
+        _noisy_png(files_dir / "fig_0.png")
+
+        generate_markdown.optimize_existing(tmp_path)
+
+        assert untouched.stat().st_size == before
+
+    def test_is_idempotent(self, tmp_path):
+        files_dir = tmp_path / "markdown" / "page_files"
+        files_dir.mkdir(parents=True)
+        _noisy_png(files_dir / "fig_0.png")
+
+        generate_markdown.optimize_existing(tmp_path)
+        settled = (files_dir / "fig_0.png").read_bytes()
+        second_before, second_after = generate_markdown.optimize_existing(tmp_path)
+
+        assert (files_dir / "fig_0.png").read_bytes() == settled
+        assert second_after == second_before
+
+    def test_no_markdown_dir_is_noop(self, tmp_path):
+        assert generate_markdown.optimize_existing(tmp_path) == (0, 0)
+
+    def test_empty_markdown_dir_is_noop(self, tmp_path):
+        (tmp_path / "markdown").mkdir()
+        assert generate_markdown.optimize_existing(tmp_path) == (0, 0)
 
 
 class TestMarkdownHeader: