From 1e32e2911d731bf895054b15de072e12139d71d3 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Thu, 20 Aug 2026 13:31:50 -0400 Subject: [PATCH] feat(generate_markdown): --optimize-only retro pass over committed PNGs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit optimize_pngs() is forward-only by construction — it sees only the render in progress, so pages built before it shipped keep their unoptimized images forever. 405 of the 429 PNGs committed across the six markdown workspaces are still RGBA; a measured sample puts them at 88.4MB -> ~23MB through the existing algorithm. Add --optimize-only, which renders nothing and walks markdown/**/_files/ through the same optimize_pngs(), so a retro pass and a future re-render produce identical bytes (rather than a one-off pngquant sweep that would diverge from what the tool emits). optimize_pngs() now returns (before, after) byte totals so the walk can report what it reclaimed. The flag is checked before the PYAUTO_TEST_MODE guard on purpose: it executes no script and runs no search, so a truncated-search build cannot corrupt it. No dpi knob — quantize already delivers ~74% and a dpi drop would degrade every future render's readability for a smaller marginal gain. Verified end-to-end on a scratch copy of real autolens pages: 3.1MB -> 0.9MB (28%), dimensions preserved, already-optimized P-mode images left alone, and a second pass byte-identical (idempotent). PyAutoHands#247 Co-Authored-By: Claude Opus 5 --- autohands/generate_markdown.py | 78 ++++++++++++++++++++++++++++++++- docs/internals.md | 2 +- tests/test_generate_markdown.py | 75 ++++++++++++++++++++++++++++++- 3 files changed, 152 insertions(+), 3 deletions(-) diff --git a/autohands/generate_markdown.py b/autohands/generate_markdown.py index e5380513..39969f89 100644 --- a/autohands/generate_markdown.py +++ b/autohands/generate_markdown.py @@ -29,6 +29,17 @@ from script side-effects and will be reverted with them. - Regeneration is manual / at-release, only when a curated script changes — never per-commit. + +Extracted figures are optimized (256-colour quantize + optimized encode) as part +of every render. That is forward-only — it never touches images committed by an +earlier render — so pages built before the optimizer shipped keep their original +PNGs. To bring those up to date, run the retro pass:: + + python ../PyAutoHands/autohands/generate_markdown.py autolens --optimize-only + +which walks ``markdown/**/_files/`` through the same function, renders +nothing, and reports the bytes reclaimed. It is idempotent — re-running it on an +already-optimized workspace changes nothing. """ import argparse @@ -244,11 +255,15 @@ def optimize_pngs(files_dir: Path): corpus). Forward-only by construction: it touches the files of the render in progress, never previously committed images. Skips any image that does not get smaller. + + Returns ``(bytes_before, bytes_after)`` over the PNGs it looked at, so + ``--optimize-only`` can report what a retro pass actually reclaimed. """ if not files_dir.exists(): - return + return 0, 0 from PIL import Image + before = after = 0 for png in sorted(files_dir.glob("*.png")): original_size = png.stat().st_size tmp = png.with_name(png.name + ".opt") @@ -263,10 +278,57 @@ def optimize_pngs(files_dir: Path): ) image = image.quantize(colors=256, method=method) image.save(tmp, format="PNG", optimize=True) + before += original_size if tmp.stat().st_size < original_size: + after += tmp.stat().st_size tmp.replace(png) else: + after += original_size tmp.unlink() + return before, after + + +def optimize_existing(workspace_path: Path): + """ + Retro-optimize the PNGs of pages rendered *before* ``optimize_pngs`` shipped. + + ``optimize_pngs`` only ever sees the render in progress, so pages committed + earlier keep their unoptimized images forever. This walks every + ``markdown/**/_files/`` directory in the workspace and puts them + through the same function, so a retro pass and a future re-render produce + identical bytes. Idempotent: already-optimized images do not shrink again + and are left alone. + """ + markdown_path = workspace_path / MARKDOWN_DIR + if not markdown_path.is_dir(): + print(f"No {MARKDOWN_DIR}/ directory in {workspace_path} — nothing to optimize.") + return 0, 0 + + total_before = total_after = 0 + for files_dir in sorted(markdown_path.rglob("*_files")): + if not files_dir.is_dir(): + continue + before, after = optimize_pngs(files_dir) + if not before: + continue + total_before += before + total_after += after + print( + f" {files_dir.relative_to(workspace_path)}: " + f"{before / 1e6:.2f}MB -> {after / 1e6:.2f}MB " + f"({100 * after / before:.0f}%)" + ) + + if not total_before: + print(f"No PNGs found under {MARKDOWN_DIR}/.") + else: + print( + f"Optimized {MARKDOWN_DIR}/: {total_before / 1e6:.1f}MB -> " + f"{total_after / 1e6:.1f}MB " + f"({100 * total_after / total_before:.0f}%, " + f"{(total_before - total_after) / 1e6:.1f}MB reclaimed)" + ) + return total_before, total_after def _markdown_header(script_rel: Path, md_dir: Path) -> str: @@ -445,8 +507,22 @@ def main(): default=None, help="Only render curated scripts whose path contains this substring", ) + parser.add_argument( + "--optimize-only", + action="store_true", + help=( + "Skip rendering: re-encode the PNGs of pages already committed " + "under markdown/ through the same optimizer a fresh render uses" + ), + ) args = parser.parse_args() + # Checked before the TEST_MODE guard on purpose: --optimize-only executes no + # script and runs no search, so a truncated-search build cannot corrupt it. + if args.optimize_only: + optimize_existing(Path.cwd()) + return + if os.environ.get("PYAUTO_TEST_MODE"): sys.exit( "generate_markdown.py refuses to run with PYAUTO_TEST_MODE set: a " diff --git a/docs/internals.md b/docs/internals.md index 62100d2f..6261bc6a 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -117,7 +117,7 @@ All scripts in `autohands/` are run from within a checked-out workspace director - **`run_python.py `** — Executes Python scripts in a workspace folder, skipping files listed in `config/no_run.yaml` - **`run.py [--visualise]`** — Executes Jupyter notebooks in a workspace folder, skipping files in `config/no_run.yaml` - **`generate.py `** — Converts Python scripts in `scripts/` to `.ipynb` notebooks in `notebooks/`, run from within the workspace root -- **`generate_markdown.py [--only ]`** — Renders the curated scripts listed in the workspace's `config/build/markdown_examples.yaml` to **executed** markdown pages with output images under `markdown/`, plus an index, committed so examples are readable on GitHub. Manual / at-release only, never per-commit; refuses `PYAUTO_TEST_MODE` (truncated searches make wrong images — model-fit reruns instead resume from the completed `output/` cache); never renders `features/` scripts; restores tracked files a script modifies (e.g. simulators rewriting `dataset/`). Rules and rationale in the module docstring. +- **`generate_markdown.py [--only ] [--optimize-only]`** — Renders the curated scripts listed in the workspace's `config/build/markdown_examples.yaml` to **executed** markdown pages with output images under `markdown/`, plus an index, committed so examples are readable on GitHub. Manual / at-release only, never per-commit; refuses `PYAUTO_TEST_MODE` (truncated searches make wrong images — model-fit reruns instead resume from the completed `output/` cache); never renders `features/` scripts; restores tracked files a script modifies (e.g. simulators rewriting `dataset/`). Extracted figures are optimized on the way out, but only for the render in progress — `--optimize-only` renders nothing and puts already-committed `markdown/**/_files/` PNGs through the same optimizer, for pages built before it shipped. Rules and rationale in the module docstring. - **`script_matrix.py [project2 ...]`** — Outputs a JSON matrix of `{name, directory}` pairs for GitHub Actions matrix strategy - **`tag_and_merge.sh --version `** — Commits pending changes and tags library repos (PyAutoNerves, PyAutoFit, PyAutoArray, PyAutoGalaxy, PyAutoLens) for release - **`url_check`** — URL hygiene moved to PyAutoHeart (Heart owns all health checking). `autohands url_check` is now a thin shim to `pyauto-heart url_check`; the ecosystem-wide sweep runs from PyAutoHeart's central `url-check.yml` workflow (replacing the old per-repo `url_check.yml` workflows). The runnable scripts live at `PyAutoHeart/heart/checks/url_check*.{sh,py}`. diff --git a/tests/test_generate_markdown.py b/tests/test_generate_markdown.py index 3ae6bd14..38d5f521 100644 --- a/tests/test_generate_markdown.py +++ b/tests/test_generate_markdown.py @@ -238,7 +238,80 @@ def test_quantizes_in_place_and_shrinks(self, tmp_path): assert not list(files_dir.glob("*.opt")) def test_missing_dir_is_noop(self, tmp_path): - generate_markdown.optimize_pngs(tmp_path / "absent") + assert generate_markdown.optimize_pngs(tmp_path / "absent") == (0, 0) + + +def _noisy_png(path: Path, seed: int = 0) -> None: + """A 128x128 image that quantizes well — the shape matplotlib figures have.""" + from PIL import Image + import random + + random.seed(seed) + image = Image.new("RGB", (128, 128)) + image.putdata( + [ + (random.randrange(50, 200), random.randrange(50, 200), 30) + for _ in range(128 * 128) + ] + ) + image.save(path) + + +class TestOptimizeExisting: + def test_walks_every_files_dir_and_shrinks(self, tmp_path): + pages = [ + tmp_path / "markdown" / "start_here_files", + tmp_path / "markdown" / "imaging" / "modeling_files", + ] + pngs = [] + for i, files_dir in enumerate(pages): + files_dir.mkdir(parents=True) + png = files_dir / "fig_0.png" + _noisy_png(png, seed=i) + pngs.append(png) + before_sizes = [png.stat().st_size for png in pngs] + + total_before, total_after = generate_markdown.optimize_existing(tmp_path) + + assert total_after < total_before + assert total_before == sum(before_sizes) + for png in pngs: + assert png.stat().st_size < before_sizes[pngs.index(png)] + assert not list(tmp_path.rglob("*.opt")) + + def test_leaves_everything_outside_markdown_alone(self, tmp_path): + outside = tmp_path / "dataset" / "imaging_files" + outside.mkdir(parents=True) + untouched = outside / "fig_0.png" + _noisy_png(untouched) + before = untouched.stat().st_size + + files_dir = tmp_path / "markdown" / "page_files" + files_dir.mkdir(parents=True) + _noisy_png(files_dir / "fig_0.png") + + generate_markdown.optimize_existing(tmp_path) + + assert untouched.stat().st_size == before + + def test_is_idempotent(self, tmp_path): + files_dir = tmp_path / "markdown" / "page_files" + files_dir.mkdir(parents=True) + _noisy_png(files_dir / "fig_0.png") + + generate_markdown.optimize_existing(tmp_path) + settled = (files_dir / "fig_0.png").read_bytes() + second_before, second_after = generate_markdown.optimize_existing(tmp_path) + + assert (files_dir / "fig_0.png").read_bytes() == settled + assert second_after == second_before + + def test_no_markdown_dir_is_noop(self, tmp_path): + assert generate_markdown.optimize_existing(tmp_path) == (0, 0) + + def test_empty_markdown_dir_is_noop(self, tmp_path): + (tmp_path / "markdown").mkdir() + assert generate_markdown.optimize_existing(tmp_path) == (0, 0) class TestMarkdownHeader: