diff --git a/autohands/generate_markdown.py b/autohands/generate_markdown.py index e538051..39969f8 100644 --- a/autohands/generate_markdown.py +++ b/autohands/generate_markdown.py @@ -29,6 +29,17 @@ from script side-effects and will be reverted with them. - Regeneration is manual / at-release, only when a curated script changes — never per-commit. + +Extracted figures are optimized (256-colour quantize + optimized encode) as part +of every render. That is forward-only — it never touches images committed by an +earlier render — so pages built before the optimizer shipped keep their original +PNGs. To bring those up to date, run the retro pass:: + + python ../PyAutoHands/autohands/generate_markdown.py autolens --optimize-only + +which walks ``markdown/**/_files/`` through the same function, renders +nothing, and reports the bytes reclaimed. It is idempotent — re-running it on an +already-optimized workspace changes nothing. """ import argparse @@ -244,11 +255,15 @@ def optimize_pngs(files_dir: Path): corpus). Forward-only by construction: it touches the files of the render in progress, never previously committed images. Skips any image that does not get smaller. + + Returns ``(bytes_before, bytes_after)`` over the PNGs it looked at, so + ``--optimize-only`` can report what a retro pass actually reclaimed. """ if not files_dir.exists(): - return + return 0, 0 from PIL import Image + before = after = 0 for png in sorted(files_dir.glob("*.png")): original_size = png.stat().st_size tmp = png.with_name(png.name + ".opt") @@ -263,10 +278,57 @@ def optimize_pngs(files_dir: Path): ) image = image.quantize(colors=256, method=method) image.save(tmp, format="PNG", optimize=True) + before += original_size if tmp.stat().st_size < original_size: + after += tmp.stat().st_size tmp.replace(png) else: + after += original_size tmp.unlink() + return before, after + + +def optimize_existing(workspace_path: Path): + """ + Retro-optimize the PNGs of pages rendered *before* ``optimize_pngs`` shipped. + + ``optimize_pngs`` only ever sees the render in progress, so pages committed + earlier keep their unoptimized images forever. This walks every + ``markdown/**/_files/`` directory in the workspace and puts them + through the same function, so a retro pass and a future re-render produce + identical bytes. Idempotent: already-optimized images do not shrink again + and are left alone. + """ + markdown_path = workspace_path / MARKDOWN_DIR + if not markdown_path.is_dir(): + print(f"No {MARKDOWN_DIR}/ directory in {workspace_path} — nothing to optimize.") + return 0, 0 + + total_before = total_after = 0 + for files_dir in sorted(markdown_path.rglob("*_files")): + if not files_dir.is_dir(): + continue + before, after = optimize_pngs(files_dir) + if not before: + continue + total_before += before + total_after += after + print( + f" {files_dir.relative_to(workspace_path)}: " + f"{before / 1e6:.2f}MB -> {after / 1e6:.2f}MB " + f"({100 * after / before:.0f}%)" + ) + + if not total_before: + print(f"No PNGs found under {MARKDOWN_DIR}/.") + else: + print( + f"Optimized {MARKDOWN_DIR}/: {total_before / 1e6:.1f}MB -> " + f"{total_after / 1e6:.1f}MB " + f"({100 * total_after / total_before:.0f}%, " + f"{(total_before - total_after) / 1e6:.1f}MB reclaimed)" + ) + return total_before, total_after def _markdown_header(script_rel: Path, md_dir: Path) -> str: @@ -445,8 +507,22 @@ def main(): default=None, help="Only render curated scripts whose path contains this substring", ) + parser.add_argument( + "--optimize-only", + action="store_true", + help=( + "Skip rendering: re-encode the PNGs of pages already committed " + "under markdown/ through the same optimizer a fresh render uses" + ), + ) args = parser.parse_args() + # Checked before the TEST_MODE guard on purpose: --optimize-only executes no + # script and runs no search, so a truncated-search build cannot corrupt it. + if args.optimize_only: + optimize_existing(Path.cwd()) + return + if os.environ.get("PYAUTO_TEST_MODE"): sys.exit( "generate_markdown.py refuses to run with PYAUTO_TEST_MODE set: a " diff --git a/docs/internals.md b/docs/internals.md index 62100d2..6261bc6 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -117,7 +117,7 @@ All scripts in `autohands/` are run from within a checked-out workspace director - **`run_python.py `** — Executes Python scripts in a workspace folder, skipping files listed in `config/no_run.yaml` - **`run.py [--visualise]`** — Executes Jupyter notebooks in a workspace folder, skipping files in `config/no_run.yaml` - **`generate.py `** — Converts Python scripts in `scripts/` to `.ipynb` notebooks in `notebooks/`, run from within the workspace root -- **`generate_markdown.py [--only ]`** — Renders the curated scripts listed in the workspace's `config/build/markdown_examples.yaml` to **executed** markdown pages with output images under `markdown/`, plus an index, committed so examples are readable on GitHub. Manual / at-release only, never per-commit; refuses `PYAUTO_TEST_MODE` (truncated searches make wrong images — model-fit reruns instead resume from the completed `output/` cache); never renders `features/` scripts; restores tracked files a script modifies (e.g. simulators rewriting `dataset/`). Rules and rationale in the module docstring. +- **`generate_markdown.py [--only ] [--optimize-only]`** — Renders the curated scripts listed in the workspace's `config/build/markdown_examples.yaml` to **executed** markdown pages with output images under `markdown/`, plus an index, committed so examples are readable on GitHub. Manual / at-release only, never per-commit; refuses `PYAUTO_TEST_MODE` (truncated searches make wrong images — model-fit reruns instead resume from the completed `output/` cache); never renders `features/` scripts; restores tracked files a script modifies (e.g. simulators rewriting `dataset/`). Extracted figures are optimized on the way out, but only for the render in progress — `--optimize-only` renders nothing and puts already-committed `markdown/**/_files/` PNGs through the same optimizer, for pages built before it shipped. Rules and rationale in the module docstring. - **`script_matrix.py [project2 ...]`** — Outputs a JSON matrix of `{name, directory}` pairs for GitHub Actions matrix strategy - **`tag_and_merge.sh --version `** — Commits pending changes and tags library repos (PyAutoNerves, PyAutoFit, PyAutoArray, PyAutoGalaxy, PyAutoLens) for release - **`url_check`** — URL hygiene moved to PyAutoHeart (Heart owns all health checking). `autohands url_check` is now a thin shim to `pyauto-heart url_check`; the ecosystem-wide sweep runs from PyAutoHeart's central `url-check.yml` workflow (replacing the old per-repo `url_check.yml` workflows). The runnable scripts live at `PyAutoHeart/heart/checks/url_check*.{sh,py}`. diff --git a/tests/test_generate_markdown.py b/tests/test_generate_markdown.py index 3ae6bd1..38d5f52 100644 --- a/tests/test_generate_markdown.py +++ b/tests/test_generate_markdown.py @@ -238,7 +238,80 @@ def test_quantizes_in_place_and_shrinks(self, tmp_path): assert not list(files_dir.glob("*.opt")) def test_missing_dir_is_noop(self, tmp_path): - generate_markdown.optimize_pngs(tmp_path / "absent") + assert generate_markdown.optimize_pngs(tmp_path / "absent") == (0, 0) + + +def _noisy_png(path: Path, seed: int = 0) -> None: + """A 128x128 image that quantizes well — the shape matplotlib figures have.""" + from PIL import Image + import random + + random.seed(seed) + image = Image.new("RGB", (128, 128)) + image.putdata( + [ + (random.randrange(50, 200), random.randrange(50, 200), 30) + for _ in range(128 * 128) + ] + ) + image.save(path) + + +class TestOptimizeExisting: + def test_walks_every_files_dir_and_shrinks(self, tmp_path): + pages = [ + tmp_path / "markdown" / "start_here_files", + tmp_path / "markdown" / "imaging" / "modeling_files", + ] + pngs = [] + for i, files_dir in enumerate(pages): + files_dir.mkdir(parents=True) + png = files_dir / "fig_0.png" + _noisy_png(png, seed=i) + pngs.append(png) + before_sizes = [png.stat().st_size for png in pngs] + + total_before, total_after = generate_markdown.optimize_existing(tmp_path) + + assert total_after < total_before + assert total_before == sum(before_sizes) + for png in pngs: + assert png.stat().st_size < before_sizes[pngs.index(png)] + assert not list(tmp_path.rglob("*.opt")) + + def test_leaves_everything_outside_markdown_alone(self, tmp_path): + outside = tmp_path / "dataset" / "imaging_files" + outside.mkdir(parents=True) + untouched = outside / "fig_0.png" + _noisy_png(untouched) + before = untouched.stat().st_size + + files_dir = tmp_path / "markdown" / "page_files" + files_dir.mkdir(parents=True) + _noisy_png(files_dir / "fig_0.png") + + generate_markdown.optimize_existing(tmp_path) + + assert untouched.stat().st_size == before + + def test_is_idempotent(self, tmp_path): + files_dir = tmp_path / "markdown" / "page_files" + files_dir.mkdir(parents=True) + _noisy_png(files_dir / "fig_0.png") + + generate_markdown.optimize_existing(tmp_path) + settled = (files_dir / "fig_0.png").read_bytes() + second_before, second_after = generate_markdown.optimize_existing(tmp_path) + + assert (files_dir / "fig_0.png").read_bytes() == settled + assert second_after == second_before + + def test_no_markdown_dir_is_noop(self, tmp_path): + assert generate_markdown.optimize_existing(tmp_path) == (0, 0) + + def test_empty_markdown_dir_is_noop(self, tmp_path): + (tmp_path / "markdown").mkdir() + assert generate_markdown.optimize_existing(tmp_path) == (0, 0) class TestMarkdownHeader: