From e88d99fccf0bf1e779484f929309fb101c0d4b17 Mon Sep 17 00:00:00 2001 From: James Mineau Date: Tue, 20 Oct 2026 23:51:45 -0600 Subject: [PATCH] feat(test): docstring examples run as tests pytest runs the `>>>` examples in the docstrings under src/ with --doctest-modules, next to the tests folder (testpaths, addopts in [tool.pytest]), so `just test`, `just cov` and CI check them. An example that needs data CI lacks is skipped with `# doctest: +SKIP`. `just test` (the template's own) checks that pytest collects the fixture's docstring example and the tests folder in each bake. AGENTS.md, CONTRIBUTING.md and the README say so. --- CONTRIBUTING.md | 3 ++- README.md | 2 +- justfile | 4 ++++ template/AGENTS.md.jinja | 3 +++ template/CONTRIBUTING.md.jinja | 2 +- template/pyproject.toml.jinja | 4 ++-- 6 files changed, 13 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7246316..1a7804b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,7 +28,8 @@ Before the checks, `just test` copies `fixtures/` into the baked project: `always/` (a function whose docstring draws a figure, an `.rst` page with IPython blocks) and, in the bake with `docs_notebooks`, `notebooks/` (a Markdown page with a code cell, a notebook). `build-docs` runs them, and `just test` checks that -their output reaches the HTML. +their output reaches the HTML. The function's `>>>` lines are also a doctest, and +`just test` checks that pytest collects it next to the tests folder. `just test` is what the Template workflow runs on every pull request. pre-commit needs git 2.31 or newer. diff --git a/README.md b/README.md index ee4a5c3..088389b 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ first time is by hand; `copier update` takes over from then on. | Versions | [setuptools-scm](https://setuptools-scm.readthedocs.io): the version is the git tag; `.devN` versions between releases | | Lint and format | [ruff](https://docs.astral.sh/ruff/) (`E F UP B SIM I D D213 NPY RUF100`; pydocstyle with the NumPy convention) | | Types | [pyrefly](https://pyrefly.org) | -| Tests | pytest 9 (strict, warnings are errors, `network`/`slow` markers), parallel with pytest-xdist, pytest-cov, Codecov | +| Tests | pytest 9 (strict, warnings are errors, `network`/`slow` markers), parallel with pytest-xdist, pytest-cov, Codecov; the `>>>` examples in docstrings run as tests (`--doctest-modules`) | | Docstrings | NumPy style, [docstr-coverage](https://github.com/HunterMcGushion/docstr_coverage) ≥ 95% | | Docs | Sphinx + PyData theme, pandas-style API pages (a page per class, with tables of its attributes and methods, and a page per member), copy buttons, examples that run when the docs build (matplotlib's `.. plot::` in docstrings, IPython's `.. ipython::` in `.rst` pages, and with `docs_notebooks`, notebooks and code cells in Markdown pages), live preview (`just docs-serve`), built with warnings as errors; versioned on GitHub Pages (`dev/`, one folder per release, `stable/`) with a version dropdown | | pre-commit | pre-commit-hooks, rST checks, validate-pyproject, [zizmor](https://docs.zizmor.sh); `uv lock`, ruff and pyrefly run from `.venv`, so `uv.lock` pins their versions | diff --git a/justfile b/justfile index f7691fc..29f4ceb 100644 --- a/justfile +++ b/justfile @@ -55,6 +55,10 @@ test: echo "::group::checks optional=$opt" uv run just quality-check + # pytest collects the docstring examples under src/ next to the tests folder. + collected="$(uv run pytest --collect-only -q)" + grep -qF 'src/testpackage/plotting.py::testpackage.plotting.square' <<<"$collected" + grep -qF 'tests/test_pkg.py::test_version' <<<"$collected" uv run just build-docs # The examples ran: each page's output and figures are in its HTML. sed 's/<[^>]*>//g' docs/_build/html/session.html | grep -qF 'square(5) = 25' diff --git a/template/AGENTS.md.jinja b/template/AGENTS.md.jinja index 7044554..e9d9afc 100644 --- a/template/AGENTS.md.jinja +++ b/template/AGENTS.md.jinja @@ -174,6 +174,9 @@ when one breaks. warning is ignored by message and module in `[tool.pytest]` `filterwarnings`. - `just test` runs in parallel (pytest-xdist); tests must not depend on order or shared state. Debug with a serial `uv run pytest tests/test_x.py -x`. +- Docstring examples (`>>>` lines under `src/`) run as tests too (`--doctest-modules`), + so an example that stops matching the code fails `just test`. One that needs data + CI lacks gets `# doctest: +SKIP` on its `>>>` line. - Tests must not depend on machine-specific paths or data. ## Commits, changelog, releases diff --git a/template/CONTRIBUTING.md.jinja b/template/CONTRIBUTING.md.jinja index 1d8e8aa..dea6665 100644 --- a/template/CONTRIBUTING.md.jinja +++ b/template/CONTRIBUTING.md.jinja @@ -30,7 +30,7 @@ shows them all). | Recipe | What it does | |---|---| | `just quality-check` | `lint`, `type-check`, `docstr` and `test`: what CI checks | -| `just test` | pytest in parallel, skipping tests marked `network` or `slow` (extra args go to pytest; plain `uv run pytest` runs serially, for debugging) | +| `just test` | pytest in parallel, skipping tests marked `network` or `slow`; it also runs the `>>>` examples in the docstrings under `src/` (extra args go to pytest; plain `uv run pytest` runs serially, for debugging) | | `just format` | fix lint and format the code with ruff | | `just build-docs` | build the HTML docs into `docs/_build/`, running every example; warnings are errors | | `just docs-serve` | preview the docs at , rebuilt on every save (on a remote machine, forward the port, e.g. VS Code's Ports panel) | diff --git a/template/pyproject.toml.jinja b/template/pyproject.toml.jinja index b8a2bfc..e7e05d9 100644 --- a/template/pyproject.toml.jinja +++ b/template/pyproject.toml.jinja @@ -77,9 +77,9 @@ python-version = "{{ python_min }}" # as CI checks, against the oldest supporte [tool.pytest] minversion = "9.0" -testpaths = ["tests"] +testpaths = ["tests", "src"] # src: the docstring examples (--doctest-modules below) strict = true # strict markers, config, xfail and parametrize ids -addopts = ["-ra"] +addopts = ["-ra", "--doctest-modules"] # run the `>>>` examples in docstrings as tests log_level = "INFO" # show captured logs for failing tests # Warnings are errors, so deprecations surface early. Ignore a specific # third-party warning with e.g. "ignore:some message:DeprecationWarning:pkg".