diff --git a/.github/workflows/autofix.yaml b/.github/workflows/autofix.yaml new file mode 100644 index 0000000..aed4e6c --- /dev/null +++ b/.github/workflows/autofix.yaml @@ -0,0 +1,29 @@ +name: autofix.ci + +on: + pull_request: + +permissions: + contents: read + +jobs: + autofix: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: j178/prek-action@7b101f8338a08d2bd6fdb3625e4e7692428763a3 # v3.1.1 + with: + install-only: true + + - name: Run prek + id: prek + continue-on-error: true + run: prek run --all-files + + - name: Verify fixes + if: steps.prek.outcome == 'failure' + run: prek run --all-files + + - name: Commit fixes + if: always() && !cancelled() + uses: autofix-ci/action@c5b2d67aa2274e7b5a18224e8171550871fc7e4a # v1.3.4 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index 2ec8662..0000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,134 +0,0 @@ -# .pre-commit-config.yaml -# -# Setup: -# pip install pre-commit # or: uv tool install pre-commit -# pre-commit install # installs the git hook -# pre-commit install --hook-type commit-msg # only if you enable commitizen below -# pre-commit run --all-files # run once across the whole repo -# -# Maintenance: -# pre-commit autoupdate # bump all `rev:` pins to latest -# -# Docs: https://pre-commit.com | Hook index: https://pre-commit.com/hooks.html - -minimum_pre_commit_version: "3.5.0" - -default_language_version: - python: python3.11 - -# Keep commits fast: only run on the commit stage by default. -default_stages: [pre-commit] - -# Don't run hooks on generated / vendored / data paths. -exclude: | - (?x)^( - \.git/.*| - \.venv/.*| - build/.*| - dist/.*| - docs/_build/.*| - .*\.egg-info/.*| - data/.*| - datasets/.*| - .*\.ipynb_checkpoints/.* - )$ - -# Configure pre-commit.ci to autofix pre-commit errors in PRs -ci: - autofix_commit_msg: | - [pre-commit.ci] auto fixes from pre-commit.com hooks - - for more information, see https://pre-commit.ci - autofix_prs: true - autoupdate_branch: '' - autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' - autoupdate_schedule: weekly - skip: [] - submodules: false - -repos: - # --------------------------------------------------------------------------- - # Core hygiene: cheap, fast, catch-everything checks. - # --------------------------------------------------------------------------- - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v6.0.0 - hooks: - - id: trailing-whitespace - args: [--markdown-linebreak-ext=md] - - id: end-of-file-fixer - - id: mixed-line-ending - args: [--fix=lf] - - id: check-yaml - - id: check-toml - - id: check-json - - id: check-merge-conflict - - id: check-case-conflict - - id: check-symlinks - - id: debug-statements # blocks stray breakpoint()/pdb - - id: detect-private-key # blocks committed private keys - - id: check-added-large-files - args: [--maxkb=1024] - - # --------------------------------------------------------------------------- - # Ruff: linter + formatter (replaces flake8, isort, black, pyupgrade, etc.). - # Lint (with --fix) MUST run before the formatter. - # --------------------------------------------------------------------------- - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.17.5 - hooks: - - id: ruff-check - args: [--fix] - types_or: [python, pyi, jupyter] - - id: ruff-format - types_or: [python, pyi, jupyter] - - # --------------------------------------------------------------------------- - # Static type checking (run on the package source only, to stay fast). - # --------------------------------------------------------------------------- - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v2.4.1 - hooks: - - id: mypy - files: ^src/ - args: [--ignore-missing-imports, --install-types, --non-interactive] - additional_dependencies: [typer>=0.26.8] - - # --------------------------------------------------------------------------- - # Spelling: catches typos in code, docstrings, and docs. - # --------------------------------------------------------------------------- - - repo: https://github.com/codespell-project/codespell - rev: v2.4.3 - hooks: - - id: codespell - additional_dependencies: [tomli] - # Configure ignores in pyproject.toml under [tool.codespell]. - - # --------------------------------------------------------------------------- - # Notebooks: strip outputs before commit (clean diffs, no data leakage). - # Essential for ML repos that carry .ipynb files. - # --------------------------------------------------------------------------- - - repo: https://github.com/kynan/nbstripout - rev: 0.9.1 - hooks: - - id: nbstripout - - # --------------------------------------------------------------------------- - # Secret scanning (baseline-driven). Generate the baseline once with: - # detect-secrets scan > .secrets.baseline - # --------------------------------------------------------------------------- - - repo: https://github.com/Yelp/detect-secrets - rev: v1.5.0 - hooks: - - id: detect-secrets - args: [--baseline, .secrets.baseline] - - - # --------------------------------------------------------------------------- - # uv lock file update to make sure it is properly synced even if the - # dependencies have been updated. - # --------------------------------------------------------------------------- - - repo: https://github.com/astral-sh/uv-pre-commit - # uv version. - rev: 0.13.8 - hooks: - - id: uv-lock diff --git a/docs/roadmap.md b/docs/roadmap.md index b6bf184..0a7753a 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -47,6 +47,7 @@ A first-class command-line experience ready for daily use. - [ ] `--fail-on-drift` exit code for CI pipelines - [ ] `--json` flag on all commands - [ ] Shell completion +- [x] Replace `mypy` by `ty` ## v0.4 - Format and Integration Expansion diff --git a/prek.toml b/prek.toml new file mode 100644 index 0000000..dea9090 --- /dev/null +++ b/prek.toml @@ -0,0 +1,125 @@ +#:schema https://www.schemastore.org/prek.json +# prek.toml +# +# Setup: +# uv tool install prek # or: pip install prek +# prek install # installs the git hook +# prek run --all-files # run once across the whole repo +# +# Maintenance: +# prek auto-update # bump all `rev` pins to latest +# prek util list-builtins # hooks prek implements natively in Rust +# +# Docs: https://prek.j178.dev + +minimum_prek_version = "0.5.11" + +default_language_version = { python = "python3.11" } + +# Keep commits fast: only run on the commit stage by default. +default_stages = ["pre-commit"] + +# Don't run hooks on generated / vendored / data paths. +exclude = ''' +(?x)^( + \.git/.*| + \.venv/.*| + build/.*| + dist/.*| + docs/_build/.*| + .*\.egg-info/.*| + data/.*| + datasets/.*| + .*\.ipynb_checkpoints/.* +)$ +''' + +# ----------------------------------------------------------------------------- +# Core hygiene: cheap, fast, catch-everything checks. +# prek's automatic fast path runs its built-in Rust implementation for every +# hook here except `debug-statements`, which falls back to the pinned Python +# hook. See https://prek.j178.dev/built-in-hooks/#use-the-automatic-fast-path +# ----------------------------------------------------------------------------- +[[repos]] +repo = "https://github.com/pre-commit/pre-commit-hooks" +rev = "v6.0.0" +hooks = [ + { id = "trailing-whitespace", args = ["--markdown-linebreak-ext=md"] }, + { id = "end-of-file-fixer" }, + { id = "mixed-line-ending", args = ["--fix=lf"] }, + { id = "check-yaml" }, + { id = "check-toml" }, + { id = "check-json" }, + { id = "check-merge-conflict" }, + { id = "check-case-conflict" }, + { id = "check-symlinks" }, + { id = "debug-statements" }, # blocks stray breakpoint()/pdb (no builtin) + { id = "detect-private-key" }, # blocks committed private keys + { id = "check-added-large-files", args = ["--maxkb=1024"] }, +] + +# ----------------------------------------------------------------------------- +# Ruff: linter + formatter (replaces flake8, isort, black, pyupgrade, etc.). +# Lint (with --fix) MUST run before the formatter. +# ----------------------------------------------------------------------------- +[[repos]] +repo = "https://github.com/astral-sh/ruff-pre-commit" +rev = "v0.17.5" +hooks = [ + { id = "ruff-check", args = ["--fix"], types_or = ["python", "pyi", "jupyter"] }, + { id = "ruff-format", types_or = ["python", "pyi", "jupyter"] }, +] + +# ----------------------------------------------------------------------------- +# Static type checking. Paths are configured in pyproject.toml under [tool.ty]. +# ----------------------------------------------------------------------------- +[[repos]] +repo = "https://github.com/astral-sh/ty-pre-commit" +rev = "v0.0.107" +hooks = [ + { id = "ty" }, +] + +# ----------------------------------------------------------------------------- +# Spelling: catches typos in code, docstrings, and docs. +# Configure ignores in pyproject.toml under [tool.codespell]. +# ----------------------------------------------------------------------------- +[[repos]] +repo = "https://github.com/codespell-project/codespell" +rev = "v2.4.3" +hooks = [ + { id = "codespell", additional_dependencies = ["tomli"] }, +] + +# ----------------------------------------------------------------------------- +# Notebooks: strip outputs before commit (clean diffs, no data leakage). +# Essential for ML repos that carry .ipynb files. +# ----------------------------------------------------------------------------- +[[repos]] +repo = "https://github.com/kynan/nbstripout" +rev = "0.9.1" +hooks = [ + { id = "nbstripout" }, +] + +# ----------------------------------------------------------------------------- +# Secret scanning (baseline-driven). Generate the baseline once with: +# detect-secrets scan > .secrets.baseline +# ----------------------------------------------------------------------------- +[[repos]] +repo = "https://github.com/Yelp/detect-secrets" +rev = "v1.5.0" +hooks = [ + { id = "detect-secrets", args = ["--baseline", ".secrets.baseline"] }, +] + +# ----------------------------------------------------------------------------- +# uv lock file update to make sure it is properly synced even if the +# dependencies have been updated. +# ----------------------------------------------------------------------------- +[[repos]] +repo = "https://github.com/astral-sh/uv-pre-commit" +rev = "0.13.8" # uv version +hooks = [ + { id = "uv-lock" }, +] diff --git a/pyproject.toml b/pyproject.toml index 990c8be..5801add 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,27 +81,17 @@ docstring-code-format = true quote-style = "double" # ----------------------------------------------------------------------------- -# mypy: static type checking +# static type checking # ----------------------------------------------------------------------------- -[tool.mypy] -files = ["src"] -python_version = "3.11" -strict = true -show_error_codes = true -warn_unreachable = true -warn_redundant_casts = true -warn_unused_ignores = true -enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] +[tool.ty.environment] +python-version = "3.11" -# Silence missing stubs for third-party libs that don't ship types. -[[tool.mypy.overrides]] -module = [ - "numpy.*", - "pandas.*", - "reportlab.*", - "sklearn.*", -] -ignore_missing_imports = true +[tool.ty.src] +include = ["src", "tests"] +exclude = ["src/generated"] + +[tool.ty.rules] +unused-ignore-comment = "error" # ----------------------------------------------------------------------------- # coverage @@ -124,4 +114,5 @@ dev = [ "pytest>=9.1.1", "pytest-cov>=7.1.0", "zensical>=0.0.50", + "reportlab>=4.2" ] diff --git a/src/dstrack/_benchmark/_cli.py b/src/dstrack/_benchmark/_cli.py index 7795b55..9a686fa 100644 --- a/src/dstrack/_benchmark/_cli.py +++ b/src/dstrack/_benchmark/_cli.py @@ -2,10 +2,10 @@ import shutil import tempfile -from collections.abc import Iterator +from collections.abc import Generator from contextlib import contextmanager from pathlib import Path -from typing import Annotated +from typing import Annotated, Protocol import typer @@ -33,8 +33,16 @@ def _main() -> None: """ +class _CsvKeptSink(Protocol): + """What [_workspace][dstrack._benchmark._cli._workspace] needs from a reporter.""" + + def csv_kept(self, path: Path) -> None: ... + + @contextmanager -def _workspace(reporter: ConsoleReporter, *, keep_csv: bool) -> Iterator[Path]: +def _workspace( + reporter: _CsvKeptSink, *, keep_csv: bool +) -> Generator[Path, None, None]: """Yield a path for the synthetic CSV, discarding it unless ``keep_csv``.""" tmp_dir = Path(tempfile.mkdtemp(prefix="dstrack-benchmark-")) csv_path = tmp_dir / "synthetic_dataset.csv" diff --git a/src/dstrack/_benchmark/_profiling.py b/src/dstrack/_benchmark/_profiling.py index 70fdc1a..13921ce 100644 --- a/src/dstrack/_benchmark/_profiling.py +++ b/src/dstrack/_benchmark/_profiling.py @@ -104,7 +104,7 @@ class CallGraph: @classmethod def from_profile(cls, profiler: cProfile.Profile, scope: CallScope) -> "CallGraph": """Build the in-scope call graph of a profiler that has finished running.""" - raw: _RawStats = pstats.Stats(profiler).stats # type: ignore[attr-defined] + raw: _RawStats = pstats.Stats(profiler).stats # ty: ignore[missing-attribute] graph = cls(_scope=scope) frames = {key for key in raw if scope.contains(key[0])} callees: set[FrameKey] = set() diff --git a/src/dstrack/_benchmark/_runner.py b/src/dstrack/_benchmark/_runner.py index b426be2..3f91ddb 100644 --- a/src/dstrack/_benchmark/_runner.py +++ b/src/dstrack/_benchmark/_runner.py @@ -9,7 +9,7 @@ import cProfile import time -from collections.abc import Iterator +from collections.abc import Generator from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path @@ -175,7 +175,7 @@ def run(self, csv_path: Path) -> BenchmarkRun: ) @contextmanager - def _timed(self, *, profiled: bool = False) -> Iterator[_Elapsed]: + def _timed(self, *, profiled: bool = False) -> Generator[_Elapsed, None, None]: """Time the block, optionally recording it into the shared profiler.""" profiler = self._profiler if profiled else None elapsed = _Elapsed() diff --git a/src/dstrack/_diff_report.py b/src/dstrack/_diff_report.py index 0f632d9..7450ff0 100644 --- a/src/dstrack/_diff_report.py +++ b/src/dstrack/_diff_report.py @@ -705,7 +705,7 @@ def _numbered_canvas(kit: _Kit, *, now: datetime) -> Any: moment = now.astimezone(UTC) if now.tzinfo is not None else now stamp = moment.strftime("%Y-%m-%d %H:%M UTC") - class _NumberedCanvas(kit.canvas.Canvas): # type: ignore[misc,name-defined] + class _NumberedCanvas(kit.canvas.Canvas): def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) self._pages: list[dict[str, Any]] = [] diff --git a/src/dstrack/readers/_registry.py b/src/dstrack/readers/_registry.py index 5339eba..23472ec 100644 --- a/src/dstrack/readers/_registry.py +++ b/src/dstrack/readers/_registry.py @@ -77,11 +77,7 @@ def check_reader_class(obj: object, *, origin: str) -> type[TabularReader]: f"{qualname} (from {origin}) does not satisfy the TabularReader " "protocol: it must define columns() and iter_batches()." ) - # Checked through an object-typed name: mypy assumes any class object - # already satisfies ReaderFactory, so narrowing from `object` is what keeps - # this branch live for the type checker as well as at runtime. - candidate: object = obj - if not isinstance(candidate, ReaderFactory): + if not isinstance(obj, ReaderFactory): raise TypeError( f"{qualname} (from {origin}) does not satisfy the ReaderFactory " "protocol: it must define a from_path(path) classmethod, which is " diff --git a/src/dstrack/snapshot/_stats.py b/src/dstrack/snapshot/_stats.py index 54c73c9..f1ef908 100644 --- a/src/dstrack/snapshot/_stats.py +++ b/src/dstrack/snapshot/_stats.py @@ -1,7 +1,7 @@ import math from collections import Counter from dataclasses import dataclass, field -from typing import Final, Protocol +from typing import Final, Protocol, TypeAlias from dstrack.errors import InputTooLargeError from dstrack.readers import Cell, TabularReader @@ -12,6 +12,12 @@ _NUM_HISTOGRAM_BINS: int = 20 _TOP_VALUES_K: int = 50 +_NonNullCell: TypeAlias = int | float | str | bool +"""A [Cell][dstrack.readers._protocol.Cell] known not to be ``None``. + +Nulls are counted by the row scan and never reach an accumulator's ``update``. +""" + DEFAULT_MAX_ROWS: Final = 10_000_000 """Default ceiling on the number of rows a single snapshot pass will accumulate. @@ -123,7 +129,7 @@ class _ColumnAcc(Protocol): null_count: int - def update(self, val: Cell) -> None: ... + def update(self, val: _NonNullCell) -> None: ... def is_constant(self) -> bool: ... @@ -144,8 +150,8 @@ class _NumericAcc: vals: list[float] = field(default_factory=list) null_count: int = 0 - def update(self, val: Cell) -> None: - f = float(val) # type: ignore[arg-type] + def update(self, val: _NonNullCell) -> None: + f = float(val) if not math.isfinite(f): self.null_count += 1 return @@ -172,7 +178,7 @@ class _StringAcc: non_null_count: int = 0 null_count: int = 0 - def update(self, val: Cell) -> None: + def update(self, val: _NonNullCell) -> None: s = str(val) self.non_null_count += 1 self.counter[s] += 1 @@ -206,7 +212,7 @@ class _DatetimeAcc: non_null_count: int = 0 null_count: int = 0 - def update(self, val: Cell) -> None: + def update(self, val: _NonNullCell) -> None: s = str(val) if self.non_null_count == 0 or s < self.min_val: self.min_val = s @@ -230,7 +236,7 @@ class _OtherAcc: null_count: int = 0 - def update(self, val: Cell) -> None: + def update(self, val: _NonNullCell) -> None: pass def is_constant(self) -> bool: diff --git a/tests/test_cli.py b/tests/test_cli.py index 00c204e..109dedd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -79,7 +79,9 @@ def test_init_local_store_does_not_touch_destination_until_built( original_materialize = _store_template.materialize - def _check_and_materialize(template: object, parent: Path) -> Path: + def _check_and_materialize( + template: _store_template.TemplateDir, parent: Path + ) -> Path: assert not store_path.exists() return original_materialize(template, parent) diff --git a/tests/test_diff_cli.py b/tests/test_diff_cli.py index b0a252f..92315b2 100644 --- a/tests/test_diff_cli.py +++ b/tests/test_diff_cli.py @@ -6,7 +6,7 @@ from pathlib import Path import pytest -from typer.testing import CliRunner +from typer.testing import CliRunner, Result from dstrack import cache, store from dstrack._cli import app @@ -69,7 +69,7 @@ def tracked(tmp_path: Path, store_root: Path) -> Path: return csv -def _diff(*args: str, width: str = "160") -> object: +def _diff(*args: str, width: str = "160") -> Result: """Invoke `dstrack diff` with `--yes`, at a fixed terminal width.""" return runner.invoke(app, ["diff", *args, "--yes"], env={"COLUMNS": width}) diff --git a/tests/test_diff_resolve.py b/tests/test_diff_resolve.py index 0c8d8b9..a89e5be 100644 --- a/tests/test_diff_resolve.py +++ b/tests/test_diff_resolve.py @@ -9,7 +9,7 @@ from dstrack import cache, store from dstrack._cli import app -from dstrack._resolve import resolve_pair, resolve_snapshot_ref +from dstrack._resolve import ResolvedSnapshot, resolve_pair, resolve_snapshot_ref from dstrack.errors import ( AmbiguousSnapshotError, DatasetNotFoundError, @@ -64,7 +64,7 @@ def _synced(root: Path) -> Path: return root -def _resolve(spec: str, root: Path) -> object: +def _resolve(spec: str, root: Path) -> ResolvedSnapshot: """Resolve one reference against a synced store.""" return resolve_snapshot_ref(spec, store_root=_synced(root), root=None) @@ -362,7 +362,7 @@ def _pair( *, from_ref: str | None = None, to_ref: str | None = None, -) -> tuple[object, object]: +) -> tuple[ResolvedSnapshot, ResolvedSnapshot]: """Resolve both sides against a synced store.""" return resolve_pair( specs, diff --git a/tests/test_log.py b/tests/test_log.py index cb86b47..d3a917c 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -10,7 +10,7 @@ import pytest import typer -from typer.testing import CliRunner +from typer.testing import CliRunner, Result from dstrack import _resolve, cache from dstrack._cli import app @@ -81,7 +81,7 @@ def _pretend_terminal(monkeypatch: pytest.MonkeyPatch) -> None: ) -def _log_cmd(*args: str) -> object: +def _log_cmd(*args: str) -> Result: """Invoke `dstrack log` with `--yes`, so the index is built unprompted.""" return runner.invoke(app, ["log", *args, "--yes"]) diff --git a/tests/test_logging.py b/tests/test_logging.py index 991cfa7..3524f4a 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -59,18 +59,20 @@ def test_configure_default_logging_is_idempotent() -> None: def test_emits_when_unconfigured(monkeypatch: pytest.MonkeyPatch) -> None: """With no root handlers and no other dstrack handlers, records are written out.""" - handler = _DefaultHandler(io.StringIO()) + stream = io.StringIO() + handler = _DefaultHandler(stream) monkeypatch.setattr(logging.root, "handlers", [], raising=False) monkeypatch.setattr(logger, "handlers", [handler], raising=False) handler.emit(make_record(msg="uh oh")) - assert "uh oh" in handler.stream.getvalue() + assert "uh oh" in stream.getvalue() def test_yields_when_root_is_configured(monkeypatch: pytest.MonkeyPatch) -> None: """If the root logger has handlers (app called basicConfig etc.), stay silent.""" - handler = _DefaultHandler(io.StringIO()) + stream = io.StringIO() + handler = _DefaultHandler(stream) monkeypatch.setattr( logging.root, "handlers", [logging.NullHandler()], raising=False ) @@ -78,14 +80,15 @@ def test_yields_when_root_is_configured(monkeypatch: pytest.MonkeyPatch) -> None handler.emit(make_record(msg="should not appear")) - assert handler.stream.getvalue() == "" + assert stream.getvalue() == "" def test_yields_when_dstrack_logger_has_other_handlers( monkeypatch: pytest.MonkeyPatch, ) -> None: """If the app attached its own handler to the dstrack logger, stay silent.""" - handler = _DefaultHandler(io.StringIO()) + stream = io.StringIO() + handler = _DefaultHandler(stream) monkeypatch.setattr(logging.root, "handlers", [], raising=False) monkeypatch.setattr( logger, "handlers", [handler, logging.NullHandler()], raising=False @@ -93,4 +96,4 @@ def test_yields_when_dstrack_logger_has_other_handlers( handler.emit(make_record(msg="should not appear")) - assert handler.stream.getvalue() == "" + assert stream.getvalue() == "" diff --git a/tests/test_paths.py b/tests/test_paths.py index c779a78..fb7a5dd 100644 --- a/tests/test_paths.py +++ b/tests/test_paths.py @@ -112,9 +112,9 @@ def __init__(self, real_result: os.stat_result, st_dev: int) -> None: def __getattr__(self, name: str) -> object: return getattr(self._real_result, name) - def fake_stat(self: Path, *args: object, **kwargs: object) -> _FakeStat: + def fake_stat(self: Path, *, follow_symlinks: bool = True) -> _FakeStat: on_mount = self == nested or nested in self.parents - real_result = real_stat(self, *args, **kwargs) + real_result = real_stat(self, follow_symlinks=follow_symlinks) return _FakeStat(real_result, st_dev=1 if on_mount else 2) monkeypatch.setattr(Path, "stat", fake_stat) diff --git a/tests/test_reader_registry.py b/tests/test_reader_registry.py index af531f4..04c0a69 100644 --- a/tests/test_reader_registry.py +++ b/tests/test_reader_registry.py @@ -296,7 +296,7 @@ def test_instance_method_from_path_is_rejected_unconstructed(tmp_path: Path) -> def test_register_reader_rejects_instance_method_from_path() -> None: """The non-classmethod check also guards registration, not just build time.""" with pytest.raises(TypeError, match="classmethod"): - register_reader(InstanceFromPathReader, name="bad") # type: ignore[arg-type] + register_reader(InstanceFromPathReader, name="bad") assert "bad" not in available_readers() @@ -330,7 +330,7 @@ def test_non_class_target_is_rejected() -> None: def test_register_reader_rejects_invalid_class() -> None: """A class that fails validation leaves no half-finished registration behind.""" with pytest.raises(TypeError, match="ReaderFactory"): - register_reader(NoFromPathReader, name="bad") # type: ignore[arg-type] + register_reader(NoFromPathReader, name="bad") assert "bad" not in available_readers() diff --git a/tests/test_readers.py b/tests/test_readers.py index b536b61..a41df3f 100644 --- a/tests/test_readers.py +++ b/tests/test_readers.py @@ -3,7 +3,7 @@ import pytest -from dstrack.readers import ColumnInfo, CsvReader, TabularReader +from dstrack.readers import Cell, ColumnInfo, CsvReader, TabularReader # --------------------------------------------------------------------------- # Helpers @@ -17,7 +17,7 @@ def write_csv(tmp_path: Path, name: str, content: str) -> Path: return p -def flat_rows(reader: CsvReader, batch_size: int = 1000) -> list[list[object]]: +def flat_rows(reader: CsvReader, batch_size: int = 1000) -> list[list[Cell]]: """Flatten all batches from *reader* into a single list of rows.""" return [row for batch in reader.iter_batches(batch_size) for row in batch] @@ -40,7 +40,7 @@ class _Stub: def columns(self) -> list[ColumnInfo]: return [] - def iter_batches(self, batch_size: int = 1000): # type: ignore[override] + def iter_batches(self, batch_size: int = 1000): return iter([]) assert isinstance(_Stub(), TabularReader) @@ -219,7 +219,7 @@ def test_column_info_is_immutable() -> None: """ColumnInfo fields cannot be reassigned after construction.""" col = ColumnInfo(name="x", dtype="int64") with pytest.raises((AttributeError, TypeError)): - col.name = "y" # type: ignore[misc] + col.name = "y" # ty: ignore[invalid-assignment] def test_raise_when_file_is_modified(tmp_path: Path) -> None: diff --git a/tests/test_snapshot_metadata.py b/tests/test_snapshot_metadata.py index 7a90c2b..d5048f3 100644 --- a/tests/test_snapshot_metadata.py +++ b/tests/test_snapshot_metadata.py @@ -5,7 +5,7 @@ from collections.abc import Iterator from pathlib import Path, PurePath, PureWindowsPath -from dstrack.readers import ColumnInfo, TabularReader +from dstrack.readers import Cell, ColumnInfo, TabularReader from dstrack.snapshot import MetadataBuilder, SnapshotMetadata # --------------------------------------------------------------------------- @@ -18,10 +18,10 @@ class _Stub: def columns(self) -> list[ColumnInfo]: return list(cols) - def iter_batches(self, batch_size: int = 1000) -> Iterator[list[list[object]]]: + def iter_batches(self, batch_size: int = 1000) -> Iterator[list[list[Cell]]]: return iter([]) - return _Stub() # type: ignore[return-value] + return _Stub() def _build( diff --git a/tests/test_snapshot_stats.py b/tests/test_snapshot_stats.py index 09fb675..ee05644 100644 --- a/tests/test_snapshot_stats.py +++ b/tests/test_snapshot_stats.py @@ -8,7 +8,13 @@ from dstrack.errors import InputTooLargeError from dstrack.readers import Cell, ColumnInfo, TabularReader -from dstrack.snapshot import DatasetStats, StatsComputer +from dstrack.snapshot import ( + DatasetStats, + DatetimeColumnStats, + NumericColumnStats, + StatsComputer, + StringColumnStats, +) # --------------------------------------------------------------------------- # Helpers @@ -24,7 +30,7 @@ def iter_batches(self, batch_size: int = 1000) -> Iterator[list[list[Cell]]]: if rows: yield rows - return _Stub() # type: ignore[return-value] + return _Stub() def _compute(cols: list[ColumnInfo], rows: list[list[Cell]]) -> DatasetStats: @@ -172,6 +178,7 @@ def test_numeric_basic_stats() -> None: [[1], [2], [3], [4], [5]], ) s = stats.column_stats["v"] + assert isinstance(s, NumericColumnStats) assert s.min == 1.0 assert s.max == 5.0 assert s.mean == pytest.approx(3.0, abs=1e-9) @@ -197,7 +204,9 @@ def test_numeric_std_single_value() -> None: [ColumnInfo("v", "int64")], [[42]], ) - assert stats.column_stats["v"].std == pytest.approx(0.0, abs=1e-9) + s = stats.column_stats["v"] + assert isinstance(s, NumericColumnStats) + assert s.std == pytest.approx(0.0, abs=1e-9) def test_numeric_percentiles_present() -> None: @@ -206,7 +215,9 @@ def test_numeric_percentiles_present() -> None: [ColumnInfo("v", "int64")], [[i] for i in range(1, 101)], ) - p = stats.column_stats["v"].percentiles + s = stats.column_stats["v"] + assert isinstance(s, NumericColumnStats) + p = s.percentiles assert {f.name for f in fields(p)} == {"p5", "p25", "p50", "p75", "p95", "p99"} assert p.p50 == pytest.approx(50.0, abs=2) assert p.p5 == pytest.approx(5.0, abs=2) @@ -218,12 +229,14 @@ def test_numeric_percentiles_present() -> None: def test_numeric_histogram_bin_edge_count() -> None: """Histogram bin_edges has one more entry than counts, and counts sum to num_rows.""" - rows = [[i] for i in range(1, 101)] + rows: list[list[Cell]] = [[i] for i in range(1, 101)] stats = _compute( [ColumnInfo("v", "int64")], rows, ) - hist = stats.column_stats["v"].histogram + s = stats.column_stats["v"] + assert isinstance(s, NumericColumnStats) + hist = s.histogram counts = hist.counts edges = hist.bin_edges assert len(edges) == len(counts) + 1 @@ -237,6 +250,7 @@ def test_numeric_all_null_returns_zeros() -> None: [[None], [None]], ) s = stats.column_stats["v"] + assert isinstance(s, NumericColumnStats) assert s.mean == 0.0 assert s.std == 0.0 assert s.num_unique == 0 @@ -249,6 +263,7 @@ def test_bool_treated_as_numeric() -> None: [[True], [False], [True]], ) s = stats.column_stats["flag"] + assert isinstance(s, NumericColumnStats) assert s.mean == pytest.approx(2 / 3, abs=1e-9) assert s.min == 0.0 assert s.max == 1.0 @@ -262,6 +277,7 @@ def test_numeric_nan_treated_as_null() -> None: [[1.0], [nan], [3.0]], ) s = stats.column_stats["v"] + assert isinstance(s, NumericColumnStats) assert s.null_count == 1 assert s.null_fraction == pytest.approx(1 / 3, abs=1e-9) assert s.min == 1.0 @@ -278,6 +294,7 @@ def test_numeric_infinity_treated_as_null() -> None: [[1.0], [float("inf")], [float("-inf")], [5.0]], ) s = stats.column_stats["v"] + assert isinstance(s, NumericColumnStats) assert s.null_count == 2 assert s.null_fraction == pytest.approx(0.5, abs=1e-9) assert s.min == 1.0 @@ -293,6 +310,7 @@ def test_numeric_all_non_finite_returns_zeros() -> None: [[float("nan")], [float("inf")]], ) s = stats.column_stats["v"] + assert isinstance(s, NumericColumnStats) assert s.null_count == 2 assert s.mean == 0.0 assert s.std == 0.0 @@ -305,7 +323,9 @@ def test_numeric_non_finite_histogram_does_not_raise() -> None: rows: list[list[Cell]] = [[float(i)] for i in range(10)] rows.append([float("nan")]) stats = _compute([ColumnInfo("v", "float64")], rows) - hist = stats.column_stats["v"].histogram + s = stats.column_stats["v"] + assert isinstance(s, NumericColumnStats) + hist = s.histogram assert len(hist.bin_edges) == len(hist.counts) + 1 assert sum(hist.counts) == 10 @@ -322,6 +342,7 @@ def test_string_basic_stats() -> None: [["hello"], ["world"], ["hello"]], ) s = stats.column_stats["s"] + assert isinstance(s, StringColumnStats) assert s.null_count == 0 assert s.num_unique == 2 assert s.top_values == {"hello": 2, "world": 1} @@ -335,6 +356,7 @@ def test_string_char_length_stats() -> None: [["a"], ["bb"], ["ccc"]], ) s = stats.column_stats["s"] + assert isinstance(s, StringColumnStats) assert s.min_char_length == 1 assert s.max_char_length == 3 assert s.avg_char_length == pytest.approx(2.0, abs=1e-9) @@ -347,6 +369,7 @@ def test_string_token_count_stats() -> None: [["one"], ["two three"], ["four five six"]], ) s = stats.column_stats["s"] + assert isinstance(s, StringColumnStats) assert s.min_token_count == 1 assert s.max_token_count == 3 assert s.avg_token_count == pytest.approx(2.0, abs=1e-9) @@ -369,6 +392,7 @@ def test_string_top_values_coverage_partial() -> None: rows: list[list[Cell]] = [[str(i)] for i in range(200)] stats = _compute(cols, rows) s = stats.column_stats["s"] + assert isinstance(s, StringColumnStats) # 50 top values out of 200 = 0.25 coverage assert s.top_values_coverage == pytest.approx(50 / 200, abs=1e-9) @@ -385,6 +409,7 @@ def test_datetime_min_max() -> None: [["2024-01-01"], ["2024-06-15"], ["2023-12-31"]], ) s = stats.column_stats["dt"] + assert isinstance(s, DatetimeColumnStats) assert s.min == "2023-12-31" assert s.max == "2024-06-15" @@ -396,6 +421,7 @@ def test_datetime_range_days() -> None: [["2024-01-01"], ["2024-01-11"]], ) s = stats.column_stats["dt"] + assert isinstance(s, DatetimeColumnStats) assert s.range_days == pytest.approx(10.0, abs=1e-9) @@ -417,6 +443,7 @@ def test_datetime_all_null() -> None: [[None], [None]], ) s = stats.column_stats["dt"] + assert isinstance(s, DatetimeColumnStats) assert s.min == "" assert s.max == "" assert s.range_days == pytest.approx(0.0, abs=1e-9) @@ -439,7 +466,7 @@ def columns(self) -> list[ColumnInfo]: def iter_batches(self, batch_size: int = 1000) -> Iterator[list[list[Cell]]]: yield from batches - return _Stub() # type: ignore[return-value] + return _Stub() def test_max_rows_at_limit_computes_normally() -> None: diff --git a/uv.lock b/uv.lock index e840bcb..6cc0eda 100644 --- a/uv.lock +++ b/uv.lock @@ -233,6 +233,7 @@ dev = [ { name = "mkdocstrings-python" }, { name = "pytest" }, { name = "pytest-cov" }, + { name = "reportlab" }, { name = "zensical" }, ] @@ -250,6 +251,7 @@ dev = [ { name = "mkdocstrings-python", specifier = ">=2.0.5" }, { name = "pytest", specifier = ">=9.1.1" }, { name = "pytest-cov", specifier = ">=7.1.0" }, + { name = "reportlab", specifier = ">=4.2" }, { name = "zensical", specifier = ">=0.0.50" }, ]