From f55c36a6b9aa34e53e37ce4897eb230b4c049148 Mon Sep 17 00:00:00 2001 From: Leonardo Ayala Date: Sat, 21 Nov 2026 19:24:46 +0100 Subject: [PATCH] Enhance documentation and constants in the dstrack module - Updated docstrings for DIFF_FORMAT_VERSION, DEFAULT_OPTIONS, and Cell type alias to improve clarity. - Added a docstring for STORE_DIRNAME to describe its purpose. - Introduced DEFAULT_MAX_ROWS constant with a detailed explanation of its usage in snapshot statistics. - Improved the organization of the snapshot module's __init__.py to include DEFAULT_MAX_ROWS in the exported symbols. - Adjusted theme palette configuration in zensical.toml to clarify media queries and toggle functionality for light/dark modes. --- docs/API/errors.md | 5 ++ docs/getting_started.md | 145 ++++++++++++++++++------------- src/dstrack/diff/_model.py | 20 +++-- src/dstrack/paths.py | 1 + src/dstrack/readers/_protocol.py | 2 +- src/dstrack/snapshot/__init__.py | 7 ++ src/dstrack/snapshot/_stats.py | 17 ++-- zensical.toml | 14 ++- 8 files changed, 137 insertions(+), 74 deletions(-) create mode 100644 docs/API/errors.md diff --git a/docs/API/errors.md b/docs/API/errors.md new file mode 100644 index 0000000..17123d0 --- /dev/null +++ b/docs/API/errors.md @@ -0,0 +1,5 @@ +--- +icon: lucide/triangle-alert +--- + +::: dstrack.errors diff --git a/docs/getting_started.md b/docs/getting_started.md index 71d8e76..813ffd5 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -129,22 +129,16 @@ dataset's path, and `dstrack` works out which dataset that is: dstrack log data.csv ``` -```text -● a587afd9 Customers HEAD -│ 2 minutes ago by you -│ 4 rows +1 3 cols -│ data.csv -│ -● 605eb940 Customers -│ 5 minutes ago by you -│ 3 rows +1 3 cols -│ data.csv -│ -● 0c82297f Customers - 1 hour ago by you - 2 rows 3 cols - data.csv -``` +
● e9d6f0d7  sample  HEAD
+│ 1 week ago  by menjivar
+│ 54 rows +4   8 cols +1
+│ data/sample.csv        
+│  
+● e83517d4  sample  
+  1 week ago  by menjivar
+  50 rows   7 cols  
+  data/sample.csv        
+
Each node is one snapshot, showing how its row and column counts changed from the snapshot below it. `HEAD` marks the latest. @@ -226,48 +220,83 @@ for column in result.columns: print(column.name, column.distribution_shift.score) ``` -??? example "Exemplary bash diff output" - ```bash title="Command lind output" - ✔ Wrote the comparison report to report.pdf - ╭────────────────── dstrack diff ───────────────────╮ - │ before after │ - │ snapshot 0d86445c 6f6577b3 │ - │ name sample sample │ - │ path data/sample.csv data/sample.csv │ - │ taken 1 minute ago 55 seconds ago │ - │ by menjivar menjivar │ - │ rows 50 54 │ - │ columns 7 8 │ - │ source 2dfa4d9c36ae… 5887d2bc5a31… changed │ - ╰─────────────────── CRITICAL ────────────────────╯ - - Schema - column before after - ──────────────────────────────────────────────── - + country — string not null - ~ age int64 not null int64 null - - Dataset - • num rows 50 54 +4 (+8.0%) - ⚠ num columns 7 8 +1 (+14.3%) - - Columns - Numeric columns - column nulls mean std p50 min max uniq shift - ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── - ⚠ id 0 25.5 → 28.11 14.43 → 16.31 25.5 → 27.5 1 50 → 56 50 → 54 PSI 0.23 - ⚠ age 0 → 0.09259 44.14 → 45.37 18.31 → 18.94 37.5 → 39 18 75 33 → 31 PSI 0.15 - ❗ balance 0 1,235 → 2,213 684.5 → 1,180 1,226 → 2,192 97.28 → 170.2 2,403 → 4,202 50 → 54 PSI 1.09 - • is_active 0 0.5 0.5 0.5 0 1 2 PSI 0.00 - Text columns - column nulls uniq coverage avg len shift - ────────────────────────────────────────────────────────────────────────────────────────────────────────────── - • name 0 28 1 13.88 → 13.93 TVD 0.10 +Margaret Hopper -Katherine Torvalds - ⚠ plan 0 3 1 5.9 → 7 TVD 0.18 - • country (new) — → 0 — → 5 — → 1 — → 2 — - - +4 rows +1 columns 4 drifted - ``` +
+Exemplary bash diff output +
╭────────────────── dstrack diff ───────────────────╮
+│           before           after                  │
+│ snapshot  e83517d4         e9d6f0d7               │
+│ name      sample           sample                 │
+│ path      data/sample.csv  data/sample.csv        │
+│ taken     1 week ago       1 week ago             │
+│ by        menjivar         menjivar               │
+│ rows      50               54                     │
+│ columns   7                8                      │
+│ source    2dfa4d9c36ae…    1746ddc049a2…  changed │
+╰───────────────────  CRITICAL  ────────────────────╯
+
+Schema
+     column        before                after           
+─────────────────────────────────────────────────────────
+ +   country       —                     string not null
+ ~   age           int64 not null        int64 null  
+ ~   signup_date   datetime64 not null   datetime64 null
+
+Dataset
+•  num rows     50  54   +4 (+8.0%)
+⚠  num columns   7   8  +1 (+14.3%)
+
+Column sets
+⚠  constant columns   +is_active  
+⚠  high null columns  +signup_date
+
+Columns
+  Numeric columns
+      column            nulls            mean             std             p50             min             max      uniq   shift     
+────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
+ ⚠    id                    0    25.5 → 29.22   14.43 → 16.76     25.5 → 29.5               1         50 → 57   50 → 54   PSI 0.22 
+ ⚠    age         0 → 0.09259   44.14 → 44.94   18.31 → 17.45       37.5 → 39              18              75   33 → 35   PSI 0.10 
+ ❗   balance               0   1,235 → 1,678   684.5 → 920.2   1,226 → 1,748   97.28 → 80.57   2,403 → 3,190   50 → 54   PSI 0.84 
+ ❗   is_active             0         0.5 → 1         0.5 → 0         0.5 → 1           0 → 1               1     2 → 1   pctl 1.00
+  Text columns
+     column          nulls      uniq   coverage         avg len   shift                              
+─────────────────────────────────────────────────────────────────────────────────────────────────────
+ •   name                0   28 → 30          1   13.88 → 13.83   TVD 0.08 +Ada Turing +Alan Johnson
+ ⚠   plan                0         3          1     5.9 → 6.944   TVD 0.16                          
+ •   country (new)   — → 0     — → 5      — → 1           — → 2   —                                 
+  Datetime columns
+      column             nulls   range (d) 
+───────────────────────────────────────────
+ ❗   signup_date   0 → 0.7407   360 → 353
+
+Other column changes
+•   id           percentiles.p5                                 3.45 → 3.65        +0.2 (+5.8%)
+•   id           percentiles.p25                               13.25 → 14.5       +1.25 (+9.4%)
+⚠   id           percentiles.p75                              37.75 → 43.75         +6 (+15.9%)
+⚠   id           percentiles.p95                              47.55 → 54.35       +6.8 (+14.3%)
+⚠   id           percentiles.p99                              49.51 → 56.47      +6.96 (+14.1%)
+⚠   age          null_count                                           0 → 5                  +5
+•   age          percentiles.p5                                 20.9 → 20.8        -0.1 (-0.5%)
+•   age          percentiles.p25                                    29 → 30          +1 (+3.4%)
+•   age          percentiles.p75                                 60.75 → 59       -1.75 (-2.9%)
+⚠   balance      percentiles.p5                               154.1 → 179.6     +25.58 (+16.6%)
+❗  balance      percentiles.p25                              601.7 → 992.7       +391 (+65.0%)
+⚠   balance      percentiles.p75                              1,828 → 2,504       +676 (+37.0%)
+⚠   balance      percentiles.p95                              2,237 → 2,965     +727.2 (+32.5%)
+⚠   balance      percentiles.p99                              2,385 → 3,164     +779.3 (+32.7%)
+⚠   is_active    percentiles.p5                                       0 → 1                  +1
+⚠   is_active    percentiles.p25                                      0 → 1                  +1
+•   country      null_count                                           — → 0  
+•   country      min_char_length                                      — → 2  
+•   country      max_char_length                                      — → 2  
+•   country      avg_token_count                                      — → 1  
+•   country      min_token_count                                      — → 1  
+•   country      max_token_count                                      — → 1  
+⚠   signup_date  null_count                                          0 → 40                 +40
+•   signup_date  min              2023-01-02 00:00:00 → 2023-01-09 00:00:00  +6.048e+05 (+0.0%)
+
++4 rows   +1 columns   6 columns drifted   +1 constant columns   +1 high null columns
+
+
??? example "Exemplary PDF diff report"
diff --git a/src/dstrack/diff/_model.py b/src/dstrack/diff/_model.py index 14a541d..f980882 100644 --- a/src/dstrack/diff/_model.py +++ b/src/dstrack/diff/_model.py @@ -18,10 +18,13 @@ from enum import StrEnum from typing import Any, Final -# Bumped only when a key already emitted changes meaning or disappears. Adding -# a key is not a version change: consumers are expected to ignore what they do -# not recognize. DIFF_FORMAT_VERSION: Final = "1" +"""Version of the serialized diff format. + +Bumped only when a key already emitted changes meaning or disappears. Adding a +key is not a version change: consumers are expected to ignore what they do not +recognize. +""" # Relative thresholds, for counts and other unbounded magnitudes. DEFAULT_REL_TOLERANCE: Final = 1e-3 DEFAULT_ABS_TOLERANCE: Final = 1e-12 @@ -385,11 +388,8 @@ class DiffOptions: DEFAULT_OPTIONS: Final = DiffOptions() +"""The thresholds used when no options are given; same as the `normal` preset.""" -# Named sensitivities, so the CLI can offer a choice without exposing twelve -# float flags. `low` is for datasets that move constantly and where only a -# structural break matters; `high` is for a curated table that should not move -# at all. PRESETS: Final[dict[str, DiffOptions]] = { "low": DiffOptions( warn_pct_change=0.25, @@ -417,6 +417,12 @@ class DiffOptions: critical_percentile_shift=0.10, ), } +"""Named sensitivities, so the CLI can offer a choice without exposing twelve +float flags. + +`low` is for datasets that move constantly and where only a structural break +matters; `high` is for a curated table that should not move at all. +""" def preset(name: str) -> DiffOptions: diff --git a/src/dstrack/paths.py b/src/dstrack/paths.py index 7997cbb..b607d03 100644 --- a/src/dstrack/paths.py +++ b/src/dstrack/paths.py @@ -12,6 +12,7 @@ from dstrack.utils import get_invocation_path STORE_DIRNAME: Final = ".dstrack" +"""Name of the directory that holds the local store.""" ROOT_PATH_ENV_VAR: Final = "DSTRACK_ROOT_PATH" diff --git a/src/dstrack/readers/_protocol.py b/src/dstrack/readers/_protocol.py index ae3230e..d57f9b3 100644 --- a/src/dstrack/readers/_protocol.py +++ b/src/dstrack/readers/_protocol.py @@ -3,8 +3,8 @@ from pathlib import Path from typing import Protocol, TypeAlias, runtime_checkable -# A single cell value after dtype coercion; None represents a missing value. Cell: TypeAlias = int | float | str | bool | None +"""A single cell value after dtype coercion; `None` represents a missing value.""" @dataclass(frozen=True) diff --git a/src/dstrack/snapshot/__init__.py b/src/dstrack/snapshot/__init__.py index fffe4b6..8349aca 100644 --- a/src/dstrack/snapshot/__init__.py +++ b/src/dstrack/snapshot/__init__.py @@ -15,11 +15,17 @@ ------------ [SnapshotMetadata][dstrack.snapshot._metadata.SnapshotMetadata], [DatasetStats][dstrack.snapshot._stats.DatasetStats] + +Constants +--------- +[DEFAULT_MAX_ROWS][dstrack.snapshot.DEFAULT_MAX_ROWS] + Default row limit of a [StatsComputer][dstrack.snapshot._stats.StatsComputer]. """ from ._builder import SnapshotBuilder, build_snapshot_dict from ._metadata import MetadataBuilder, SnapshotMetadata from ._stats import ( + DEFAULT_MAX_ROWS, DatasetStats, DatetimeColumnStats, HistogramStats, @@ -31,6 +37,7 @@ ) __all__ = [ + "DEFAULT_MAX_ROWS", "DatasetStats", "DatetimeColumnStats", "HistogramStats", diff --git a/src/dstrack/snapshot/_stats.py b/src/dstrack/snapshot/_stats.py index 451d5d3..54c73c9 100644 --- a/src/dstrack/snapshot/_stats.py +++ b/src/dstrack/snapshot/_stats.py @@ -1,7 +1,7 @@ import math from collections import Counter from dataclasses import dataclass, field -from typing import Protocol +from typing import Final, Protocol from dstrack.errors import InputTooLargeError from dstrack.readers import Cell, TabularReader @@ -11,10 +11,13 @@ ) _NUM_HISTOGRAM_BINS: int = 20 _TOP_VALUES_K: int = 50 -# Default ceiling on the number of rows a single snapshot pass will accumulate. -# The pass keeps per-column values, the distinct-row set, and per-string counts -# in memory, so worst-case memory grows with the row count; this bounds it. -_DEFAULT_MAX_ROWS: int = 10_000_000 + +DEFAULT_MAX_ROWS: Final = 10_000_000 +"""Default ceiling on the number of rows a single snapshot pass will accumulate. + +The pass keeps per-column values, the distinct-row set, and per-string counts in +memory, so worst-case memory grows with the row count; this bounds it. +""" @dataclass(frozen=True, slots=True) @@ -255,11 +258,11 @@ class StatsComputer: yields more than this, an [InputTooLargeError][dstrack.errors.InputTooLargeError] is raised. Defaults to - [_DEFAULT_MAX_ROWS][dstrack.snapshot._stats._DEFAULT_MAX_ROWS]. + [DEFAULT_MAX_ROWS][dstrack.snapshot.DEFAULT_MAX_ROWS]. Pass ``None`` to disable the limit when enough memory is available. """ - def __init__(self, *, max_rows: int | None = _DEFAULT_MAX_ROWS) -> None: + def __init__(self, *, max_rows: int | None = DEFAULT_MAX_ROWS) -> None: self._max_rows = max_rows def compute(self, reader: TabularReader) -> DatasetStats: diff --git a/zensical.toml b/zensical.toml index 2073a20..a79cad7 100644 --- a/zensical.toml +++ b/zensical.toml @@ -295,15 +295,27 @@ repo = "fontawesome/brands/github" # Read more: # - https://zensical.org/docs/setup/colors/ # ---------------------------------------------------------------------------- +# +# The toggle cycles through the entries in order. On a first visit, the first +# entry whose "media" query matches is used, so visitors start by following +# their system preference. The light/dark "media" values must stay exactly as +# written: the "(prefers-color-scheme)" entry resolves to them. [[project.theme.palette]] +media = "(prefers-color-scheme)" +toggle.icon = "lucide/sun-moon" +toggle.name = "Switch to light mode" + +[[project.theme.palette]] +media = "(prefers-color-scheme: light)" scheme = "default" toggle.icon = "lucide/sun" toggle.name = "Switch to dark mode" [[project.theme.palette]] +media = "(prefers-color-scheme: dark)" scheme = "slate" toggle.icon = "lucide/moon" -toggle.name = "Switch to light mode" +toggle.name = "Switch to system preference" # ---------------------------------------------------------------------------- # The "extra" section contains miscellaneous settings.