Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/API/errors.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
icon: lucide/triangle-alert
---

::: dstrack.errors
145 changes: 87 additions & 58 deletions docs/getting_started.md

Large diffs are not rendered by default.

20 changes: 13 additions & 7 deletions src/dstrack/diff/_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,13 @@
from enum import StrEnum
from typing import Any, Final

# Bumped only when a key already emitted changes meaning or disappears. Adding
# a key is not a version change: consumers are expected to ignore what they do
# not recognize.
DIFF_FORMAT_VERSION: Final = "1"
"""Version of the serialized diff format.

Bumped only when a key already emitted changes meaning or disappears. Adding a
key is not a version change: consumers are expected to ignore what they do not
recognize.
"""
# Relative thresholds, for counts and other unbounded magnitudes.
DEFAULT_REL_TOLERANCE: Final = 1e-3
DEFAULT_ABS_TOLERANCE: Final = 1e-12
Expand Down Expand Up @@ -385,11 +388,8 @@ class DiffOptions:


DEFAULT_OPTIONS: Final = DiffOptions()
"""The thresholds used when no options are given; same as the `normal` preset."""

# Named sensitivities, so the CLI can offer a choice without exposing twelve
# float flags. `low` is for datasets that move constantly and where only a
# structural break matters; `high` is for a curated table that should not move
# at all.
PRESETS: Final[dict[str, DiffOptions]] = {
"low": DiffOptions(
warn_pct_change=0.25,
Expand Down Expand Up @@ -417,6 +417,12 @@ class DiffOptions:
critical_percentile_shift=0.10,
),
}
"""Named sensitivities, so the CLI can offer a choice without exposing twelve
float flags.

`low` is for datasets that move constantly and where only a structural break
matters; `high` is for a curated table that should not move at all.
"""


def preset(name: str) -> DiffOptions:
Expand Down
1 change: 1 addition & 0 deletions src/dstrack/paths.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from dstrack.utils import get_invocation_path

STORE_DIRNAME: Final = ".dstrack"
"""Name of the directory that holds the local store."""
ROOT_PATH_ENV_VAR: Final = "DSTRACK_ROOT_PATH"


Expand Down
2 changes: 1 addition & 1 deletion src/dstrack/readers/_protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
from pathlib import Path
from typing import Protocol, TypeAlias, runtime_checkable

# A single cell value after dtype coercion; None represents a missing value.
Cell: TypeAlias = int | float | str | bool | None
"""A single cell value after dtype coercion; `None` represents a missing value."""


@dataclass(frozen=True)
Expand Down
7 changes: 7 additions & 0 deletions src/dstrack/snapshot/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,17 @@
------------
[SnapshotMetadata][dstrack.snapshot._metadata.SnapshotMetadata],
[DatasetStats][dstrack.snapshot._stats.DatasetStats]

Constants
---------
[DEFAULT_MAX_ROWS][dstrack.snapshot.DEFAULT_MAX_ROWS]
Default row limit of a [StatsComputer][dstrack.snapshot._stats.StatsComputer].
"""

from ._builder import SnapshotBuilder, build_snapshot_dict
from ._metadata import MetadataBuilder, SnapshotMetadata
from ._stats import (
DEFAULT_MAX_ROWS,
DatasetStats,
DatetimeColumnStats,
HistogramStats,
Expand All @@ -31,6 +37,7 @@
)

__all__ = [
"DEFAULT_MAX_ROWS",
"DatasetStats",
"DatetimeColumnStats",
"HistogramStats",
Expand Down
17 changes: 10 additions & 7 deletions src/dstrack/snapshot/_stats.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import math
from collections import Counter
from dataclasses import dataclass, field
from typing import Protocol
from typing import Final, Protocol

from dstrack.errors import InputTooLargeError
from dstrack.readers import Cell, TabularReader
Expand All @@ -11,10 +11,13 @@
)
_NUM_HISTOGRAM_BINS: int = 20
_TOP_VALUES_K: int = 50
# Default ceiling on the number of rows a single snapshot pass will accumulate.
# The pass keeps per-column values, the distinct-row set, and per-string counts
# in memory, so worst-case memory grows with the row count; this bounds it.
_DEFAULT_MAX_ROWS: int = 10_000_000

DEFAULT_MAX_ROWS: Final = 10_000_000
"""Default ceiling on the number of rows a single snapshot pass will accumulate.

The pass keeps per-column values, the distinct-row set, and per-string counts in
memory, so worst-case memory grows with the row count; this bounds it.
"""


@dataclass(frozen=True, slots=True)
Expand Down Expand Up @@ -255,11 +258,11 @@ class StatsComputer:
yields more than this, an
[InputTooLargeError][dstrack.errors.InputTooLargeError] is raised.
Defaults to
[_DEFAULT_MAX_ROWS][dstrack.snapshot._stats._DEFAULT_MAX_ROWS].
[DEFAULT_MAX_ROWS][dstrack.snapshot.DEFAULT_MAX_ROWS].
Pass ``None`` to disable the limit when enough memory is available.
"""

def __init__(self, *, max_rows: int | None = _DEFAULT_MAX_ROWS) -> None:
def __init__(self, *, max_rows: int | None = DEFAULT_MAX_ROWS) -> None:
self._max_rows = max_rows

def compute(self, reader: TabularReader) -> DatasetStats:
Expand Down
14 changes: 13 additions & 1 deletion zensical.toml
Original file line number Diff line number Diff line change
Expand Up @@ -295,15 +295,27 @@ repo = "fontawesome/brands/github"
# Read more:
# - https://zensical.org/docs/setup/colors/
# ----------------------------------------------------------------------------
#
# The toggle cycles through the entries in order. On a first visit, the first
# entry whose "media" query matches is used, so visitors start by following
# their system preference. The light/dark "media" values must stay exactly as
# written: the "(prefers-color-scheme)" entry resolves to them.
[[project.theme.palette]]
media = "(prefers-color-scheme)"
toggle.icon = "lucide/sun-moon"
toggle.name = "Switch to light mode"

[[project.theme.palette]]
media = "(prefers-color-scheme: light)"
scheme = "default"
toggle.icon = "lucide/sun"
toggle.name = "Switch to dark mode"

[[project.theme.palette]]
media = "(prefers-color-scheme: dark)"
scheme = "slate"
toggle.icon = "lucide/moon"
toggle.name = "Switch to light mode"
toggle.name = "Switch to system preference"

# ----------------------------------------------------------------------------
# The "extra" section contains miscellaneous settings.
Expand Down
Loading