Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
504 changes: 504 additions & 0 deletions tests/data/builders/cells.py

Large diffs are not rendered by default.

18,549 changes: 18,549 additions & 0 deletions tests/data/synthetic-cells/answer-key.json

Large diffs are not rendered by default.

Binary file not shown.
Binary file added tests/data/synthetic-cells/cells-1900.xls
Binary file not shown.
Binary file added tests/data/synthetic-cells/cells-1900.xlsx
Binary file not shown.
Binary file added tests/data/synthetic-cells/cells-1904.xls
Binary file not shown.
Binary file added tests/data/synthetic-cells/cells-1904.xlsx
Binary file not shown.
Binary file added tests/data/synthetic-cells/overflow.xls
Binary file not shown.
Binary file added tests/data/synthetic-cells/overflow.xlsx
Binary file not shown.
138 changes: 138 additions & 0 deletions tests/test_cell_rule.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
# SPDX-License-Identifier: BSD-3-Clause
"""How a cell reads is whyqd's rule, not its reader's: every reader passes by giving what the rule gives.

The rule is `CellType.describe` and `ExcelParser`'s docstring. `tests/data/builders/cells.py` applies it to what each
workbook holds, with neither whyqd nor the reader in its path, so a reader's upgrade or replacement meets the rule or
fails here. A re-save that changes a cell's type or text changes that sheet's `fp0`, as it should.
"""

import json
from pathlib import Path
import shutil

import pytest

import whyqd as qd
from whyqd.dtypes import MimeType
from whyqd.models import DialectModel
from whyqd.parsers.excel_reader import ExcelParser

CELLS = Path(__file__).resolve().parent / "data" / "synthetic-cells"
KEY = json.loads((CELLS / "answer-key.json").read_text(encoding="utf-8"))
SHEETS = [(name, sheet) for name, entry in KEY.items() for sheet in entry["sheets"]]


def faults(name: str, sheet: str) -> list[str]:
"""The cells below 1 under a date's code that an `.xls` cannot read back, having no XML; none in an `.xlsx`."""
if not name.endswith(".xls"):
return []
return [ref for refs in KEY[name]["sheets"][sheet]["cases"].values() for ref in refs]


def column(index: int) -> str:
letters, index = "", index + 1
while index:
index, rest = divmod(index - 1, 26)
letters = chr(65 + rest) + letters
return letters


def read(name: str, sheet: str) -> dict[str, list[str]]:
mime = MimeType.XLS if name.endswith(".xls") else MimeType.XLSX
grid, types, _, _ = ExcelParser().grid(source=CELLS / name, mime=mime, dialect=DialectModel(sheet=sheet))
return {
f"{column(j)}{i + 1}": [tag, text]
for j, c in enumerate(grid.columns)
for i, (text, tag) in enumerate(zip(grid[c].to_list(), types[c].to_list(), strict=True))
if text is not None
}


FAULTS = [
pytest.param(
name,
sheet,
marks=pytest.mark.xfail(strict=True, reason="the reader collapses values below 1, and an .xls has no XML"),
id=f"{name}-{sheet}",
)
for name, sheet in SHEETS
if faults(name, sheet)
]


class TestRule:
@pytest.mark.parametrize(("name", "sheet"), SHEETS)
def test_every_cell_reads_as_the_rule_gives(self, name, sheet):
wanted, found = KEY[name]["sheets"][sheet]["cells"], read(name, sheet)
leave = set(faults(name, sheet))
assert {r: c for r, c in found.items() if r not in leave} == {r: c for r, c in wanted.items() if r not in leave}

@pytest.mark.parametrize(("name", "sheet"), FAULTS)
def test_a_value_below_one_in_an_xls_keeps_its_value(self, name, sheet):
wanted, found = KEY[name]["sheets"][sheet]["cells"], read(name, sheet)
assert {r: found.get(r) for r in faults(name, sheet)} == {r: wanted[r] for r in faults(name, sheet)}

def test_the_date_systems_differ_by_their_days(self):
one, four = (KEY[f"cells-{system}.xlsx"]["sheets"]["Dates"]["cells"] for system in (1900, 1904))
row = next(r for r in range(1, 100) if one[f"A{r}"] == ["d", "2024-04-01"])
assert four[f"A{row}"] == ["d", "2028-04-02"]

def test_the_day_that_never_was_reads_as_the_day_before(self):
one = KEY["cells-1900.xlsx"]["sheets"]["Dates"]["cells"]
row = next(r for r in range(1, 100) if one[f"A{r}"] == ["d", "2024-04-01"])
assert one[f"F{row}"] == one[f"G{row}"] == ["d", "1900-02-28"]


class TestResave:
def test_a_resave_that_changes_cells_fails_where_it_changes_them(self, tmp_path):
source = tmp_path / "cells.xlsx"
shutil.copy(CELLS / "cells-1900.xlsx", source)
datasource = qd.DataSourceDefinition()
datasource.derive_model(source=source, mimetype="XLSX")
models = datasource.get
shutil.copy(CELLS / "cells-1900-resaved.xlsx", source)
before, after = (KEY[n]["sheets"] for n in ("cells-1900.xlsx", "cells-1900-resaved.xlsx"))
changed = {s for s in before if before[s]["cells"] != after[s]["cells"]}
assert changed
for model in models:
definition = qd.DataSourceDefinition(source=model)
if model.dialect.sheet in changed:
with pytest.raises(AssertionError, match="source-changed"):
definition.validate()
else:
with pytest.warns(UserWarning, match="source-saved-again"):
assert definition.validate()


class TestBelowOne:
def test_a_sheet_with_no_time_does_not_open_its_xml(self, monkeypatch):
def refuse(*args, **kwargs):
raise AssertionError("The sheet's XML was opened.")

monkeypatch.setattr("whyqd.parsers.excel_reader.zipfile.ZipFile", refuse)
assert read("cells-1900.xlsx", "Cells")

def test_a_tiny_value_rounding_down_stays_a_time(self):
found = read("cells-1900.xlsx", "Dates")
row = next(r for r in range(1, 100) if found[f"A{r}"] == ["d", "2024-04-01"])
assert found[f"M{row}"] == ["d", "00:00:00"]


class TestOverflow:
@pytest.mark.parametrize("suffix", [("xlsx", MimeType.XLSX), ("xls", MimeType.XLS)], ids=["xlsx", "xls"])
@pytest.mark.parametrize(("sheet", "row"), [("Lengths", 3), ("Late", 5)])
def test_a_duration_too_large_is_refused_by_sheet_and_row(self, suffix, sheet, row):
name = f"overflow.{suffix[0]}"
with pytest.raises(ValueError, match=rf"overflow\.{suffix[0]}.*{sheet}.*row {row} .*\[whyqd:duration-range\]"):
ExcelParser().grid(source=CELLS / name, mime=suffix[1], dialect=DialectModel(sheet=sheet))

def test_a_sheet_beside_it_reads(self):
grid, _, _, _ = ExcelParser().grid(
source=CELLS / "overflow.xlsx", mime=MimeType.XLSX, dialect=DialectModel(sheet="Plain")
)
assert grid.to_dict(as_series=False) == {"column_0": ["1"]}

def test_it_blocks_deriving_the_workbook(self):
datasource = qd.DataSourceDefinition()
with pytest.raises(ValueError, match="duration-range"):
datasource.derive_model(source=CELLS / "overflow.xlsx", mimetype="XLSX")
116 changes: 101 additions & 15 deletions tests/test_fingerprint.py
Original file line number Diff line number Diff line change
@@ -1,20 +1,23 @@
# SPDX-License-Identifier: BSD-3-Clause
"""Two proofs of a source: its byte hash, which any byte changes, and `fp0`, which only a cell changes.
"""Two proofs of a source: its file's byte hash, which any byte changes, and `fp0`, which only its data changes.

`fp0` hashes the file's grid as parsed, every sheet in workbook order, each cell its stored type and whyqd's text,
before any row is skipped, joined, or taken as the header, and before any cell is trimmed: it proves the source, not
the selection. A file saved again without a cell changed keeps its `fp0`, so it verifies with a warning. A PDF has the
byte hash only, since its grid depends on how it is read.
`fp0` hashes one sheet's grid as parsed, or a file's without sheets, each cell its type and text, before any row is
skipped, joined, or taken as the header, and before any cell is trimmed: it proves the source, not the selection. Its
bytes are written out here, so its definition is held apart from any reader. A file saved again with a sheet's data
unchanged verifies that sheet with a warning. A PDF has the byte hash only, since its grid depends on how it is read.
"""

import hashlib
from pathlib import Path
import warnings

import openpyxl
import polars as pl
import pytest
import rfc8785

import whyqd as qd
from whyqd.parsers import CoreParser
from whyqd.parsers import CoreParser, FingerprintParser

DATA = Path(__file__).resolve().parent / "data"
SYNTHETIC = DATA / "synthetic"
Expand Down Expand Up @@ -79,18 +82,25 @@ def test_a_trimmed_cell_reads_trimmed(self, tmp_path):
df = datasource.get_data()
assert df["Name"].tolist() == ["a"]

def test_every_sheet_of_a_workbook_shares_one_fp0(self, tmp_path):
def test_each_sheet_has_its_own_fp0(self, tmp_path):
source = workbook(tmp_path / "two.xlsx", {"A1": "Name", "A2": "a"}, sheets=("One", "Two"))
models = derive(source, "XLSX").get
assert len(models) == 2
assert models[0].get_checksum("fp0") and models[0].get_checksum("fp0") == models[1].get_checksum("fp0")
wb = openpyxl.load_workbook(source)
wb["Two"]["A2"] = "b"
wb.save(source)
one, two = derive(source, "XLSX").get
assert one.get_checksum("fp0") != two.get_checksum("fp0")

def test_a_sheets_fp0_is_its_alone(self, tmp_path):
alone = derive(workbook(tmp_path / "one.xlsx", {"A1": "Name", "A2": "a"}), "XLSX").get
among = derive(workbook(tmp_path / "two.xlsx", {"A1": "Name", "A2": "a"}, sheets=("Data", "More")), "XLSX").get
assert alone.get_checksum("fp0") == among[0].get_checksum("fp0")

def test_every_sheet_is_in_fp0(self, tmp_path):
one = derive(workbook(tmp_path / "one.xlsx", {"A1": "Name", "A2": "a"}), "XLSX").get
two = derive(workbook(tmp_path / "two.xlsx", {"A1": "Name", "A2": "a"}, sheets=("Data", "More")), "XLSX").get
assert one.get_checksum("fp0") != two[0].get_checksum("fp0")
def test_a_sheets_name_is_not_in_fp0(self, tmp_path):
one = derive(workbook(tmp_path / "one.xlsx", {"A1": "Name"}, sheets=("One",)), "XLSX").get
two = derive(workbook(tmp_path / "two.xlsx", {"A1": "Name"}, sheets=("Two",)), "XLSX").get
assert one.get_checksum("fp0") == two.get_checksum("fp0")

def test_a_cells_stored_type_is_in_fp0(self, tmp_path):
def test_a_cells_type_is_in_fp0(self, tmp_path):
number = derive(workbook(tmp_path / "number.xlsx", {"A1": "Code", "A2": 12}), "XLSX").get
text = derive(workbook(tmp_path / "text.xlsx", {"A1": "Code", "A2": "12"}), "XLSX").get
assert number.get_checksum("fp0") != text.get_checksum("fp0")
Expand All @@ -102,7 +112,83 @@ def test_a_pdf_has_the_byte_hash_only(self):
assert model.get_checksum("fp0") is None


class TestForm:
"""The bytes written by hand and hashed with `hashlib` alone, so a change to the form, or to any reader, fails."""

PLAIN = b'fp0\n[["s","Code"],["s","Name"]]\n[["s","12"],["s","a"]]'
PLAIN_DIGEST = (
"a4274d48092cbecf450d8fbd77d01e8ba74188232e35bacabf6bb5a02e4cc08d"
"fb405a716fdbe336550046ff82cd3d61769cd1386fa1117ab598310030876f13"
)
AWKWARD_TEXT = 'a"\\\n
\U0001f600\x01\x7fé'
AWKWARD = 'fp0\n[["s","Code"],null]\n[["n","12"],["s","a\\"\\\\\\n
\U0001f600\\u0001\x7fé"]]\n[null,null]'
AWKWARD_DIGEST = (
"b78278d120d0adf816b251b02f83f6cf3c12b9c5c9fc0c17e4c20983c276f6c0"
"e4508756e15958b74e01b036d32e606cc2f757c2308a488704b0704eff38984f"
)

def awkward(self) -> tuple[pl.DataFrame, pl.DataFrame]:
grid = pl.DataFrame({"column_0": ["Code", "12", None], "column_1": [None, self.AWKWARD_TEXT, None]})
types = pl.DataFrame({"column_0": ["s", "n", None], "column_1": [None, "s", None]})
return grid, types

def test_the_digests_are_of_the_bytes_written_here(self):
assert hashlib.blake2b(self.PLAIN).hexdigest() == self.PLAIN_DIGEST
assert hashlib.blake2b(self.AWKWARD.encode("utf-8")).hexdigest() == self.AWKWARD_DIGEST

def test_fp0_hashes_those_bytes(self):
grid, types = self.awkward()
assert FingerprintParser().fp0(grid=grid, types=types) == self.AWKWARD_DIGEST

def test_those_bytes_are_rfc_8785(self):
grid, types = self.awkward()
lines = [
rfc8785.dumps([None if text is None else [tag, text] for text, tag in zip(texts, tags, strict=True)])
for texts, tags in zip(grid.iter_rows(), types.iter_rows(), strict=True)
]
assert b"\n".join([b"fp0", *lines]) == self.AWKWARD.encode("utf-8")

def test_a_csv_reaches_the_same_digest(self, tmp_path):
source = tmp_path / "plain.csv"
source.write_text("Code,Name\n12,a\n", encoding="utf-8")
assert derive(source, "CSV").get.get_checksum("fp0") == self.PLAIN_DIGEST

def test_a_workbook_reaches_the_same_digest(self, tmp_path):
source = workbook(tmp_path / "plain.xlsx", {"A1": "Code", "B1": "Name", "A2": "12", "B2": "a"})
assert derive(source, "XLSX").get.get_checksum("fp0") == self.PLAIN_DIGEST


class TestVerify:
def test_another_sheet_changing_leaves_a_sheet_saved_again(self, tmp_path):
source = workbook(tmp_path / "two.xlsx", {"A1": "Name", "A2": "a"}, sheets=("One", "Two"))
one, two = (qd.DataSourceDefinition(source=m) for m in derive(source, "XLSX").get)
wb = openpyxl.load_workbook(source)
wb["Two"]["A2"] = "b"
wb.save(source)
with pytest.warns(UserWarning, match="saved again"):
assert one.validate()
with pytest.raises(AssertionError, match="Two.*cells"):
two.validate()

def test_a_renamed_sheet_is_refused_by_name(self, tmp_path):
source = workbook(tmp_path / "data.xlsx", {"A1": "Name"}, sheets=("One", "Two"))
one = qd.DataSourceDefinition(source=derive(source, "XLSX").get[0])
wb = openpyxl.load_workbook(source)
wb["One"].title = "First"
wb.save(source)
with pytest.raises(ValueError, match="sheet-unknown"):
one.validate()

def test_a_dialect_pointed_at_the_new_name_verifies(self, tmp_path):
source = workbook(tmp_path / "data.xlsx", {"A1": "Name"}, sheets=("One", "Two"))
model = derive(source, "XLSX").get[0]
wb = openpyxl.load_workbook(source)
wb["One"].title = "First"
wb.save(source)
model.dialect = model.dialect.model_copy(update={"sheet": "First"})
with pytest.warns(UserWarning, match="saved again"):
assert qd.DataSourceDefinition(source=model).validate()

def test_a_file_saved_again_verifies_by_fp0_with_a_warning(self, tmp_path):
source = workbook(tmp_path / "data.xlsx", {"A1": "Name", "A2": "a"})
datasource = derive(source, "XLSX")
Expand Down
2 changes: 1 addition & 1 deletion tests/test_profile.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: BSD-3-Clause
"""What each column holds, as read, in counts: its cells by stored type, blank and whitespace-only cells, the
"""What each column holds, as read, in counts: its cells by type, blank and whitespace-only cells, the
anomalies, and, for an `.xlsx`, its number and date cells by their format code. Profiling never changes a cell.

A column's profile counts the table's rows, below its header, each cell as stored, before any trim. Excel's date cells
Expand Down
5 changes: 3 additions & 2 deletions whyqd/core/datasource.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,8 +141,9 @@ def validate(self) -> bool:
"""Verify the source data against the data source's checksum.

A checksum of the table as 1.x read it (`table-1x`) verifies through 1.x's own read, which needs the
`verify-1x` extra. Once it verifies, the file's `fp0` and byte hash are added beside it. A data source without
one verifies by its byte hash, else by `fp0`, warning that the file was saved again with no cell changed.
`verify-1x` extra. Once it verifies, the `fp0` of what its dialect reads and the file's byte hash are added
beside it. A data source without one verifies by its byte hash, else by `fp0`, warning that the file was saved
again with no cell changed.

Raises:
ImportError: If a 1.x checksum needs verifying and the `verify-1x` extra is not installed.
Expand Down
14 changes: 9 additions & 5 deletions whyqd/dtypes/cell.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,15 @@ class CellType(str, Enum):
@property
def describe(self):
description = {
"s": "Text, as stored. Text that is only whitespace stays text; empty text is an empty cell.",
"n": "A number, as its shortest exact text; a whole number without `.0`.",
"d": "A date as ISO 8601, with its time only when that is not midnight; a time alone, or a duration, as "
"ISO 8601 (`PT25H30M`).",
"s": "Text, as the file holds it. A workbook's text loses the XML whitespace at its ends (space, tab, line "
"feed, carriage return) unless marked to keep it (`xml:space`); empty text is an empty cell.",
"n": "A number, as an IEEE 754 double. A whole one, its exact integer digits; any other, the shortest "
"digits that return the same double, in fixed notation from `1e-4` up, and below that `d.ddde-XX`.",
"d": "A date as ISO 8601, with its time only when that is not midnight; a time alone; or a duration as "
"ISO 8601 (`PT25H30M`). A date's or time's fraction of a second has six digits; a duration's, no "
"trailing zeros.",
"b": "A boolean, as `TRUE` or `FALSE`.",
"e": "An error cell, such as `#N/A`, as its reader reports it.",
"e": "An error cell, as its code from Excel's list: `#DIV/0!`, `#N/A`, `#NAME?`, `#NULL!`, `#NUM!`, "
"`#REF!`, or `#VALUE!`.",
}
return description[self.value]
2 changes: 1 addition & 1 deletion whyqd/parsers/cell.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ class CellParser:

Readers render the same stored cell differently: a whole number as `1`, `1.0`, or `1.00`; a boolean as `True`,
`true`, or `1`; a date with or without its midnight. `render` gives whyqd's own text, by the rules `CellType`
describes, so a source reads the same through any reader.
describes, which name no reader or language, so a source reads the same through any reader.
"""

def render(
Expand Down
Loading
Loading