From 5233f42e63b1d07500d641f8af72ef3f15fcb78f Mon Sep 17 00:00:00 2001 From: Gavin Chait Date: Mon, 2 Nov 2026 17:30:12 +0100 Subject: [PATCH 1/7] Correct the fingerprint and pandas docstrings --- whyqd/core/datasource.py | 2 +- whyqd/crud/action.py | 4 ++-- whyqd/parsers/fingerprint.py | 7 ++++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/whyqd/core/datasource.py b/whyqd/core/datasource.py index b22020e..140d6a5 100644 --- a/whyqd/core/datasource.py +++ b/whyqd/core/datasource.py @@ -87,7 +87,7 @@ def set(self, *, schema: Path | str | DataSourceModel | None = None) -> None: self.model = self.core.create_or_update_model(modelType=DataSourceModel, source=schema, model=self.model) def get_data(self, *, refresh: bool = False) -> pd.DataFrame | None: - """Get a Pandas (Modin) dataframe. + """Get the data source's table as a pandas dataframe. Parameters: refresh: Force an update of the dataframe if there have been attribute changes. diff --git a/whyqd/crud/action.py b/whyqd/crud/action.py index 7aa2264..78298fd 100644 --- a/whyqd/crud/action.py +++ b/whyqd/crud/action.py @@ -197,7 +197,7 @@ def transform(self, *, df: pd.DataFrame, script: str | ActionScriptModel) -> pd. """Return a transformed dataframe according to a single script. Parameters: - df: A Pandas (Modin) dataframe. + df: A pandas dataframe. script: A string term, or ActionScriptModel conforming to the script structure for a specific action. Returns: @@ -216,7 +216,7 @@ def transform_all(self, *, df: pd.DataFrame) -> pd.DataFrame: """Return a transformed dataframe after processing all scripts. Parameters: - df: A Pandas (Modin) dataframe. + df: A pandas dataframe. Returns: Transformed dataframe. diff --git a/whyqd/parsers/fingerprint.py b/whyqd/parsers/fingerprint.py index abca790..87d566b 100644 --- a/whyqd/parsers/fingerprint.py +++ b/whyqd/parsers/fingerprint.py @@ -14,9 +14,10 @@ class FingerprintParser: `fp0` proves a source's data, not its file, however the means of reading it changes, so it is defined here without reference to any library, and anyone can recompute it. Each sheet of a workbook has its own, a hidden one included, - and each other file one. It is taken before any row is skipped, joined, or taken as the header, and before any cell - is trimmed, so no setting changes it. A file saved again with its data unchanged keeps it, in any encoding; a save - that changes a cell's type or text does not, even if the file looks the same. Both are BLAKE2b-512. + and each other file one, but for a PDF, which has none, since its grid depends on how it is read. It is taken + before any row is skipped, joined, or taken as the header, and before any cell is trimmed, so no setting changes it. + A file saved again with its data unchanged keeps it, in any encoding; a save that changes a cell's type or text does + not, even if the file looks the same. Both are BLAKE2b-512. The bytes hashed, in UTF-8: `fp0`, then, each after a `\\n`, a line for each row of the grid, its trailing empty rows and columns trimmed. A line is a JSON array in the canonical form of From a2d7430052250a12b787827f171986cea968613d Mon Sep 17 00:00:00 2001 From: Gavin Chait Date: Mon, 2 Nov 2026 17:44:19 +0100 Subject: [PATCH 2/7] Define the load report and what each layout should find --- tests/data/builders/layouts.py | 59 +++--- tests/data/synthetic-layouts/answer-key.json | 207 +++++++++++++------ tests/test_diagnosis.py | 120 +++++++++++ whyqd/dtypes/__init__.py | 1 + whyqd/dtypes/report.py | 33 +++ whyqd/models/__init__.py | 1 + whyqd/models/report.py | 68 ++++++ 7 files changed, 395 insertions(+), 94 deletions(-) create mode 100644 tests/test_diagnosis.py create mode 100644 whyqd/dtypes/report.py create mode 100644 whyqd/models/report.py diff --git a/tests/data/builders/layouts.py b/tests/data/builders/layouts.py index fbce7e3..fa5d99c 100644 --- a/tests/data/builders/layouts.py +++ b/tests/data/builders/layouts.py @@ -8,9 +8,11 @@ with the defects real files carry added (dates beside text, `N/A`, whitespace). The key is the contract. For each file or sheet: `lines` (CSV), each row's fields as written; `dialect`, the settings -that read it as intended, which a wrangler declares and diagnosis should propose; `names` and `rows`, the table those -settings give, each row padded to the names; and `propose`, what diagnosis should find beyond the dialect. Workbook -cells are given as stored, a date as its ISO text, with `date_cells` naming them. Two files are finished by hand in +that read it as intended, which a wrangler declares; `names` and `rows`, the table those settings give, each row padded +to the names; `findings`, by code, what a report on the file read with no setting declared should find and where; and +`proposals`, the settings it should propose when asked, which is none unless the file cannot be read as written, its +layout being the wrangler's to interpret. Rows count as a wrangler would look for them, a line or sheet row from 1; +columns by position from 0. Workbook cells are given as stored, a date as its ISO text, with `date_cells` naming them. Two files are finished by hand in LibreOffice: layouts.xlsx saved as layouts.xls, and pdf-source.xlsx exported as pdf-source.pdf, sheet by sheet. """ @@ -70,6 +72,10 @@ def padded(rows, width): return [row + [""] * (width - len(row)) for row in rows] +def expect(findings=None, proposals=None): + return {"findings": findings or {}, "proposals": proposals or {}} + + def csv_cases(): """Each case: its file's lines, the text written (where it is not csv_text of the lines), and its key.""" width = len(HEADER) @@ -79,26 +85,26 @@ def csv_cases(): cases["title-above.csv"] = (lines, None, { "problem": "a title and two blank lines above the header", "dialect": {"skipRows": 3}, "names": HEADER, "rows": RECORDS, - "propose": {"skipRows": 3, "why": "the first line with as many fields as the data is line 3"}}) + **expect({"blank-rows": {"rows": [2, 3]}})}) lines = [HEADER, *RECORDS[:3], [], *RECORDS[3:]] cases["blank-line-inside.csv"] = (lines, None, { "problem": "a blank line inside the data, kept as a row and numbered", "dialect": {}, "names": HEADER, "rows": padded([*RECORDS[:3], [], *RECORDS[3:]], width), - "propose": {"blank_rows": [3]}}) + **expect({"blank-rows": {"rows": [5]}})}) records = [r.copy() for r in RECORDS] records[1][1] = "27-31, Meadow Road\nNewbury, RG14 7AH" lines = [HEADER, *records] cases["quoted-line-break.csv"] = (lines, None, { "problem": "a quoted cell holding a line break, one record over two lines of the file", - "dialect": {}, "names": HEADER, "rows": records}) + "dialect": {}, "names": HEADER, "rows": records, **expect()}) lines = [HEADER, *RECORDS] cases["semicolon.csv"] = (lines, csv_text(lines, delimiter=";"), { "problem": "`;` between cells, and commas inside the addresses unquoted", "dialect": {"delimiter": ";"}, "names": HEADER, "rows": RECORDS, - "propose": {"delimiter": ";"}}) + **expect(proposals={"delimiter": ";"})}) # No cell is quoted, so the addresses lose their commas; one cell opens with a quote that never closes records = [[v.replace(",", "") for v in r] for r in RECORDS] @@ -109,13 +115,13 @@ def csv_cases(): "problem": "a cell opening with a quote that never closes, which swallows the rest of the file if read as a " "quote (1.x needed `quoting=csv.QUOTE_NONE`)", "dialect": {"quoteChar": None}, "names": HEADER, "rows": records, - "propose": {"quoteChar": None, "why": "an opening quote with no close before the end of the file"}}) + **expect({"csv-quote": {"rows": [4], "setting": "quoteChar"}}, {"quoteChar": None})}) lines = [[*line, ""] for line in [HEADER, *RECORDS]] cases["trailing-delimiter.csv"] = (lines, None, { "problem": "every line ends with a delimiter, an empty last column with an empty header", "dialect": {}, "names": HEADER, "rows": RECORDS, - "propose": {"why": "the empty last column is trimmed with the grid, as fp0 trims it"}}) + **expect()}) # Each record over two lines of the file, as every line has the same fields: the second holds one value first = HEADER[:5] @@ -125,7 +131,7 @@ def csv_cases(): cases["two-line-records.csv"] = (lines, None, { "problem": "each record, header included, over two lines: five fields, then one in the first field", "dialect": {"rowsPerRecord": [5, 1]}, "names": HEADER, "rows": RECORDS, - "propose": {"rowsPerRecord": [5, 1], "why": "every second line has one value, in its first field"}, + **expect({"blank-rows": {"rows": [10]}}, {"rowsPerRecord": [5, 1]}), "refuse": "a record without its second line, or a second line with values past its first field"}) # The same records with nothing after a record's one value: a lone empty value is a blank line @@ -135,7 +141,7 @@ def csv_cases(): cases["two-line-records-unpadded.csv"] = (lines, None, { "problem": "each record over two lines, the second holding its one value and no delimiters, so an empty " "value is a blank line", "dialect": {"rowsPerRecord": [5, 1]}, "names": HEADER, "rows": RECORDS, - "propose": {"rowsPerRecord": [5, 1], "why": "every second line has at most one value"}}) + **expect({"blank-rows": {"rows": [10]}}, {"rowsPerRecord": [5, 1]})}) records = [r.copy() for r in RECORDS] records[2][1] = 'The "Old" Mill, Station Road, Pangbourne, Reading, RG8 7AN' @@ -150,29 +156,29 @@ def escaped(value): cases["backslash-escape.csv"] = (lines, text, { "problem": "a quote inside a quoted cell escaped by a backslash, not doubled", "dialect": {"doubleQuote": False}, "names": HEADER, "rows": records, - "propose": {"doubleQuote": False, "why": "a backslash before a quote inside a quoted cell"}}) + **expect({"csv-quote": {"rows": [4], "setting": "doubleQuote"}}, {"doubleQuote": False})}) lines = [HEADER, *RECORDS] cases["cr-line-ends.csv"] = (lines, csv_text(lines, eol="\r"), { "problem": "every line ending `\\r` alone, as old Mac files do", "dialect": {}, "names": HEADER, - "rows": RECORDS}) + "rows": RECORDS, **expect()}) lines = [HEADER[:5], *RECORDS] cases["header-narrower.csv"] = (lines, None, { "problem": "the header a field narrower than the data, so the last column has no name", "dialect": {}, "names": [*HEADER[:5], "Unnamed: 5"], "rows": RECORDS, - "renamed": "the column past the header by position, as an empty name is"}) + "renamed": "the column past the header by position, as an empty name is", **expect()}) lines = [RECORDS[0][:5], *RECORDS[1:]] cases["first-row-narrower.csv"] = (lines, None, { "problem": "no header, and the first row a field narrower than the rest", "dialect": {"headerRowCount": 0}, "names": [f"column_{i}" for i in range(width)], - "rows": padded(lines, width)}) + "rows": padded(lines, width), **expect()}) lines = [HEADER, *RECORDS[:2], RECORDS[2][:4], *RECORDS[3:]] cases["row-narrower.csv"] = (lines, None, { "problem": "one row two fields narrower than the header, its last cells empty", - "dialect": {}, "names": HEADER, "rows": padded(lines[1:], width)}) + "dialect": {}, "names": HEADER, "rows": padded(lines[1:], width), **expect()}) rows = [[f"v{i}{j}" for j in range(len(HOSTILE))] for i in range(2)] lines = [HOSTILE, *rows] @@ -182,13 +188,13 @@ def escaped(value): "names": [f"Unnamed: {i}" if n == "" else n for i, n in enumerate(HOSTILE[:-3])] + ["Rate.1", "Rate Relief", "__row__.1"], "renamed": "the empty name by position, as stored schemas name it; the repeat and the row key's own name " - "by RowKeyParser.deconflict"}) + "by RowKeyParser.deconflict", **expect()}) rows = [[v, str(i)] for i, v in enumerate(MISSING_LIKE)] lines = [["Contact", "n"], *rows] cases["missing-like-text.csv"] = (lines, None, { "problem": "text pandas reads as missing by default; every one is text, and only the source schema can " - "say it means missing", "dialect": {}, "names": ["Contact", "n"], "rows": rows}) + "say it means missing", "dialect": {}, "names": ["Contact", "n"], "rows": rows, **expect()}) return cases @@ -233,7 +239,7 @@ def layouts_workbook(): dates = write_records(ws, top=4) key["Titled"] = {"problem": "a title and subtitle, then a blank row, above the header on row 4", "dialect": {"skipRows": 3}, "names": HEADER, "rows": workbook_rows(), "date_cells": dates, - "propose": {"skipRows": 3}} + **expect({"blank-rows": {"rows": [3]}})} ws = wb.create_sheet("MergedHeader") ws["A1"], ws["B1"], ws["D1"] = "Propref", "Property", "Relief" @@ -254,7 +260,7 @@ def layouts_workbook(): "names": "supplied by the wrangler, one per column: a merged cell over two may be one title or a heading " "over subheadings, so whyqd never builds a name from it", "refuse": "headerRowCount 2 without names: the header rows and merged ranges named, and names asked for", - "propose": {"headerRowCount": 2, "why": "merged ranges in rows 1 and 2, and data from row 3"}} + **expect({"merged-range": {"cells": ["A1:A2", "B1:C1", "D1:E1"]}})} ws = wb.create_sheet("BlankColumns") names = HEADER[:2] + [None] + HEADER[2:] @@ -271,7 +277,7 @@ def layouts_workbook(): "problem": "an empty column inside the table, and three empty but formatted columns after it", "dialect": {}, "names": [*HEADER[:2], "Unnamed: 2", *HEADER[2:]], "rows": [[v if v != "" else None for v in r[:2] + [None] + r[2:]] for r in RECORDS], - "propose": {"empty_columns": [2], "why": "the trailing columns hold formatting only, and are trimmed"}} + **expect({"blank-columns": {"columns": [2]}})} ws = wb.create_sheet("SubTables") write_records(ws, top=1, dates=False) @@ -285,9 +291,7 @@ def layouts_workbook(): key["SubTables"] = { "problem": "a second table below a blank row, then footnotes; loaded as given, every row is in the table", "dialect": {}, "names": HEADER, "rows": [[v if v != "" else None for v in r] for r in rows], - "propose": {"table_rows": [0, len(RECORDS) - 1], - "second_table": {"header_row": len(RECORDS) + 1, "rows": len(CODES) - 1}, - "footnotes": [len(RECORDS) + len(CODES) + 2, len(rows) - 1]}} + **expect({"blank-rows": {"rows": [len(RECORDS) + 2, len(RECORDS) + len(CODES) + 3]}})} ws = wb.create_sheet("Errors") write_records(ws, top=1, dates=False) @@ -299,7 +303,7 @@ def layouts_workbook(): for coordinate, value in errors.items(): rows[int(coordinate[1:]) - 2][3] = value key["Errors"] = {"problem": "error cells in a number column", "dialect": {}, "names": HEADER, "rows": rows, - "error_cells": errors} + "error_cells": errors, **expect({"error-cells": {"cells": list(errors)}})} ws = wb.create_sheet("Workings") ws.sheet_state = "hidden" @@ -307,7 +311,8 @@ def layouts_workbook(): key["Workings"] = {"problem": "a hidden sheet, read like any other and flagged", "state": "hidden", "dialect": {}, "names": ["Total R/V"], "rows": [], "formula_cells": {"A2": "a formula written without its cached value, so the row is empty " - "here and trimmed; the .xls, saved by LibreOffice, caches 0"}} + "here and trimmed; the .xls, saved by LibreOffice, caches 0"}, + **expect({"hidden-sheet": {}})} return wb, key @@ -321,7 +326,7 @@ def single_sheet_workbook(): for c, value in enumerate(row, start=1): ws.cell(row=r, column=c, value=value) key = {"problem": "the only sheet, with row 2 blank: 1.x's one-row probe judged it empty and dropped it", - "dialect": {}, "names": CODES[0], "rows": [[None, None], *CODES[1:]], "propose": {"blank_rows": [0]}} + "dialect": {}, "names": CODES[0], "rows": [[None, None], *CODES[1:]], **expect({"blank-rows": {"rows": [2]}})} return wb, key diff --git a/tests/data/synthetic-layouts/answer-key.json b/tests/data/synthetic-layouts/answer-key.json index 3ae000c..f673bd3 100644 --- a/tests/data/synthetic-layouts/answer-key.json +++ b/tests/data/synthetic-layouts/answer-key.json @@ -1,5 +1,5 @@ { - "generated": "2026-10-30T19:48:46", + "generated": "2026-11-02T17:36:36", "csv": { "title-above.csv": { "lines": [ @@ -127,10 +127,15 @@ " Landlord " ] ], - "propose": { - "skipRows": 3, - "why": "the first line with as many fields as the data is line 3" - } + "findings": { + "blank-rows": { + "rows": [ + 2, + 3 + ] + } + }, + "proposals": {} }, "blank-line-inside.csv": { "lines": [ @@ -260,11 +265,14 @@ " Landlord " ] ], - "propose": { - "blank_rows": [ - 3 - ] - } + "findings": { + "blank-rows": { + "rows": [ + 5 + ] + } + }, + "proposals": {} }, "quoted-line-break.csv": { "lines": [ @@ -384,7 +392,9 @@ "30/01/2026", " Landlord " ] - ] + ], + "findings": {}, + "proposals": {} }, "semicolon.csv": { "lines": [ @@ -507,7 +517,8 @@ " Landlord " ] ], - "propose": { + "findings": {}, + "proposals": { "delimiter": ";" } }, @@ -632,9 +643,16 @@ " Landlord " ] ], - "propose": { - "quoteChar": null, - "why": "an opening quote with no close before the end of the file" + "findings": { + "csv-quote": { + "rows": [ + 4 + ], + "setting": "quoteChar" + } + }, + "proposals": { + "quoteChar": null } }, "trailing-delimiter.csv": { @@ -763,9 +781,8 @@ " Landlord " ] ], - "propose": { - "why": "the empty last column is trimmed with the grid, as fp0 trims it" - } + "findings": {}, + "proposals": {} }, "two-line-records.csv": { "lines": [ @@ -933,12 +950,18 @@ " Landlord " ] ], - "propose": { + "findings": { + "blank-rows": { + "rows": [ + 10 + ] + } + }, + "proposals": { "rowsPerRecord": [ 5, 1 - ], - "why": "every second line has one value, in its first field" + ] }, "refuse": "a record without its second line, or a second line with values past its first field" }, @@ -1080,12 +1103,18 @@ " Landlord " ] ], - "propose": { + "findings": { + "blank-rows": { + "rows": [ + 10 + ] + } + }, + "proposals": { "rowsPerRecord": [ 5, 1 - ], - "why": "every second line has at most one value" + ] } }, "backslash-escape.csv": { @@ -1209,9 +1238,16 @@ " Landlord " ] ], - "propose": { - "doubleQuote": false, - "why": "a backslash before a quote inside a quoted cell" + "findings": { + "csv-quote": { + "rows": [ + 4 + ], + "setting": "doubleQuote" + } + }, + "proposals": { + "doubleQuote": false } }, "cr-line-ends.csv": { @@ -1332,7 +1368,9 @@ "30/01/2026", " Landlord " ] - ] + ], + "findings": {}, + "proposals": {} }, "header-narrower.csv": { "lines": [ @@ -1452,7 +1490,9 @@ " Landlord " ] ], - "renamed": "the column past the header by position, as an empty name is" + "renamed": "the column past the header by position, as an empty name is", + "findings": {}, + "proposals": {} }, "first-row-narrower.csv": { "lines": [ @@ -1565,7 +1605,9 @@ "30/01/2026", " Landlord " ] - ] + ], + "findings": {}, + "proposals": {} }, "row-narrower.csv": { "lines": [ @@ -1683,7 +1725,9 @@ "30/01/2026", " Landlord " ] - ] + ], + "findings": {}, + "proposals": {} }, "hostile-headers.csv": { "lines": [ @@ -1797,7 +1841,9 @@ "Rate Relief", "__row__.1" ], - "renamed": "the empty name by position, as stored schemas name it; the repeat and the row key's own name by RowKeyParser.deconflict" + "renamed": "the empty name by position, as stored schemas name it; the repeat and the row key's own name by RowKeyParser.deconflict", + "findings": {}, + "proposals": {} }, "missing-like-text.csv": { "lines": [ @@ -1877,7 +1923,9 @@ "None", "7" ] - ] + ], + "findings": {}, + "proposals": {} } }, "workbook": { @@ -1949,9 +1997,14 @@ "E5", "E7" ], - "propose": { - "skipRows": 3 - } + "findings": { + "blank-rows": { + "rows": [ + 3 + ] + } + }, + "proposals": {} }, "MergedHeader": { "problem": "a header over two rows: one name merged down both, two merged across two columns each", @@ -2025,10 +2078,16 @@ ], "names": "supplied by the wrangler, one per column: a merged cell over two may be one title or a heading over subheadings, so whyqd never builds a name from it", "refuse": "headerRowCount 2 without names: the header rows and merged ranges named, and names asked for", - "propose": { - "headerRowCount": 2, - "why": "merged ranges in rows 1 and 2, and data from row 3" - } + "findings": { + "merged-range": { + "cells": [ + "A1:A2", + "B1:C1", + "D1:E1" + ] + } + }, + "proposals": {} }, "BlankColumns": { "problem": "an empty column inside the table, and three empty but formatted columns after it", @@ -2098,12 +2157,14 @@ " Landlord " ] ], - "propose": { - "empty_columns": [ - 2 - ], - "why": "the trailing columns hold formatting only, and are trimmed" - } + "findings": { + "blank-columns": { + "columns": [ + 2 + ] + } + }, + "proposals": {} }, "SubTables": { "problem": "a second table below a blank row, then footnotes; loaded as given, every row is in the table", @@ -2230,20 +2291,15 @@ null ] ], - "propose": { - "table_rows": [ - 0, - 5 - ], - "second_table": { - "header_row": 7, - "rows": 3 - }, - "footnotes": [ - 12, - 13 - ] - } + "findings": { + "blank-rows": { + "rows": [ + 8, + 13 + ] + } + }, + "proposals": {} }, "Errors": { "problem": "error cells in a number column", @@ -2310,7 +2366,17 @@ "D3": "#N/A", "D5": "#DIV/0!", "D7": "#REF!" - } + }, + "findings": { + "error-cells": { + "cells": [ + "D3", + "D5", + "D7" + ] + } + }, + "proposals": {} }, "Workings": { "problem": "a hidden sheet, read like any other and flagged", @@ -2322,7 +2388,11 @@ "rows": [], "formula_cells": { "A2": "a formula written without its cached value, so the row is empty here and trimmed; the .xls, saved by LibreOffice, caches 0" - } + }, + "findings": { + "hidden-sheet": {} + }, + "proposals": {} } }, "layouts.xls": "layouts.xlsx saved as .xls in LibreOffice; read to the same key", @@ -2352,11 +2422,14 @@ "Mandatory Charity Relief" ] ], - "propose": { - "blank_rows": [ - 0 - ] - } + "findings": { + "blank-rows": { + "rows": [ + 2 + ] + } + }, + "proposals": {} } } }, diff --git a/tests/test_diagnosis.py b/tests/test_diagnosis.py new file mode 100644 index 0000000..c04193f --- /dev/null +++ b/tests/test_diagnosis.py @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: BSD-3-Clause +"""A report on how a file was read and what was found there, held to the layout fixtures' answer key. + +Each file is read with no setting declared, as a wrangler first loads it. A report interprets nothing: its findings are +what the file holds, and it proposes a setting only for a file that cannot be read as written, and only when asked. +""" + +import json +from pathlib import Path + +import pytest + +import whyqd as qd +from whyqd.models import FindingModel, ReportModel + +LAYOUTS = Path(__file__).resolve().parent / "data" / "synthetic-layouts" +KEY = json.loads((LAYOUTS / "answer-key.json").read_text(encoding="utf-8")) +CSV = [(name, "CSV", {}, case) for name, case in KEY["csv"].items()] +WORKBOOK = [ + (name, "XLSX", {"sheet": sheet}, case) + for name, sheets in KEY["workbook"].items() + if name.endswith(".xlsx") + for sheet, case in sheets.items() +] +CASES = CSV + WORKBOOK +IDS = [f"{name}-{dialect.get('sheet', '')}" for name, _, dialect, _ in CASES] +UNBUILT = pytest.mark.xfail(strict=True, reason="diagnose is not built") + + +def diagnose(name: str, mimetype: str, dialect: dict, propose: bool = False) -> ReportModel: + reports = qd.DataSourceDefinition().diagnose( + source=LAYOUTS / name, mimetype=mimetype, dialect=dialect or None, propose=propose + ) + assert len(reports) == 1 + return reports[0] + + +def found(report: ReportModel) -> dict[str, dict]: + return { + f.code: {k: getattr(f, k) for k in ("rows", "columns", "cells", "setting") if getattr(f, k)} + for f in report.findings + } + + +@UNBUILT +@pytest.mark.parametrize("name,mimetype,dialect,case", CASES, ids=IDS) +def test_each_layout_finds_what_its_key_says(name, mimetype, dialect, case): + assert found(diagnose(name, mimetype, dialect)) == case["findings"] + + +@UNBUILT +@pytest.mark.parametrize("name,mimetype,dialect,case", CASES, ids=IDS) +def test_each_layout_proposes_what_its_key_says(name, mimetype, dialect, case): + report = diagnose(name, mimetype, dialect, propose=True) + assert {p.setting: p.value for p in report.proposals} == case["proposals"] + assert all(p.evidence for p in report.proposals) + + +@UNBUILT +@pytest.mark.parametrize("name,mimetype,dialect,case", CASES, ids=IDS) +def test_nothing_is_proposed_unasked(name, mimetype, dialect, case): + assert diagnose(name, mimetype, dialect).proposals == [] + + +@UNBUILT +@pytest.mark.parametrize("name,mimetype,dialect,case", CASES, ids=IDS) +def test_two_runs_give_the_same_report(name, mimetype, dialect, case): + assert diagnose(name, mimetype, dialect, propose=True) == diagnose(name, mimetype, dialect, propose=True) + + +@UNBUILT +def test_diagnosing_changes_nothing_about_the_read(): + def derived() -> qd.models.DataSourceModel: + datasource = qd.DataSourceDefinition() + datasource.derive_model(source=LAYOUTS / "title-above.csv", mimetype="CSV") + return datasource.get + + before = derived() + definition = qd.DataSourceDefinition() + definition.diagnose(source=LAYOUTS / "title-above.csv", mimetype="CSV", propose=True) + assert definition.get is None + after = derived() + assert after.dialect == before.dialect + assert after.checksums == before.checksums + + +@UNBUILT +def test_a_declared_setting_is_reported_as_declared(): + report = diagnose("title-above.csv", "CSV", {"skipRows": 3}) + sources = {s.name: s.source for s in report.settings} + assert sources["skipRows"] == "declared" + assert sources["delimiter"] == "default" + assert sources["encoding"] == "file" + + +@UNBUILT +def test_a_file_that_will_not_read_still_gives_a_report(): + report = diagnose("stray-quote.csv", "CSV", {}) + assert report.findings[0].kind == "fault" + assert report.findings[0].code == "csv-quote" + assert report.settings == [] + + +@UNBUILT +def test_each_sheet_of_a_workbook_has_its_own_report(): + reports = qd.DataSourceDefinition().diagnose(source=LAYOUTS / "layouts.xlsx", mimetype="XLSX") + assert [r.sheet for r in reports] == list(KEY["workbook"]["layouts.xlsx"]) + + +@UNBUILT +def test_the_rows_above_a_header_can_be_seen_in_the_preview(): + report = diagnose("title-above.csv", "CSV", {"skipRows": 3}) + assert report.place == "line" + assert "Business rates reliefs, April 2026" in report.preview + + +def test_a_finding_prints_with_its_code_and_stores_it_apart(): + finding = FindingModel(code="blank-rows", kind="fact", message="Line 5 is blank.", rows=[5]) + assert finding.message == "Line 5 is blank." + assert str(finding) == "Line 5 is blank. [whyqd:blank-rows]" diff --git a/whyqd/dtypes/__init__.py b/whyqd/dtypes/__init__.py index 3d80229..18c152f 100644 --- a/whyqd/dtypes/__init__.py +++ b/whyqd/dtypes/__init__.py @@ -6,3 +6,4 @@ from .checksum import ChecksumContent from .encoding import EncodingStep from .extension import ExtensionKind +from .report import FindingKind, SettingSource diff --git a/whyqd/dtypes/report.py b/whyqd/dtypes/report.py new file mode 100644 index 0000000..9135258 --- /dev/null +++ b/whyqd/dtypes/report.py @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: BSD-3-Clause +from enum import Enum + + +class FindingKind(str, Enum): + FAULT = "fault" + FACT = "fact" + + @property + def describe(self): + description = { + "fault": "The file could not be read as written, or was read with a loss. Its owner may correct it, or a " + "setting may read past it.", + "fact": "Something the file holds, with no claim of what it means. What to do about it, if anything, is " + "for the crosswalk.", + } + return description[self.value] + + +class SettingSource(str, Enum): + DECLARED = "declared" + DEFAULT = "default" + FILE = "file" + + @property + def describe(self): + description = { + "declared": "Declared by the wrangler.", + "default": "Not declared, so its fixed default.", + "file": "Settled by what the file shows, by a rule that never depends on anything else: an encoding's " + "steps, for one.", + } + return description[self.value] diff --git a/whyqd/models/__init__.py b/whyqd/models/__init__.py index ea1b78a..a6bd410 100644 --- a/whyqd/models/__init__.py +++ b/whyqd/models/__init__.py @@ -7,6 +7,7 @@ from .category import CategoryModel from .dialect import DialectModel from .profile import ProfileModel +from .report import FindingModel, ProposalModel, ReportModel, SettingModel from .policy import NumberFormatModel, PolicyModel from .column import ColumnModel from .actionscript import ActionScriptModel diff --git a/whyqd/models/report.py b/whyqd/models/report.py new file mode 100644 index 0000000..05a3416 --- /dev/null +++ b/whyqd/models/report.py @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: BSD-3-Clause +from __future__ import annotations +from typing import Optional, Union +from pydantic import BaseModel, ConfigDict, Field + +from whyqd.dtypes import FindingKind, MimeType, SettingSource + +SettingValue = Union[bool, int, str, list[int], None] + + +class SettingModel(BaseModel): + """One reading setting, as a read used it.""" + + name: str = Field(..., description="The setting's name in the dialect.") + value: SettingValue = Field(None, description="The value the read used.") + source: SettingSource = Field(..., description="Where the value came from.") + model_config = ConfigDict(use_enum_values=True, validate_assignment=True) + + +class FindingModel(BaseModel): + """Something found in a file, shaped as a compiler's diagnostic is: a stable code with a page of its own, what was + found, where, and, for a fault, what addresses it. A code is shared with the refusal or warning for the same thing. + """ + + code: str = Field(..., description="Short, stable, and kebab-case, naming the finding's documentation page.") + kind: FindingKind = Field(..., description="A fault in the file, or a plain fact about it.") + message: str = Field(..., description="What was found and where, without its code.") + rows: list[int] = Field([], description="The rows concerned: a text file's lines or a sheet's rows, from 1.") + columns: list[int] = Field([], description="The columns concerned, by position in the grid, from 0.") + cells: list[str] = Field([], description="The cells or ranges concerned, as a workbook names them: `B2:C3`.") + setting: Optional[str] = Field(None, description="For a fault, the dialect's setting that addresses it.") + model_config = ConfigDict(use_enum_values=True, validate_assignment=True) + + def __str__(self) -> str: + return f"{self.message} [whyqd:{self.code}]" + + +class ProposalModel(BaseModel): + """A setting that would read past a fault. Never applied: it takes effect only when the wrangler declares it.""" + + setting: str = Field(..., description="The setting's name in the dialect.") + value: SettingValue = Field(None, description="The value proposed.") + evidence: str = Field(..., description="What in the file supports it.") + model_config = ConfigDict(use_enum_values=True, validate_assignment=True) + + +class ReportModel(BaseModel): + """How one file, or one sheet of a workbook, was read, and what was found there. Computed when asked, never stored + on a definition.""" + + path: str = Field(..., description="The file read.") + mime: MimeType = Field(..., description="The file's format.") + sheet: Optional[str] = Field(None, description="The sheet read, in a workbook.") + place: str = Field("line", description="What `rows` counts: a text file's `line`, a sheet's `row`.") + settings: list[SettingModel] = Field( + [], description="Each setting the read used. Empty where the file could not be read." + ) + height: int = Field(0, ge=0, description="Rows in the grid.") + width: int = Field(0, ge=0, description="Columns in the grid.") + rowWidths: dict[int, int] = Field( + {}, + description="The grid's rows by how far each reaches: its last filled cell, counted from 1, so `0` is a blank " + "row. Evidence of a file's shape, with no claim of what it means.", + ) + findings: list[FindingModel] = Field([], description="What was found, faults first.") + proposals: list[ProposalModel] = Field([], description="Settings proposed, where the wrangler asked for them.") + preview: Optional[str] = Field(None, description="A short text of the grid as read, each row numbered by `place`.") + model_config = ConfigDict(use_enum_values=True, validate_assignment=True) From 3ff28ad64be4fa2d7b6caae8fbac19a58ac7ca7d Mon Sep 17 00:00:00 2001 From: Gavin Chait Date: Mon, 2 Nov 2026 18:06:42 +0100 Subject: [PATCH 3/7] Report how a file was read and what was found there --- tests/test_diagnosis.py | 108 ++++++++++++-- whyqd/config/patterns.py | 2 + whyqd/config/settings.py | 7 + whyqd/core/datasource.py | 39 +++++- whyqd/parsers/arrow_reader.py | 11 +- whyqd/parsers/csv_reader.py | 19 ++- whyqd/parsers/datasource.py | 32 ++++- whyqd/parsers/diagnosis.py | 256 ++++++++++++++++++++++++++++++++++ whyqd/parsers/excel_reader.py | 41 +++++- whyqd/parsers/grid.py | 9 ++ whyqd/parsers/pdf_reader.py | 13 +- 11 files changed, 510 insertions(+), 27 deletions(-) create mode 100644 whyqd/parsers/diagnosis.py diff --git a/tests/test_diagnosis.py b/tests/test_diagnosis.py index c04193f..353becc 100644 --- a/tests/test_diagnosis.py +++ b/tests/test_diagnosis.py @@ -24,17 +24,28 @@ ] CASES = CSV + WORKBOOK IDS = [f"{name}-{dialect.get('sheet', '')}" for name, _, dialect, _ in CASES] -UNBUILT = pytest.mark.xfail(strict=True, reason="diagnose is not built") +PROPOSING = [ + pytest.param( + *c, marks=[pytest.mark.xfail(strict=True, reason="proposals are not built")] if c[3]["proposals"] else [] + ) + for c in CASES +] +SYNTHETIC = LAYOUTS.parent / "synthetic" +FENLAND = LAYOUTS.parent / "sample" / "fenland" / "2026-10-25-Fenland District Council.csv" -def diagnose(name: str, mimetype: str, dialect: dict, propose: bool = False) -> ReportModel: +def diagnose_source(source: Path, mimetype: str, dialect: dict, propose: bool = False) -> ReportModel: reports = qd.DataSourceDefinition().diagnose( - source=LAYOUTS / name, mimetype=mimetype, dialect=dialect or None, propose=propose + source=source, mimetype=mimetype, dialect=dialect or None, propose=propose ) assert len(reports) == 1 return reports[0] +def diagnose(name: str, mimetype: str, dialect: dict, propose: bool = False) -> ReportModel: + return diagnose_source(LAYOUTS / name, mimetype, dialect, propose) + + def found(report: ReportModel) -> dict[str, dict]: return { f.code: {k: getattr(f, k) for k in ("rows", "columns", "cells", "setting") if getattr(f, k)} @@ -42,33 +53,28 @@ def found(report: ReportModel) -> dict[str, dict]: } -@UNBUILT @pytest.mark.parametrize("name,mimetype,dialect,case", CASES, ids=IDS) def test_each_layout_finds_what_its_key_says(name, mimetype, dialect, case): assert found(diagnose(name, mimetype, dialect)) == case["findings"] -@UNBUILT -@pytest.mark.parametrize("name,mimetype,dialect,case", CASES, ids=IDS) +@pytest.mark.parametrize("name,mimetype,dialect,case", PROPOSING, ids=IDS) def test_each_layout_proposes_what_its_key_says(name, mimetype, dialect, case): report = diagnose(name, mimetype, dialect, propose=True) assert {p.setting: p.value for p in report.proposals} == case["proposals"] assert all(p.evidence for p in report.proposals) -@UNBUILT @pytest.mark.parametrize("name,mimetype,dialect,case", CASES, ids=IDS) def test_nothing_is_proposed_unasked(name, mimetype, dialect, case): assert diagnose(name, mimetype, dialect).proposals == [] -@UNBUILT @pytest.mark.parametrize("name,mimetype,dialect,case", CASES, ids=IDS) def test_two_runs_give_the_same_report(name, mimetype, dialect, case): assert diagnose(name, mimetype, dialect, propose=True) == diagnose(name, mimetype, dialect, propose=True) -@UNBUILT def test_diagnosing_changes_nothing_about_the_read(): def derived() -> qd.models.DataSourceModel: datasource = qd.DataSourceDefinition() @@ -84,7 +90,6 @@ def derived() -> qd.models.DataSourceModel: assert after.checksums == before.checksums -@UNBUILT def test_a_declared_setting_is_reported_as_declared(): report = diagnose("title-above.csv", "CSV", {"skipRows": 3}) sources = {s.name: s.source for s in report.settings} @@ -93,7 +98,6 @@ def test_a_declared_setting_is_reported_as_declared(): assert sources["encoding"] == "file" -@UNBUILT def test_a_file_that_will_not_read_still_gives_a_report(): report = diagnose("stray-quote.csv", "CSV", {}) assert report.findings[0].kind == "fault" @@ -101,17 +105,93 @@ def test_a_file_that_will_not_read_still_gives_a_report(): assert report.settings == [] -@UNBUILT +def test_a_backslash_file_is_told_to_declare_double_quote(): + with pytest.raises(ValueError, match="declare `doubleQuote` as false"): + qd.DataSourceDefinition().derive_model(source=LAYOUTS / "backslash-escape.csv", mimetype="CSV") + + +@pytest.mark.parametrize( + "name,mimetype", + [ + ("bad.xlsx", "XLSX"), + ("bad.xls", "XLS"), + ("bad.parquet", "PARQUET"), + ("bad.feather", "FEATHER"), + ("bad.pdf", "PDF"), + ], +) +def test_a_file_that_is_not_its_format_is_a_finding_in_whyqds_terms(tmp_path, name, mimetype): + source = tmp_path / name + source.write_bytes(b"not what it says") + with pytest.raises(ValueError, match=r"\[whyqd:file-unreadable\]"): + qd.DataSourceDefinition().derive_model(source=source, mimetype=mimetype) + reports = qd.DataSourceDefinition().diagnose(source=source, mimetype=mimetype) + assert [f.code for r in reports for f in r.findings] == ["file-unreadable"] + + +def test_an_unknown_sheet_is_a_finding_naming_the_setting(): + assert found(diagnose("layouts.xlsx", "XLSX", {"sheet": "Nope"})) == {"sheet-unknown": {"setting": "sheet"}} + + +def test_a_declaration_the_file_does_not_fit_is_a_finding_beside_the_read(): + report = diagnose("title-above.csv", "CSV", {"skipRows": 50}) + assert found(report) == {"header-past-end": {"setting": "skipRows"}, "blank-rows": {"rows": [2, 3]}} + assert report.settings and report.height == 10 + + +def test_a_declaration_whyqd_cannot_read_is_refused_not_reported(): + with pytest.raises(ValueError, match=r"\[whyqd:dialect-unknown\]"): + diagnose("title-above.csv", "CSV", {"header": 3}) + + +def test_bytes_that_did_not_decode_are_a_fault_naming_the_encoding_setting(): + report = diagnose_source(SYNTHETIC / "fixture-windows-1252.csv", "CSV", {"encoding": "utf-8"}) + assert report.findings[0].code == "encoding-replaced" + assert report.findings[0].kind == "fault" + assert report.findings[0].setting == "encoding" + assert "commonest" in report.findings[0].message + + +def test_a_merged_range_reads_the_same_from_both_workbook_formats(): + cells = {"merged-range": {"cells": ["A1:A2", "B1:C1", "D1:E1"]}} + assert found(diagnose("layouts.xlsx", "XLSX", {"sheet": "MergedHeader"})) == cells + assert found(diagnose("layouts.xls", "XLS", {"sheet": "MergedHeader"})) == cells + + def test_each_sheet_of_a_workbook_has_its_own_report(): reports = qd.DataSourceDefinition().diagnose(source=LAYOUTS / "layouts.xlsx", mimetype="XLSX") assert [r.sheet for r in reports] == list(KEY["workbook"]["layouts.xlsx"]) + assert [f.code for f in reports[-1].findings] == ["hidden-sheet"] + + +def test_row_widths_show_a_files_shape_without_a_finding(): + # Fenland's records each span two lines: 47 fields, then one + report = diagnose_source(FENLAND, "CSV", {}) + assert report.rowWidths == {1: 51, 47: 51} + assert report.findings == [] -@UNBUILT def test_the_rows_above_a_header_can_be_seen_in_the_preview(): report = diagnose("title-above.csv", "CSV", {"skipRows": 3}) assert report.place == "line" - assert "Business rates reliefs, April 2026" in report.preview + assert "line 1: Business rates reliefs, April 2026" in report.preview + + +def test_the_preview_shows_the_top_and_end_of_a_long_file_with_cells_cut(tmp_path): + source = tmp_path / "long.csv" + source.write_text("".join(f"{i},{'x' * 60}\n" for i in range(100)), encoding="utf-8") + preview = diagnose_source(source, "CSV", {}).preview.splitlines() + assert preview[0].startswith("line 1: 0 | xxx") + assert preview[0].endswith("…") + assert preview[15] == "... 80 lines not shown ..." + assert preview[-1].startswith("line 100: 99 | ") + assert len(preview) == 21 + + +def test_a_columnar_file_reports_with_no_settings_of_a_text_file(): + report = diagnose_source(LAYOUTS.parent / "2023-05-09-human-development-report-interim.PARQUET", "PARQUET", {}) + assert report.place == "row" + assert "delimiter" not in {s.name for s in report.settings} def test_a_finding_prints_with_its_code_and_stores_it_apart(): diff --git a/whyqd/config/patterns.py b/whyqd/config/patterns.py index 2d37f92..81e0094 100644 --- a/whyqd/config/patterns.py +++ b/whyqd/config/patterns.py @@ -41,3 +41,5 @@ class PythonPatterns: LINE: ClassVar[re.Pattern] = re.compile(r"[^\r\n]*(?:\r\n|\n|\r)|[^\r\n]+\Z") # A part of the row key, its name given by `RowKeyParser.ROW`: `__row__`, `__row__1` ROW_PART: ClassVar[str] = r"^{row}(\d*)$" + # A refusal or warning, its text and its code: `Something was found. [whyqd:a-code]` + MESSAGE: ClassVar[re.Pattern] = re.compile(r"^(.*) \[whyqd:([a-z0-9-]+)\]$", re.DOTALL) diff --git a/whyqd/config/settings.py b/whyqd/config/settings.py index f457095..a52847e 100644 --- a/whyqd/config/settings.py +++ b/whyqd/config/settings.py @@ -12,6 +12,13 @@ class Settings(BaseSettings): FORMAT_VERSIONS: ClassVar[tuple[str, ...]] = ("1.x", "2.0") FORMAT_CURRENT: ClassVar[str] = "2.0" CHECKSUM_ALGORITHM: ClassVar[str] = "BLAKE2b-512" + # What a report shows, so the same file gives the same report anywhere: its places named in a finding, and its + # preview's rows from the top and the end, the length of a cell, and the length of a line + REPORT_PLACES: ClassVar[int] = 100 + REPORT_PREVIEW_HEAD: ClassVar[int] = 15 + REPORT_PREVIEW_TAIL: ClassVar[int] = 5 + REPORT_CELL_WIDTH: ClassVar[int] = 40 + REPORT_LINE_WIDTH: ClassVar[int] = 160 # The encodings a dialect may declare, by the WHATWG Encoding Standard's name in lower case, each with its labels # (https://encoding.spec.whatwg.org/#names-and-labels) and the Python codec that decodes it as the standard does ENCODINGS: ClassVar[dict[str, dict]] = { diff --git a/whyqd/core/datasource.py b/whyqd/core/datasource.py index 140d6a5..65e6b31 100644 --- a/whyqd/core/datasource.py +++ b/whyqd/core/datasource.py @@ -5,7 +5,7 @@ from pydantic import Json from whyqd.dtypes import ChecksumContent, MimeType -from whyqd.models import ChecksumModel, DataSourceModel, CitationModel, DialectModel +from whyqd.models import ChecksumModel, DataSourceModel, CitationModel, DialectModel, ReportModel from whyqd.core.base import BaseDefinition from whyqd.config.settings import settings @@ -133,6 +133,43 @@ def derive_model( # Held so the first `get_data` need not read the file again self.data = None if isinstance(self.model, list) else table.to_pandas() + def diagnose( + self, + *, + source: Path | str, + mimetype: str | MimeType, + dialect: DialectModel | dict | None = None, + propose: bool = False, + ) -> list[ReportModel]: + """How whyqd read a file and what it found there, to review before going on. Nothing is applied, and this + definition does not change: declare what you decide in `derive_model`'s `dialect`. + + A report says which setting took which value, and whether you declared it. A fault is a file that cannot be read + as written, or was read with a loss; a fact is something the file holds, with no claim of what it means, for the + crosswalk to act on. A file that cannot be read gives a report saying why, rather than raising. + + Parameters: + source: Source filename. + mimetype: **whyqd** supports `xls`, `xlsx`, `csv`, `parquet`, `feather`, and, with its extra, `pdf`. + dialect: How the file is written, as in `derive_model`. For a workbook, one naming its `sheet` reports on that + sheet alone; otherwise each sheet reads by it, and has its own report. + propose: Also propose settings that would read past a fault, each with its evidence. + + Returns: + One ReportModel, or one a sheet of a workbook. + + Example: + ```python + import whyqd as qd + + report = qd.DataSourceDefinition().diagnose(source=DATASOURCE_PATH, mimetype=MIMETYPE)[0] + print(report.preview) + for finding in report.findings: + print(finding) + ``` + """ + return self.reader.diagnose(source=source, mimetype=mimetype, dialect=dialect) + ######################################################################################### # VALIDATION UTILITIES ######################################################################################### diff --git a/whyqd/parsers/arrow_reader.py b/whyqd/parsers/arrow_reader.py index dc16794..dd0deb1 100644 --- a/whyqd/parsers/arrow_reader.py +++ b/whyqd/parsers/arrow_reader.py @@ -17,6 +17,7 @@ class ArrowParser: """ PLACE = "row" + DIALECT = () def __init__(self) -> None: self.cell = CellParser() @@ -31,9 +32,15 @@ def grid( Raises ------ ValueError - For a column holding a type whyqd has no text for, naming it. + For a file that is not what its format says, or a column holding a type whyqd has no text for, naming it. """ - df = pl.read_parquet(source) if mime in (MimeType.PARQUET, MimeType.PRQ) else pl.read_ipc(source) + try: + df = pl.read_parquet(source) if mime in (MimeType.PARQUET, MimeType.PRQ) else pl.read_ipc(source) + except pl.exceptions.PolarsError as e: + raise ValueError( + f"`{source.name}` cannot be opened as a {mime.name} file. Check its format, or ask its owner for a " + "sound copy. [whyqd:file-unreadable]" + ) from e columns, types = {}, {} for i, name in enumerate(df.columns): try: diff --git a/whyqd/parsers/csv_reader.py b/whyqd/parsers/csv_reader.py index cd6fbea..cbcc5d1 100644 --- a/whyqd/parsers/csv_reader.py +++ b/whyqd/parsers/csv_reader.py @@ -1,4 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause +from itertools import islice from pathlib import Path import csv @@ -8,7 +9,7 @@ from whyqd.dtypes import EncodingStep, MimeType from whyqd.models import DialectModel from whyqd.parsers.encoding import EncodingParser -from whyqd.parsers.grid import GridParser +from whyqd.parsers.grid import GridParser, ReadFault class CsvParser: @@ -23,6 +24,7 @@ class CsvParser: """ PLACE = "line" + DIALECT = ("encoding", "encodingStep", "delimiter", "quoteChar", "doubleQuote") def __init__(self) -> None: self.encoding = EncodingParser() @@ -87,9 +89,18 @@ def rows(self, *, text: str, dialect: DialectModel, source: Path) -> tuple[list[ lines.append(line) line = reader.line_num + 1 except csv.Error as e: - raise ValueError( - f"`{source.name}` cannot be read from line {line}: {e}. If a quote there is text, declare `quoteChar` " - "as null. [whyqd:csv-quote]" + found = next(islice(self.patterns.LINE.finditer(text), line - 1, None), None) + escaped = found is not None and f"\\{dialect.quoteChar}" in found.group() + setting = "doubleQuote" if escaped else "quoteChar" + advice = ( + "If a backslash escapes a quote there, declare `doubleQuote` as false." + if escaped + else "If a quote there is text, declare `quoteChar` as null." + ) + raise ReadFault( + f"`{source.name}` cannot be read from line {line}: {e}. {advice} [whyqd:csv-quote]", + rows=[line], + setting=setting, ) from e return rows, lines diff --git a/whyqd/parsers/datasource.py b/whyqd/parsers/datasource.py index 35c215a..1a1f0a0 100644 --- a/whyqd/parsers/datasource.py +++ b/whyqd/parsers/datasource.py @@ -13,6 +13,7 @@ from whyqd.parsers.arrow_reader import ArrowParser from whyqd.parsers.core import CoreParser from whyqd.parsers.csv_reader import CsvParser +from whyqd.parsers.diagnosis import DiagnosisParser from whyqd.parsers.excel_reader import ExcelParser from whyqd.parsers.fingerprint import FingerprintParser from whyqd.parsers.pdf_reader import PdfParser @@ -20,7 +21,7 @@ from whyqd.parsers.row_key import RowKeyParser from whyqd.parsers.table import TableParser from whyqd.parsers.v1 import V1ChecksumParser -from whyqd.models import ColumnModel, DataSourceModel, DataSourceAttributeModel, DialectModel +from whyqd.models import ColumnModel, DataSourceModel, DataSourceAttributeModel, DialectModel, ReportModel from whyqd.dtypes import ChecksumContent, MimeType, FieldType if TYPE_CHECKING: @@ -35,6 +36,7 @@ class DataSourceParser: def __init__(self): self.core = CoreParser() + self.diagnoses = DiagnosisParser() self.fingerprints = FingerprintParser() self.profiles = ProfileParser() self.v1 = V1ChecksumParser() @@ -460,6 +462,34 @@ def derive_data_model( return self.derive_sheets(source=source, mime=mimetype, dialect=dialect, names=names) return self.derive(source=source, mime=mimetype, dialect=dialect, names=names) + def diagnose( + self, *, source: Path | str, mimetype: str | MimeType, dialect: DialectModel | dict | None = None + ) -> list[ReportModel]: + """How a file reads under a dialect, and what was found there: one report, or one a sheet of a workbook, or only + the sheet the dialect names. A file that cannot be read gives a report saying so, rather than raising. + + Raises + ------ + ValueError + For a declaration whyqd cannot read, or a format it does not: neither is a fault in the file. + FileNotFoundError + For a file that is not there. + """ + mime = self.get_mimetype(mimetype=mimetype) + given, path, reader = self.declare(dialect=dialect), self.locate(source=source), self.reader(mime=mime) + if mime not in (MimeType.XLS, MimeType.XLSX): + return [self.diagnoses.report(source=path, mime=mime, reader=reader, given=given)] + try: + sheets = reader.sheets(source=path) + except ValueError as e: + return [self.diagnoses.refused(source=path, mime=mime, reader=reader, error=e)] + if given.sheet is not None: + sheets = [(given.sheet, dict(sheets).get(given.sheet, False))] + return [ + self.diagnoses.report(source=path, mime=mime, reader=reader, given=given, sheet=sheet, hidden=hidden) + for sheet, hidden in sheets + ] + def get_source_data_model(self, *, df: pd.DataFrame, source: Path | str, mimetype: str | MimeType) -> DataSourceModel: """The data source of a transform's output, from the frame it was saved from, with its `table-1x` checksum. diff --git a/whyqd/parsers/diagnosis.py b/whyqd/parsers/diagnosis.py new file mode 100644 index 0000000..b61b2e7 --- /dev/null +++ b/whyqd/parsers/diagnosis.py @@ -0,0 +1,256 @@ +# SPDX-License-Identifier: BSD-3-Clause +from pathlib import Path +import warnings + +import polars as pl + +from whyqd.config.patterns import PythonPatterns +from whyqd.config.settings import settings +from whyqd.dtypes import FindingKind, MimeType, SettingSource +from whyqd.models import DialectModel, FindingModel, ReportModel, SettingModel +from whyqd.parsers.table import TableParser + + +class DiagnosisParser: + """How a file was read, and what was found there, for the wrangler to review before going on. + + It interprets nothing. A fault is a file that could not be read as written, or was read with a loss; a fact is + something the file holds, with no claim of what it means, which the crosswalk may act on. Each finding's `code` is + the one a refusal or warning for the same thing carries, and every number counts as the wrangler would look for it. + The file's read and its dialect are left as they were. + """ + + # The dialect's settings the table reads, whatever the file's format + TABLE = ("skipRows", "headerRowCount", "skipBlankRows", "trim", "rowsPerRecord") + # Those a read can settle from the file where they are not declared + SETTLED = ("encoding", "encodingStep", "cells", "sheet") + ADDRESSED_BY = { + "csv-character": "delimiter", + "encoding-replaced": "encoding", + "header-past-end": "skipRows", + "pdf-pages": "pages", + "record-overflow": "rowsPerRecord", + "record-short": "rowsPerRecord", + "sheet-unknown": "sheet", + } + + def __init__(self) -> None: + self.tables = TableParser() + self.patterns = PythonPatterns() + + def report( + self, + *, + source: Path, + mime: MimeType, + reader, + given: DialectModel, + sheet: str | None = None, + hidden: bool = False, + ) -> ReportModel: + """The report on a file, or one sheet of a workbook. + + Parameters + ---------- + source: Path + mime: MimeType + reader + The reader of its format. + given: DialectModel + The dialect as the wrangler declared it. + sheet: str | None, default None + The sheet to read, where the wrangler did not name it. + hidden: bool, default False + Whether the workbook's publisher hid the sheet. + """ + dialect = given.model_copy(update={"sheet": sheet}) if sheet else given + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + try: + grid, types, lines, read = reader.grid(source=source, mime=mime, dialect=dialect) + except ValueError as e: + return self.refused(source=source, mime=mime, reader=reader, error=e, sheet=dialect.sheet) + findings = [f for w in caught if (f := self.finding(error=str(w.message))) is not None] + for w in caught: + if not self.patterns.MESSAGE.match(str(w.message)): + warnings.warn_explicit(w.message, w.category, w.filename, w.lineno) + try: + self.tables.table(grid=grid, lines=lines, dialect=read, place=reader.PLACE) + except ValueError as e: + findings.append(self.finding(error=e)) + widths = self.widths(grid=grid) + findings += self.facts( + source=source, + reader=reader, + sheet=read.sheet, + grid=grid, + types=types, + lines=lines, + widths=widths, + hidden=hidden, + ) + return ReportModel( + path=str(source), + mime=mime, + sheet=read.sheet, + place=reader.PLACE, + settings=self.settings(given=given, read=read, names=(*reader.DIALECT, *self.TABLE)), + height=grid.height, + width=grid.width, + rowWidths=dict(sorted(widths.value_counts().iter_rows())), + findings=findings, + preview=self.preview(grid=grid, lines=lines, place=reader.PLACE), + ) + + def refused( + self, *, source: Path, mime: MimeType, reader, error: ValueError, sheet: str | None = None + ) -> ReportModel: + """The report on a file that could not be read: what failed and where, since no grid exists.""" + return ReportModel( + path=str(source), mime=mime, sheet=sheet, place=reader.PLACE, findings=[self.finding(error=error)] + ) + + def finding(self, *, error: ValueError | str) -> FindingModel | None: + """A refusal or warning as a fault. A message with no code is whyqd's own, not the file's: `None` for a warning, + and an error is raised again, so a fault in the code is not reported as one in the file. + """ + message = self.patterns.MESSAGE.match(str(error)) + if message is None: + if isinstance(error, str): + return None + raise error + code = message.group(2) + return FindingModel( + code=code, + kind=FindingKind.FAULT, + message=message.group(1), + rows=getattr(error, "rows", []), + setting=getattr(error, "setting", None) or self.ADDRESSED_BY.get(code), + ) + + def settings(self, *, given: DialectModel, read: DialectModel, names: tuple[str, ...]) -> list[SettingModel]: + """Each setting the read used, and where its value came from. A byte-order mark overrides a declared encoding, + so an encoding is declared only where the step that settled it says so.""" + models = [] + for name in names: + declared = name in given.model_fields_set + if name == "encoding": + declared = read.encodingStep == "declared" + if declared: + source = SettingSource.DECLARED + elif name in self.SETTLED and getattr(read, name) is not None: + source = SettingSource.FILE + else: + source = SettingSource.DEFAULT + models.append(SettingModel(name=name, value=getattr(read, name), source=source)) + return models + + def widths(self, *, grid: pl.DataFrame) -> pl.Series: + """How far each of the grid's rows reaches: its last filled cell, counted from 1, so `0` is a blank row.""" + if grid.width == 0: + return pl.Series("width", [], dtype=pl.Int64) + reach = [pl.when(pl.col(c).is_not_null()).then(i + 1).otherwise(0) for i, c in enumerate(grid.columns)] + return grid.select(pl.max_horizontal(reach).cast(pl.Int64).alias("width")).to_series() + + def facts( + self, + *, + source: Path, + reader, + sheet: str | None, + grid: pl.DataFrame, + types: pl.DataFrame, + lines: list[int], + widths: pl.Series, + hidden: bool, + ) -> list[FindingModel]: + """What the file holds: its blank rows and columns from the grid, and what only its reader can show.""" + findings = [] + blank = [lines[i] for i in widths.eq(0).arg_true().to_list()] + if blank: + place = reader.PLACE.capitalize() + findings.append( + self.fact( + code="blank-rows", + message=f"{place}{'s' if len(blank) > 1 else ''} {self.listed(blank)} {self.are(blank)} blank.", + rows=blank[: settings.REPORT_PLACES], + ) + ) + empty = [i for i, c in enumerate(grid.columns) if grid.height and grid[c].null_count() == grid.height] + if empty: + findings.append( + self.fact( + code="blank-columns", + message=f"Column{'s' if len(empty) > 1 else ''} {self.listed(empty)} {self.are(empty)} blank, " + "counted from 0.", + columns=empty[: settings.REPORT_PLACES], + ) + ) + if hidden: + findings.append( + self.fact( + code="hidden-sheet", message=f"Sheet `{sheet}` is hidden by its publisher, and read like any other." + ) + ) + if hasattr(reader, "facts"): + cells = reader.facts(source=source, sheet=sheet, types=types) + if merged := cells["merged"]: + findings.append( + self.fact( + code="merged-range", + message=f"Cells {self.listed(merged)} are merged; the first of each holds its value, and the " + "rest read as empty.", + cells=merged[: settings.REPORT_PLACES], + ) + ) + if errors := cells["errors"]: + findings.append( + self.fact( + code="error-cells", + message=f"Cell{'s' if len(errors) > 1 else ''} {self.listed(errors)} {self.are(errors)} error " + "cells, each read as its code.", + cells=errors[: settings.REPORT_PLACES], + ) + ) + return findings + + def fact(self, *, code: str, message: str, **where) -> FindingModel: + return FindingModel(code=code, kind=FindingKind.FACT, message=message, **where) + + def listed(self, items: list) -> str: + """Items in words, up to the number a report names: `2 and 3`; `1, 2, and 3`; `1, 2, and 98 more`.""" + limit = settings.REPORT_PLACES + shown = [str(i) for i in items[:limit]] + if len(items) > limit: + return f"{', '.join(shown)}, and {len(items) - limit} more" + if len(shown) < 3: + return " and ".join(shown) + return f"{', '.join(shown[:-1])}, and {shown[-1]}" + + def are(self, items: list) -> str: + return "are" if len(items) > 1 else "is" + + def preview(self, *, grid: pl.DataFrame, lines: list[int], place: str) -> str: + """A short text of the grid as read, from its top and its end, each row numbered as the wrangler would look for + it, so the rows above a header can be seen, which leave the table. A row's trailing empty cells are left out, a + line break in a cell shows as `\\n`, and each cell and line is cut to the length a report shows.""" + head, tail = settings.REPORT_PREVIEW_HEAD, settings.REPORT_PREVIEW_TAIL + shown = ( + list(range(grid.height)) + if grid.height <= head + tail + else [*range(head), *range(grid.height - tail, grid.height)] + ) + text = [] + for previous, i in zip([-1, *shown], shown, strict=False): + if i - previous > 1: + text.append(f"... {i - previous - 1} {place}s not shown ...") + row = list(grid.row(i)) + while row and row[-1] is None: + row.pop() + cells = " | ".join(self.shortened(value=c, width=settings.REPORT_CELL_WIDTH) for c in row) + text.append(self.shortened(value=f"{place} {lines[i]}: {cells}", width=settings.REPORT_LINE_WIDTH)) + return "\n".join(text) + + def shortened(self, *, value: str | None, width: int) -> str: + value = (value or "").replace("\r", "\\r").replace("\n", "\\n") + return value if len(value) <= width else f"{value[: width - 1]}…" diff --git a/whyqd/parsers/excel_reader.py b/whyqd/parsers/excel_reader.py index d726973..3a67cd7 100644 --- a/whyqd/parsers/excel_reader.py +++ b/whyqd/parsers/excel_reader.py @@ -11,7 +11,7 @@ import python_calamine from whyqd.config.patterns import PythonPatterns -from whyqd.dtypes import MimeType +from whyqd.dtypes import CellType, MimeType from whyqd.models import DialectModel from whyqd.parsers.cell import CellParser from whyqd.parsers.grid import GridParser @@ -45,6 +45,7 @@ class ExcelParser: """ PLACE = "row" + DIALECT = ("sheet",) # Each error's code, as Excel shows it: calamine's `CellErrorType` (https://github.com/tafia/calamine, `lib.rs`) ERRORS = { "Div0": "#DIV/0!", @@ -98,8 +99,20 @@ def __init__(self) -> None: self.patterns = PythonPatterns() def sheets(self, *, source: Path) -> list[tuple[str, bool]]: - """Each worksheet's name, in workbook order, and whether its publisher hid it.""" - workbook = python_calamine.CalamineWorkbook.from_path(str(source)) + """Each worksheet's name, in workbook order, and whether its publisher hid it. + + Raises + ------ + ValueError + For a file that is not a sound workbook. + """ + try: + workbook = python_calamine.CalamineWorkbook.from_path(str(source)) + except python_calamine.CalamineError as e: + raise ValueError( + f"`{source.name}` cannot be opened as a workbook. Check its format, or ask its owner for a sound copy. " + "[whyqd:file-unreadable]" + ) from e return [ (m.name, m.visible != python_calamine.SheetVisibleEnum.Visible) for m in workbook.sheets_metadata @@ -160,6 +173,28 @@ def grid( grid, types = self.grids.trim(grid=pl.DataFrame(columns, schema=schema), types=pl.DataFrame(types, schema=schema)) return grid, types, list(range(1, grid.height + 1)), dialect.model_copy(update={"sheet": sheet}) + def facts(self, *, source: Path, sheet: str, types: pl.DataFrame) -> dict[str, list[str]]: + """What a sheet holds that its grid's text does not show: its `merged` ranges and its `errors`, each cell named + as a workbook names it, in reading order.""" + read = python_calamine.CalamineWorkbook.from_path(str(source)).get_sheet_by_name(sheet) + merged = sorted(read.merged_cell_ranges or []) + errors = sorted( + (i, j) for j, c in enumerate(types.columns) for i in types[c].eq(CellType.ERROR.value).arg_true().to_list() + ) + reference = self.reference + return { + "merged": [f"{reference(row=a, column=b)}:{reference(row=c, column=d)}" for (a, b), (c, d) in merged], + "errors": [self.reference(row=i, column=j) for i, j in errors], + } + + def reference(self, *, row: int, column: int) -> str: + """A cell's reference from its row and column, each counted from 0: `B3` is row 2, column 1.""" + letters, column = "", column + 1 + while column: + column, remainder = divmod(column - 1, 26) + letters = chr(65 + remainder) + letters + return f"{letters}{row + 1}" + def formats(self, *, source: Path, sheet: str, height: int, width: int) -> pl.DataFrame: """Each cell's number format code in a sheet of an `.xlsx`, in the shape of its grid. diff --git a/whyqd/parsers/grid.py b/whyqd/parsers/grid.py index ac58aff..889c79e 100644 --- a/whyqd/parsers/grid.py +++ b/whyqd/parsers/grid.py @@ -4,6 +4,15 @@ from whyqd.dtypes import CellType +class ReadFault(ValueError): + """A file that cannot be read as its dialect describes, saying where, and which setting addresses it.""" + + def __init__(self, message: str, *, rows: list[int] | None = None, setting: str | None = None) -> None: + super().__init__(message) + self.rows = rows or [] + self.setting = setting + + class GridParser: """The grid every reader gives: each cell's text, null where empty, in columns `column_0` on, and beside it each cell's type, a `CellType`, in a frame of the same shape. diff --git a/whyqd/parsers/pdf_reader.py b/whyqd/parsers/pdf_reader.py index 60dfe58..7bc26ca 100644 --- a/whyqd/parsers/pdf_reader.py +++ b/whyqd/parsers/pdf_reader.py @@ -18,6 +18,7 @@ class PdfParser: """ PLACE = "row" + DIALECT = ("cells", "pages", "columns") # Points between the tops of two words on one line: pdfplumber's own `y_tolerance` LINE = 3 @@ -36,13 +37,21 @@ def grid( ImportError Without the `pdf` extra. ValueError - For a PDF with no text, such as a scan, or pages it does not have. + For a file that is not a PDF, a PDF with no text, such as a scan, or pages it does not have. """ try: import pdfplumber + from pdfplumber.utils.exceptions import PdfminerException except ImportError as e: raise ImportError("Reading a PDF needs the extra: pip install whyqd[pdf] [whyqd:pdf-extra]") from e - with pdfplumber.open(source) as pdf: + try: + opened = pdfplumber.open(source) + except PdfminerException as e: + raise ValueError( + f"`{source.name}` cannot be opened as a PDF file. Check its format, or ask its owner for a sound copy. " + "[whyqd:file-unreadable]" + ) from e + with opened as pdf: pages = self.pages(pages=pdf.pages, declared=dialect.pages, source=source) if not any(page.chars for page in pages): raise ValueError( From dc4ba5c02182337c2075c590af7185f2ba73837c Mon Sep 17 00:00:00 2001 From: Gavin Chait Date: Mon, 2 Nov 2026 18:24:39 +0100 Subject: [PATCH 4/7] Propose settings that would read past a file's faults --- pyproject.toml | 3 + tests/test_diagnosis.py | 68 ++++++++++++++++++++--- uv.lock | 6 +- whyqd/config/settings.py | 4 ++ whyqd/core/datasource.py | 2 +- whyqd/parsers/csv_reader.py | 106 ++++++++++++++++++++++++++++++++---- whyqd/parsers/datasource.py | 13 ++++- whyqd/parsers/diagnosis.py | 62 ++++++++++++++++++++- whyqd/parsers/encoding.py | 35 +++++++++++- 9 files changed, 272 insertions(+), 27 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ee0a4b7..f326e08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,9 @@ dependencies = [ ] [project.optional-dependencies] +diagnose = [ + "charset-normalizer>=3.5.2,<4", +] pdf = [ "pdfplumber>=0.11.10,<0.12", ] diff --git a/tests/test_diagnosis.py b/tests/test_diagnosis.py index 353becc..efcbe53 100644 --- a/tests/test_diagnosis.py +++ b/tests/test_diagnosis.py @@ -6,8 +6,10 @@ """ import json +import sys from pathlib import Path +import pandas as pd import pytest import whyqd as qd @@ -24,12 +26,6 @@ ] CASES = CSV + WORKBOOK IDS = [f"{name}-{dialect.get('sheet', '')}" for name, _, dialect, _ in CASES] -PROPOSING = [ - pytest.param( - *c, marks=[pytest.mark.xfail(strict=True, reason="proposals are not built")] if c[3]["proposals"] else [] - ) - for c in CASES -] SYNTHETIC = LAYOUTS.parent / "synthetic" FENLAND = LAYOUTS.parent / "sample" / "fenland" / "2026-10-25-Fenland District Council.csv" @@ -58,13 +54,71 @@ def test_each_layout_finds_what_its_key_says(name, mimetype, dialect, case): assert found(diagnose(name, mimetype, dialect)) == case["findings"] -@pytest.mark.parametrize("name,mimetype,dialect,case", PROPOSING, ids=IDS) +@pytest.mark.parametrize("name,mimetype,dialect,case", CASES, ids=IDS) def test_each_layout_proposes_what_its_key_says(name, mimetype, dialect, case): report = diagnose(name, mimetype, dialect, propose=True) assert {p.setting: p.value for p in report.proposals} == case["proposals"] assert all(p.evidence for p in report.proposals) +@pytest.mark.parametrize("name", [n for n, case in KEY["csv"].items() if case["proposals"]]) +def test_each_proposal_once_declared_reads_to_the_key(name): + case = KEY["csv"][name] + report = diagnose(name, "CSV", {}, propose=True) + datasource = qd.DataSourceDefinition() + datasource.derive_model( + source=LAYOUTS / name, mimetype="CSV", dialect={p.setting: p.value for p in report.proposals} + ) + df = datasource.get_data() + assert [c.name for c in datasource.get.columns] == case["names"] + assert [[None if pd.isna(c) else c for c in row] for row in df[case["names"]].values.tolist()] == [ + [c if c != "" else None for c in row] for row in case["rows"] + ] + + +def test_fenland_is_proposed_its_records(): + report = diagnose_source(FENLAND, "CSV", {}, propose=True) + assert [(p.setting, p.value) for p in report.proposals] == [("rowsPerRecord", [47, 1])] + + +def test_a_file_declared_its_records_is_not_proposed_them_again(): + assert diagnose_source(FENLAND, "CSV", {"rowsPerRecord": [47, 1]}, propose=True).proposals == [] + + +@pytest.mark.parametrize( + "name,dialect,proposed", + [ + ("fixture-mac-roman.csv", {}, "macintosh"), + ("fixture-windows-1252.csv", {}, None), + ("fixture-windows-1252.csv", {"encoding": "utf-8"}, "windows-1252"), + ("fixture-utf8.csv", {}, None), + ("fixture-utf8-bom.csv", {}, None), + ], +) +def test_an_encoding_is_proposed_only_for_a_file_that_did_not_decode_as_itself(name, dialect, proposed): + report = diagnose_source(SYNTHETIC / name, "CSV", dialect, propose=True) + assert [p.value for p in report.proposals if p.setting == "encoding"] == ([proposed] if proposed else []) + + +def test_without_the_extra_a_finding_says_so_where_an_encoding_would_be_proposed(monkeypatch): + monkeypatch.setitem(sys.modules, "charset_normalizer", None) + report = diagnose_source(SYNTHETIC / "fixture-mac-roman.csv", "CSV", {}, propose=True) + assert report.proposals == [] + assert "diagnose-extra" in [f.code for f in report.findings] + assert diagnose_source(SYNTHETIC / "fixture-utf8.csv", "CSV", {}, propose=True).findings == [] + + +def test_detection_does_not_run_in_a_load(monkeypatch): + monkeypatch.setitem(sys.modules, "charset_normalizer", None) + qd.DataSourceDefinition().derive_model(source=SYNTHETIC / "fixture-mac-roman.csv", mimetype="CSV") + + +def test_a_clean_comma_file_with_semicolons_in_its_cells_is_not_second_guessed(tmp_path): + source = tmp_path / "semis.csv" + source.write_text("a,b\n1;2,3\n4;5,6\n", encoding="utf-8") + assert diagnose_source(source, "CSV", {}, propose=True).proposals == [] + + @pytest.mark.parametrize("name,mimetype,dialect,case", CASES, ids=IDS) def test_nothing_is_proposed_unasked(name, mimetype, dialect, case): assert diagnose(name, mimetype, dialect).proposals == [] diff --git a/uv.lock b/uv.lock index 54ae0cd..eed99d4 100644 --- a/uv.lock +++ b/uv.lock @@ -2796,6 +2796,9 @@ dependencies = [ ] [package.optional-dependencies] +diagnose = [ + { name = "charset-normalizer" }, +] pdf = [ { name = "pdfplumber" }, ] @@ -2829,6 +2832,7 @@ notebooks = [ [package.metadata] requires-dist = [ + { name = "charset-normalizer", marker = "extra == 'diagnose'", specifier = ">=3.5.2,<4" }, { name = "fastexcel", specifier = ">=0.21.0,<0.22" }, { name = "numpy", specifier = ">=2.4.6,<3" }, { name = "openpyxl", marker = "extra == 'verify-1x'", specifier = ">=3.1.5,<4" }, @@ -2846,7 +2850,7 @@ requires-dist = [ { name = "rfc8785", specifier = ">=0.1.4,<0.2" }, { name = "xlrd", marker = "extra == 'verify-1x'", specifier = ">=2.0.2,<3" }, ] -provides-extras = ["pdf", "verify-1x"] +provides-extras = ["diagnose", "pdf", "verify-1x"] [package.metadata.requires-dev] dev = [ diff --git a/whyqd/config/settings.py b/whyqd/config/settings.py index a52847e..a860c4f 100644 --- a/whyqd/config/settings.py +++ b/whyqd/config/settings.py @@ -19,6 +19,10 @@ class Settings(BaseSettings): REPORT_PREVIEW_TAIL: ClassVar[int] = 5 REPORT_CELL_WIDTH: ClassVar[int] = 40 REPORT_LINE_WIDTH: ClassVar[int] = 160 + # The lines a proposed delimiter is judged on, and the encodings a proposal chooses among, from `ENCODINGS` below: + # unconstrained, detection misreads small files, and UTF-16 is read by its byte-order mark alone + REPORT_SAMPLE_LINES: ClassVar[int] = 1000 + ENCODING_SHORTLIST: ClassVar[tuple[str, ...]] = ("utf-8", "windows-1252", "macintosh") # The encodings a dialect may declare, by the WHATWG Encoding Standard's name in lower case, each with its labels # (https://encoding.spec.whatwg.org/#names-and-labels) and the Python codec that decodes it as the standard does ENCODINGS: ClassVar[dict[str, dict]] = { diff --git a/whyqd/core/datasource.py b/whyqd/core/datasource.py index 65e6b31..eab30f7 100644 --- a/whyqd/core/datasource.py +++ b/whyqd/core/datasource.py @@ -168,7 +168,7 @@ def diagnose( print(finding) ``` """ - return self.reader.diagnose(source=source, mimetype=mimetype, dialect=dialect) + return self.reader.diagnose(source=source, mimetype=mimetype, dialect=dialect, propose=propose) ######################################################################################### # VALIDATION UTILITIES diff --git a/whyqd/parsers/csv_reader.py b/whyqd/parsers/csv_reader.py index cbcc5d1..7dd9e9a 100644 --- a/whyqd/parsers/csv_reader.py +++ b/whyqd/parsers/csv_reader.py @@ -2,12 +2,14 @@ from itertools import islice from pathlib import Path import csv +import warnings import polars as pl from whyqd.config.patterns import PythonPatterns +from whyqd.config.settings import settings from whyqd.dtypes import EncodingStep, MimeType -from whyqd.models import DialectModel +from whyqd.models import DialectModel, FindingModel, ProposalModel from whyqd.parsers.encoding import EncodingParser from whyqd.parsers.grid import GridParser, ReadFault @@ -25,6 +27,10 @@ class CsvParser: PLACE = "line" DIALECT = ("encoding", "encodingStep", "delimiter", "quoteChar", "doubleQuote") + # What a proposed delimiter is chosen among + DELIMITERS = (",", ";", "\t", "|") + # What a `csv-quote` fault's setting takes, to read past it + PAST_QUOTE = {"quoteChar": None, "doubleQuote": False} def __init__(self) -> None: self.encoding = EncodingParser() @@ -55,16 +61,99 @@ def grid( ------- tuple[pl.DataFrame, pl.DataFrame, list[int], DialectModel] """ + text, encoding, step = self.decoded(source=source, dialect=dialect) + dialect = dialect.model_copy(update={"encoding": encoding, "encodingStep": step}) + rows, lines = self.rows(text=text, dialect=dialect, source=source) + grid, types = self.to_grid(rows=rows) + return grid, types, lines[: grid.height], dialect + + def decoded(self, *, source: Path, dialect: DialectModel) -> tuple[str, str, EncodingStep]: declared = None if dialect.encodingStep == EncodingStep.DECLARED or ( dialect.encodingStep is None and "encoding" in dialect.model_fields_set ): declared = dialect.encoding - text, encoding, step = self.encoding.decode(data=source.read_bytes(), declared=declared, source=f"`{source.name}`") - dialect = dialect.model_copy(update={"encoding": encoding, "encodingStep": step}) - rows, lines = self.rows(text=text, dialect=dialect, source=source) - grid, types = self.to_grid(rows=rows) - return grid, types, lines[: grid.height], dialect + return self.encoding.decode(data=source.read_bytes(), declared=declared, source=f"`{source.name}`") + + def proposals( + self, *, source: Path, mime: MimeType, dialect: DialectModel, faults: list[FindingModel] + ) -> list[ProposalModel]: + """The settings that would read past this file's faults, each with its evidence: what lets the file read at all, + or else a delimiter that fits its lines where the one read by does not. + + A quote setting is proposed only once the whole file has read with it. A delimiter is proposed only where one + alone gives every line the same width, over one cell, and the one read by does not, so a sound file is never + second-guessed. Where the evidence is mixed, nothing is proposed. + """ + found = [] + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + for fault in faults: + if fault.code != "csv-quote" or fault.setting not in self.PAST_QUOTE: + continue + value = self.PAST_QUOTE[fault.setting] + try: + self.grid(source=source, mime=mime, dialect=dialect.model_copy(update={fault.setting: value})) + except ValueError: + continue + found.append( + ProposalModel( + setting=fault.setting, + value=value, + evidence=f"Line {fault.rows[0]} cannot be read without it, and the whole file reads with it.", + ) + ) + if not found and not any(f.code == "csv-quote" for f in faults): + text = self.decoded(source=source, dialect=dialect)[0] + found += self.delimiter(text=text, dialect=dialect) + return found + + def delimiter(self, *, text: str, dialect: DialectModel) -> list[ProposalModel]: + lines = [m.group() for m in islice(self.patterns.LINE.finditer(text), settings.REPORT_SAMPLE_LINES)] + lines = [line for line in lines if line.strip("\r\n")] + + def width(delimiter: str) -> int | None: + try: + widths = {len(row) for row in csv.reader(lines, delimiter=delimiter) if row} + except csv.Error: + return None + return widths.pop() if len(widths) == 1 else None + + if width(dialect.delimiter) is not None: + return [] + fits = {d: w for d in self.DELIMITERS if d != dialect.delimiter and (w := width(d)) is not None and w > 1} + if len(fits) != 1: + return [] + delimiter, cells = fits.popitem() + return [ + ProposalModel( + setting="delimiter", + value=delimiter, + evidence=f"It gives each of the first {len(lines)} lines {cells} cells; " + f"`{dialect.delimiter}` gives them differing widths.", + ) + ] + + def encoding_proposal( + self, *, source: Path, dialect: DialectModel, faults: list[FindingModel] + ) -> ProposalModel | None: + """The encoding this file more likely is, where its read could only fall back on `windows-1252`, or replaced + bytes it could not decode. A file that decoded as itself is never second-guessed. + + Raises + ------ + ImportError + Without the `diagnose` extra, only where a proposal is called for. + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + _, read, step = self.decoded(source=source, dialect=dialect) + if step != EncodingStep.WINDOWS_1252 and not any(f.code == "encoding-replaced" for f in faults): + return None + detected = self.encoding.detect(data=source.read_bytes()) + if detected is None or detected[0] == read: + return None + return ProposalModel(setting="encoding", value=detected[0], evidence=detected[1]) def rows(self, *, text: str, dialect: DialectModel, source: Path) -> tuple[list[list[str]], list[int]]: settings = {"delimiter": dialect.delimiter, "doublequote": dialect.doubleQuote, "strict": True} @@ -107,10 +196,7 @@ def rows(self, *, text: str, dialect: DialectModel, source: Path) -> tuple[list[ def to_grid(self, *, rows: list[list[str]]) -> tuple[pl.DataFrame, pl.DataFrame]: width = max((len(r) for r in rows), default=0) grid = pl.DataFrame( - { - f"column_{i}": [r[i] if i < len(r) and r[i] != "" else None for r in rows] - for i in range(width) - }, + {f"column_{i}": [r[i] if i < len(r) and r[i] != "" else None for r in rows] for i in range(width)}, schema={f"column_{i}": pl.String for i in range(width)}, ) return self.grids.trim(grid=grid) diff --git a/whyqd/parsers/datasource.py b/whyqd/parsers/datasource.py index 1a1f0a0..8aa4cd2 100644 --- a/whyqd/parsers/datasource.py +++ b/whyqd/parsers/datasource.py @@ -463,7 +463,12 @@ def derive_data_model( return self.derive(source=source, mime=mimetype, dialect=dialect, names=names) def diagnose( - self, *, source: Path | str, mimetype: str | MimeType, dialect: DialectModel | dict | None = None + self, + *, + source: Path | str, + mimetype: str | MimeType, + dialect: DialectModel | dict | None = None, + propose: bool = False, ) -> list[ReportModel]: """How a file reads under a dialect, and what was found there: one report, or one a sheet of a workbook, or only the sheet the dialect names. A file that cannot be read gives a report saying so, rather than raising. @@ -478,7 +483,7 @@ def diagnose( mime = self.get_mimetype(mimetype=mimetype) given, path, reader = self.declare(dialect=dialect), self.locate(source=source), self.reader(mime=mime) if mime not in (MimeType.XLS, MimeType.XLSX): - return [self.diagnoses.report(source=path, mime=mime, reader=reader, given=given)] + return [self.diagnoses.report(source=path, mime=mime, reader=reader, given=given, propose=propose)] try: sheets = reader.sheets(source=path) except ValueError as e: @@ -486,7 +491,9 @@ def diagnose( if given.sheet is not None: sheets = [(given.sheet, dict(sheets).get(given.sheet, False))] return [ - self.diagnoses.report(source=path, mime=mime, reader=reader, given=given, sheet=sheet, hidden=hidden) + self.diagnoses.report( + source=path, mime=mime, reader=reader, given=given, sheet=sheet, hidden=hidden, propose=propose + ) for sheet, hidden in sheets ] diff --git a/whyqd/parsers/diagnosis.py b/whyqd/parsers/diagnosis.py index b61b2e7..0a8cc24 100644 --- a/whyqd/parsers/diagnosis.py +++ b/whyqd/parsers/diagnosis.py @@ -7,7 +7,7 @@ from whyqd.config.patterns import PythonPatterns from whyqd.config.settings import settings from whyqd.dtypes import FindingKind, MimeType, SettingSource -from whyqd.models import DialectModel, FindingModel, ReportModel, SettingModel +from whyqd.models import DialectModel, FindingModel, ProposalModel, ReportModel, SettingModel from whyqd.parsers.table import TableParser @@ -47,6 +47,7 @@ def report( given: DialectModel, sheet: str | None = None, hidden: bool = False, + propose: bool = False, ) -> ReportModel: """The report on a file, or one sheet of a workbook. @@ -62,6 +63,8 @@ def report( The sheet to read, where the wrangler did not name it. hidden: bool, default False Whether the workbook's publisher hid the sheet. + propose: bool, default False + Also propose settings that would read past a fault. Never applied. """ dialect = given.model_copy(update={"sheet": sheet}) if sheet else given with warnings.catch_warnings(record=True) as caught: @@ -69,7 +72,12 @@ def report( try: grid, types, lines, read = reader.grid(source=source, mime=mime, dialect=dialect) except ValueError as e: - return self.refused(source=source, mime=mime, reader=reader, error=e, sheet=dialect.sheet) + report = self.refused(source=source, mime=mime, reader=reader, error=e, sheet=dialect.sheet) + return ( + self.propose(report=report, source=source, mime=mime, reader=reader, dialect=dialect) + if propose + else report + ) findings = [f for w in caught if (f := self.finding(error=str(w.message))) is not None] for w in caught: if not self.patterns.MESSAGE.match(str(w.message)): @@ -89,7 +97,7 @@ def report( widths=widths, hidden=hidden, ) - return ReportModel( + report = ReportModel( path=str(source), mime=mime, sheet=read.sheet, @@ -101,6 +109,54 @@ def report( findings=findings, preview=self.preview(grid=grid, lines=lines, place=reader.PLACE), ) + if propose: + return self.propose(report=report, source=source, mime=mime, reader=reader, dialect=dialect, widths=widths) + return report + + def propose( + self, + *, + report: ReportModel, + source: Path, + mime: MimeType, + reader, + dialect: DialectModel, + widths: pl.Series | None = None, + ) -> ReportModel: + """A report with the settings that would read past the file's faults added. Without the extra that one needs, a + finding says so, in place of that proposal.""" + faults = report.findings + proposals = [] + if hasattr(reader, "proposals"): + proposals += reader.proposals(source=source, mime=mime, dialect=dialect, faults=faults) + try: + if encoding := reader.encoding_proposal(source=source, dialect=dialect, faults=faults): + proposals.append(encoding) + except ImportError as e: + message = self.patterns.MESSAGE.match(str(e)) + report.findings.append(self.fact(code=message.group(2), message=message.group(1))) + if widths is not None and (records := self.records(widths=widths, dialect=dialect)): + proposals.append(records) + report.proposals = proposals + return report + + def records(self, *, widths: pl.Series, dialect: DialectModel) -> ProposalModel | None: + """`rowsPerRecord`, where rows alternate through the whole file, past any declared `skipRows`: every first row + of a pair reaches past every second, and every second holds something somewhere. Anything less is left silent: + a record over more rows, or a mix, is for the wrangler to read and declare.""" + if dialect.rowsPerRecord: + return None + reaches = widths.to_list()[dialect.skipRows :] + first, second = reaches[::2], reaches[1::2] + if len(reaches) < 4 or len(first) != len(second) or min(first) <= max(second) or max(second) == 0: + return None + fields = [max(first), max(second)] + return ProposalModel( + setting="rowsPerRecord", + value=fields, + evidence=f"All {len(reaches)} rows alternate: one reaching up to {fields[0]} cells, then one up to " + f"{fields[1]}, and every first reaches past every second.", + ) def refused( self, *, source: Path, mime: MimeType, reader, error: ValueError, sheet: str | None = None diff --git a/whyqd/parsers/encoding.py b/whyqd/parsers/encoding.py index a1b8fde..eac0119 100644 --- a/whyqd/parsers/encoding.py +++ b/whyqd/parsers/encoding.py @@ -45,7 +45,9 @@ def name(self, *, label: str) -> str: "[whyqd:encoding-unknown]" ) - def decode(self, *, data: bytes, declared: str | None = None, source: str = "The file") -> tuple[str, str, EncodingStep]: + def decode( + self, *, data: bytes, declared: str | None = None, source: str = "The file" + ) -> tuple[str, str, EncodingStep]: """A file's text, the encoding that decoded it, and the step that settled it. A declared encoding that cannot decode some bytes still loads: each such byte reads as U+FFFD, as the standard's @@ -67,7 +69,7 @@ def decode(self, *, data: bytes, declared: str | None = None, source: str = "The """ for mark, name in self.BOMS: if data.startswith(mark): - return self.read(data=data[len(mark):], name=name, source=source), name, EncodingStep.BOM + return self.read(data=data[len(mark) :], name=name, source=source), name, EncodingStep.BOM if declared is not None: name = self.name(label=declared) return self.read(data=data, name=name, source=source), name, EncodingStep.DECLARED @@ -76,6 +78,35 @@ def decode(self, *, data: bytes, declared: str | None = None, source: str = "The except UnicodeDecodeError: return self.read(data=data, name="windows-1252"), "windows-1252", EncodingStep.WINDOWS_1252 + def detect(self, *, data: bytes) -> tuple[str, str] | None: + """The encoding a file's bytes most likely are, among those whyqd shortlists, and the evidence. A guess, so only + ever a proposal, and it never runs in a load. Read through charset-normalizer, the `diagnose` extra. + + Raises + ------ + ImportError + Without the `diagnose` extra. + + Returns + ------- + tuple[str, str] | None + The encoding's name and the evidence, or `None` where none of the shortlist decodes the bytes. + """ + try: + import charset_normalizer + except ImportError as e: + raise ImportError( + "Proposing an encoding needs the extra: pip install whyqd[diagnose] [whyqd:diagnose-extra]" + ) from e + names = {codecs.lookup(settings.ENCODINGS[n]["codec"]).name: n for n in settings.ENCODING_SHORTLIST} + best = charset_normalizer.from_bytes(data, cp_isolation=list(names)).best() + if best is None: + return None + return names[codecs.lookup(best.encoding).name], ( + f"Of {', '.join(settings.ENCODING_SHORTLIST)}, its bytes read most cleanly as this, by charset-normalizer: " + f"{best.chaos:.1%} of its text looks wrong. Check the accented letters and symbols in the preview." + ) + def read(self, *, data: bytes, name: str, source: str = "The file") -> str: if name == "windows-1252": return codecs.charmap_decode(data, "strict", self.WINDOWS_1252)[0] From 8649c16ae282095471071cd320be27c5a7f58b79 Mon Sep 17 00:00:00 2001 From: Gavin Chait Date: Mon, 2 Nov 2026 19:15:15 +0100 Subject: [PATCH 5/7] Propose any delimiter and clear the repository's lint --- tests/data/builders/fixtures.py | 188 ++++++-- tests/data/builders/fixtures_typing.py | 176 +++++-- tests/data/builders/layouts.py | 451 +++++++++++++----- ...piled-epoch-0038-changeupdate-0001-csv.csv | 5 + ...piled-epoch-0038-changeupdate-0001-csv.csv | 34 ++ tests/test_datasource.py | 8 +- tests/test_diagnosis.py | 33 ++ tests/test_header.py | 8 +- tests/test_known_faults.py | 9 +- tests/test_polars.py | 8 +- tests/test_tutorial_4.py | 2 +- whyqd/__main__.py | 1 + whyqd/config/settings.py | 20 +- whyqd/core/datasource.py | 4 +- whyqd/core/schema.py | 4 +- whyqd/crosswalk/actions/__init__.py | 10 +- whyqd/crosswalk/actions/collate.py | 11 +- whyqd/crosswalk/base/category_base.py | 2 +- whyqd/models/action_category.py | 2 +- whyqd/models/action_morph.py | 2 +- whyqd/models/datasource.py | 7 +- whyqd/parsers/action.py | 4 +- whyqd/parsers/arrow_reader.py | 4 +- whyqd/parsers/category.py | 18 +- whyqd/parsers/core.py | 7 +- whyqd/parsers/csv_reader.py | 42 +- whyqd/parsers/datasource.py | 26 +- whyqd/parsers/excel_reader.py | 12 +- whyqd/parsers/pdf_reader.py | 4 +- whyqd/parsers/table.py | 4 +- 30 files changed, 806 insertions(+), 300 deletions(-) create mode 100644 tests/data/sample/voa/uk-englandwales-ndr-2010-listentries-compiled-epoch-0038-changeupdate-0001-csv.csv create mode 100644 tests/data/sample/voa/uk-englandwales-ndr-2010-summaryvaluations-compiled-epoch-0038-changeupdate-0001-csv.csv diff --git a/tests/data/builders/fixtures.py b/tests/data/builders/fixtures.py index 8090664..230256e 100644 --- a/tests/data/builders/fixtures.py +++ b/tests/data/builders/fixtures.py @@ -24,39 +24,84 @@ # One column per defect. "text" is the cell as a CSV carries it; "cell" (workbook only) is what is stored when it # differs from the text, with its number format; None in "cell" means the cell is left empty. COLUMNS = [ - ("ref_text", "a reference with leading zeros, stored as text", - ["0000130842", "0000000001", "1234567890", "0012"], None), - ("ref_padded", "a reference stored as a number, shown zero-padded by its number format", - ["0000130842", "0000000001", "1234567890", "0000000012"], - {"values": [130842, 1, 1234567890, 12], "format": "0000000000"}), - ("amount", "currency text, including a sign after the symbol and a bare decimal", - ["£43.40", "£-826.80", "£1,234.50", "£.00"], None), - ("amount_mojibake", "the same with a UTF-8 £ read as Mac Roman, as Amber Valley's workbooks carry it", - ["¬£43.40", "¬£-826.80", "¬£1,234.50", "¬£.00"], None), - ("number_as_text", "numbers stored as text, one with a leading apostrophe-free space", - ["12.50", "0.10", "1e3", " 7"], None), - ("number", "numbers stored as numbers, shown to two places", - ["12.50", "0.10", "1000.00", "7.00"], {"values": [12.5, 0.1, 1000, 7], "format": "0.00"}), - ("blank_like", "a hard space only, spaces only, an empty cell, and a hard space inside text", - [" ", " ", "", "a b"], {"values": [" ", " ", None, "a b"], "format": "General"}), - ("padded_text", "leading and trailing spaces kept as published", - [" lead", "trail ", " both ", "none"], None), - ("accented", "characters every single-byte encoding here can carry", - ["Café", "naïve", "Ætna", "Müller £"], None), - ("date_text", "dates as text: day-first, ambiguous, year-first as pandas re-saves them, and ISO", - ["13/04/2024", "04/01/2024", "2010-04-01 00:00:00", "2024-04-01"], None), - ("date_mixed", "the Stockport case: date cells where Excel could read a date, text where it could not", - ["04/01/2024", "13/04/2024", "05/02/2024", "25/12/2024"], - {"values": [date(2024, 1, 4), "13/04/2024", date(2024, 2, 5), "25/12/2024"], "format": "dd/mm/yyyy"}), - ("date_locale", "a date cell in Excel's built-in short date format, whose display follows the machine's locale", - [None, None, None, None], {"values": [date(2024, 1, 4), date(2024, 12, 25), datetime(2024, 4, 1, 9, 30), None], - "format": "mm-dd-yy"}), - ("boolean_like", "boolean cells and boolean-looking text", - ["TRUE", "false", "Y", "FALSE"], {"values": [True, "false", "Y", False], "format": "General"}), + ( + "ref_text", + "a reference with leading zeros, stored as text", + ["0000130842", "0000000001", "1234567890", "0012"], + None, + ), + ( + "ref_padded", + "a reference stored as a number, shown zero-padded by its number format", + ["0000130842", "0000000001", "1234567890", "0000000012"], + {"values": [130842, 1, 1234567890, 12], "format": "0000000000"}, + ), + ( + "amount", + "currency text, including a sign after the symbol and a bare decimal", + ["£43.40", "£-826.80", "£1,234.50", "£.00"], + None, + ), + ( + "amount_mojibake", + "the same with a UTF-8 £ read as Mac Roman, as Amber Valley's workbooks carry it", + ["¬£43.40", "¬£-826.80", "¬£1,234.50", "¬£.00"], + None, + ), + ( + "number_as_text", + "numbers stored as text, one with a leading apostrophe-free space", + ["12.50", "0.10", "1e3", " 7"], + None, + ), + ( + "number", + "numbers stored as numbers, shown to two places", + ["12.50", "0.10", "1000.00", "7.00"], + {"values": [12.5, 0.1, 1000, 7], "format": "0.00"}, + ), + ( + "blank_like", + "a hard space only, spaces only, an empty cell, and a hard space inside text", + [" ", " ", "", "a b"], + {"values": [" ", " ", None, "a b"], "format": "General"}, + ), + ("padded_text", "leading and trailing spaces kept as published", [" lead", "trail ", " both ", "none"], None), + ("accented", "characters every single-byte encoding here can carry", ["Café", "naïve", "Ætna", "Müller £"], None), + ( + "date_text", + "dates as text: day-first, ambiguous, year-first as pandas re-saves them, and ISO", + ["13/04/2024", "04/01/2024", "2010-04-01 00:00:00", "2024-04-01"], + None, + ), + ( + "date_mixed", + "the Stockport case: date cells where Excel could read a date, text where it could not", + ["04/01/2024", "13/04/2024", "05/02/2024", "25/12/2024"], + {"values": [date(2024, 1, 4), "13/04/2024", date(2024, 2, 5), "25/12/2024"], "format": "dd/mm/yyyy"}, + ), + ( + "date_locale", + "a date cell in Excel's built-in short date format, whose display follows the machine's locale", + [None, None, None, None], + {"values": [date(2024, 1, 4), date(2024, 12, 25), datetime(2024, 4, 1, 9, 30), None], "format": "mm-dd-yy"}, + ), + ( + "boolean_like", + "boolean cells and boolean-looking text", + ["TRUE", "false", "Y", "FALSE"], + {"values": [True, "false", "Y", False], "format": "General"}, + ), ] ROWS = 4 -ENCODINGS = [("utf-8", "utf8"), ("utf-8-sig", "utf8-bom"), ("utf-16", "utf16-bom"), ("cp1252", "windows-1252"), - ("latin-1", "latin-1"), ("mac_roman", "mac-roman")] +ENCODINGS = [ + ("utf-8", "utf8"), + ("utf-8-sig", "utf8-bom"), + ("utf-16", "utf16-bom"), + ("cp1252", "windows-1252"), + ("latin-1", "latin-1"), + ("mac_roman", "mac-roman"), +] UNICODE_ONLY = [ ("zero_width", "a zero-width space and a byte-order mark inside text", ["abc​", "x"]), ("control", "a bell character and a tab inside a quoted field", ["x\u0007y", "a\tb"]), @@ -71,14 +116,19 @@ def field(value): if any(c in value for c in ',"\n\r\t') or value != value.strip(): return '"' + value.replace('"', '""') + '"' return value + return eol.join(",".join(field(v) for v in row) for row in [header, *rows]) + eol def main(): assert not TARGET.exists(), f"{TARGET} exists; fixtures are never overwritten" TARGET.mkdir(parents=True) - key = {"generated": datetime.now().isoformat(timespec="seconds"), "columns": {n: d for n, d, _, _ in COLUMNS}, - "csv": {}, "workbook": {}} + key = { + "generated": datetime.now().isoformat(timespec="seconds"), + "columns": {n: d for n, d, _, _ in COLUMNS}, + "csv": {}, + "workbook": {}, + } # CSV: the text of every column, in each encoding, LF endings; one UTF-8 file with CRLF, one with both. csv_columns = [c for c in COLUMNS if c[2][0] is not None] @@ -92,12 +142,21 @@ def main(): key["csv"]["fixture-utf8-crlf.csv"] = {"encoding": "utf-8", "eol": "CRLF", "header": header, "rows": rows} mixed = csv_text(header, rows[:2]) + csv_text(header, rows[2:], "\r\n").split("\r\n", 1)[1] (TARGET / "fixture-utf8-mixed-eol.csv").write_bytes(mixed.encode("utf-8")) - key["csv"]["fixture-utf8-mixed-eol.csv"] = {"encoding": "utf-8", "eol": "LF then CRLF", "header": header, "rows": rows} + key["csv"]["fixture-utf8-mixed-eol.csv"] = { + "encoding": "utf-8", + "eol": "LF then CRLF", + "header": header, + "rows": rows, + } u_header = [c[0] for c in UNICODE_ONLY] u_rows = [[c[2][i] for c in UNICODE_ONLY] for i in range(2)] (TARGET / "fixture-unicode-utf8.csv").write_bytes(csv_text(u_header, u_rows).encode("utf-8")) - key["csv"]["fixture-unicode-utf8.csv"] = {"encoding": "utf-8", "header": u_header, "rows": u_rows, - "columns": {n: d for n, d, _ in UNICODE_ONLY}} + key["csv"]["fixture-unicode-utf8.csv"] = { + "encoding": "utf-8", + "header": u_header, + "rows": u_rows, + "columns": {n: d for n, d, _ in UNICODE_ONLY}, + } wb = Workbook() ws = wb.active @@ -110,26 +169,55 @@ def main(): c = ws.cell(row=i + 2, column=j, value=value) if cell: c.number_format = cell["format"] - cells[c.coordinate] = {"column": name, "stored_type": c.data_type, "stored": value, - "format": c.number_format, "text": text[i]} + cells[c.coordinate] = { + "column": name, + "stored_type": c.data_type, + "stored": value, + "format": c.number_format, + "text": text[i], + } err = ws.cell(row=ROWS + 2, column=1, value="#N/A") err.data_type = TYPE_ERROR - cells[err.coordinate] = {"column": "ref_text", "stored_type": "e", "stored": "#N/A", "format": "General", "text": "#N/A", - "note": "an error cell, below the data"} + cells[err.coordinate] = { + "column": "ref_text", + "stored_type": "e", + "stored": "#N/A", + "format": "General", + "text": "#N/A", + "note": "an error cell, below the data", + } f = ws.cell(row=ROWS + 2, column=6, value="=F2*2") - cells[f.coordinate] = {"column": "number", "stored_type": "f", "stored": "=F2*2", "format": "General", "text": "25", - "note": "a formula; written here without a cached value, which only Excel adds when it saves"} - key["workbook"]["Data"] = {"header_row": 1, "cells": cells, - "note": "date_locale's display depends on the locale (built-in format 14): en-GB 04/01/2024, en-US 1/4/2024"} + cells[f.coordinate] = { + "column": "number", + "stored_type": "f", + "stored": "=F2*2", + "format": "General", + "text": "25", + "note": "a formula; written here without a cached value, which only Excel adds when it saves", + } + key["workbook"]["Data"] = { + "header_row": 1, + "cells": cells, + "note": "date_locale's display depends on the locale (built-in format 14): en-GB 04/01/2024, en-US 1/4/2024", + } titled = wb.create_sheet("Titled") titled["A1"] = "Reliefs key" - for r, row in enumerate([["Code", "Description"], ["EPCH", "Empty Property Rate Charitable"], ["MAN", "Mandatory"]], start=3): + for r, row in enumerate( + [["Code", "Description"], ["EPCH", "Empty Property Rate Charitable"], ["MAN", "Mandatory"]], start=3 + ): for c, value in enumerate(row, start=1): titled.cell(row=r, column=c, value=value) - key["workbook"]["Titled"] = {"layout": "title in A1, row 2 blank, header in row 3, two data rows", - "rows": [["Reliefs key", None], [None, None], ["Code", "Description"], - ["EPCH", "Empty Property Rate Charitable"], ["MAN", "Mandatory"]]} + key["workbook"]["Titled"] = { + "layout": "title in A1, row 2 blank, header in row 3, two data rows", + "rows": [ + ["Reliefs key", None], + [None, None], + ["Code", "Description"], + ["EPCH", "Empty Property Rate Charitable"], + ["MAN", "Mandatory"], + ], + } hidden = wb.create_sheet("Hidden") hidden.sheet_state = "hidden" hidden["A1"], hidden["A2"] = "secret", "kept" @@ -139,7 +227,9 @@ def main(): save(wb, TARGET / "fixture.xlsx") key["workbook_file"] = "fixture.xlsx" - (TARGET / "answer-key.json").write_text(json.dumps(key, indent=2, ensure_ascii=False, default=str), encoding="utf-8") + (TARGET / "answer-key.json").write_text( + json.dumps(key, indent=2, ensure_ascii=False, default=str), encoding="utf-8" + ) print(sorted(p.name for p in TARGET.iterdir())) diff --git a/tests/data/builders/fixtures_typing.py b/tests/data/builders/fixtures_typing.py index fa1702f..6259214 100644 --- a/tests/data/builders/fixtures_typing.py +++ b/tests/data/builders/fixtures_typing.py @@ -13,7 +13,9 @@ from datetime import datetime from pathlib import Path -TARGET = Path(sys.argv[1]).resolve() if len(sys.argv) > 1 else Path(__file__).resolve().parent.parent / "synthetic-typing" +TARGET = ( + Path(sys.argv[1]).resolve() if len(sys.argv) > 1 else Path(__file__).resolve().parent.parent / "synthetic-typing" +) UK = {"decimal": ".", "grouping": ",", "negatives": ["-x", "(x)", "x-"], "percent": "number"} EU = {"decimal": ",", "grouping": ".", "negatives": ["-x", "(x)", "x-"], "percent": "number"} @@ -33,54 +35,146 @@ def r(text, reason): NUMBER = { - "UK": {"policy": UK, "cases": [ - v("£43.40", "43.40"), v("¬£43.40", "43.40", "mojibake: ¬ is Sm, £ is Sc, both drop"), v("£-826.80", "-826.80"), - v("-£826.80", "-826.80"), v("£1,234.50", "1234.50"), v("£.00", "0.00"), v("12.50", "12.50"), v(" 7", "7"), - v("1e3", "1000"), v("(826.80)", "-826.80"), v("826.80-", "-826.80"), v("1,234", "1234"), - v("0000130842", "130842"), v("5%", "5", "percent policy: number"), v("−45", "-45", "Unicode minus"), - v("12", "12", "full-width digits, NFKC"), - r("1.234,50", "separators out of order for this policy"), r("1,5", "grouping of fewer than three digits"), - r("826.80 CR", "letters"), r("N/A", "letters"), r("12k", "letters"), r("£10 - £20", "two numbers"), - r("Trans Prem: £0.01", "letters"), r("1.2.3", "two decimal points"), - *[b(t) for t in BLANKS]]}, - "EU": {"policy": EU, "cases": [ - v("1.234,50", "1234.50"), v("1,5", "1.5"), v("1.234", "1234"), v("€-12,30", "-12.30"), - r("1,234.50", "separators out of order for this policy"), *[b(t) for t in BLANKS[:2]]]}, + "UK": { + "policy": UK, + "cases": [ + v("£43.40", "43.40"), + v("¬£43.40", "43.40", "mojibake: ¬ is Sm, £ is Sc, both drop"), + v("£-826.80", "-826.80"), + v("-£826.80", "-826.80"), + v("£1,234.50", "1234.50"), + v("£.00", "0.00"), + v("12.50", "12.50"), + v(" 7", "7"), + v("1e3", "1000"), + v("(826.80)", "-826.80"), + v("826.80-", "-826.80"), + v("1,234", "1234"), + v("0000130842", "130842"), + v("5%", "5", "percent policy: number"), + v("−45", "-45", "Unicode minus"), + v("12", "12", "full-width digits, NFKC"), + r("1.234,50", "separators out of order for this policy"), + r("1,5", "grouping of fewer than three digits"), + r("826.80 CR", "letters"), + r("N/A", "letters"), + r("12k", "letters"), + r("£10 - £20", "two numbers"), + r("Trans Prem: £0.01", "letters"), + r("1.2.3", "two decimal points"), + *[b(t) for t in BLANKS], + ], + }, + "EU": { + "policy": EU, + "cases": [ + v("1.234,50", "1234.50"), + v("1,5", "1.5"), + v("1.234", "1234"), + v("€-12,30", "-12.30"), + r("1,234.50", "separators out of order for this policy"), + *[b(t) for t in BLANKS[:2]], + ], + }, +} +INTEGER = { + "UK": { + "policy": UK, + "cases": [ + v("12", "12"), + v("0000130842", "130842"), + v("1,234", "1234"), + v("-7", "-7"), + r("12.50", "a fraction is not an integer"), + r("1.234,50", "separators out of order for this policy"), + r("826.80-", "a fraction is not an integer"), + *[b(t) for t in BLANKS[:2]], + ], + } +} +BOOLEAN = { + "declared": { + "policy": {"true": ["true", "yes", "y", "1"], "false": ["false", "no", "n", "0"], "case": "insensitive"}, + "cases": [ + v("TRUE", True), + v("true", True), + v("Y", True), + v("1", True), + v("yes", True), + v("FALSE", False), + v("N", False), + v("0", False), + r("maybe", "not in the declared terms"), + *[b(t) for t in BLANKS], + ], + } } -INTEGER = {"UK": {"policy": UK, "cases": [ - v("12", "12"), v("0000130842", "130842"), v("1,234", "1234"), v("-7", "-7"), - r("12.50", "a fraction is not an integer"), r("1.234,50", "separators out of order for this policy"), - r("826.80-", "a fraction is not an integer"), *[b(t) for t in BLANKS[:2]]]}} -BOOLEAN = {"declared": {"policy": {"true": ["true", "yes", "y", "1"], "false": ["false", "no", "n", "0"], "case": "insensitive"}, - "cases": [v("TRUE", True), v("true", True), v("Y", True), v("1", True), v("yes", True), - v("FALSE", False), v("N", False), v("0", False), r("maybe", "not in the declared terms"), - *[b(t) for t in BLANKS]]}} DATE = { - "day_first": {"policy": {"order": "day_first"}, "cases": [ - v("13/04/2024", "2024-04-13"), v("04/01/2024", "2024-01-04"), v("2010-04-01 00:00:00", "2010-04-01"), - v("2024-04-01", "2024-04-01"), v("2024/04/01", "2024-04-01", "year-first in any separator is year, month, day"), - v("01.04.2010", "2010-04-01"), r("31/02/2024", "no such date"), r("45295", "an Excel serial as text"), - *[b(t) for t in BLANKS[:2]]]}, - "month_first": {"policy": {"order": "month_first"}, "cases": [ - v("04/01/2024", "2024-04-01"), v("2024/04/01", "2024-04-01"), r("13/04/2024", "no month 13; never read day-first")]}, - "month_first_cells": {"policy": {"order": "day_first", "date_cells": "swap_day_month"}, - "note": "the Stockport case: date cells were swapped by Excel, text cells are day-first as published", - "cases": [{"cell_type": "date", "stored": "2024-01-04", "outcome": "value", "value": "2024-04-01"}, - {"cell_type": "text", "text": "13/04/2024", "outcome": "value", "value": "2024-04-13"}]}, + "day_first": { + "policy": {"order": "day_first"}, + "cases": [ + v("13/04/2024", "2024-04-13"), + v("04/01/2024", "2024-01-04"), + v("2010-04-01 00:00:00", "2010-04-01"), + v("2024-04-01", "2024-04-01"), + v("2024/04/01", "2024-04-01", "year-first in any separator is year, month, day"), + v("01.04.2010", "2010-04-01"), + r("31/02/2024", "no such date"), + r("45295", "an Excel serial as text"), + *[b(t) for t in BLANKS[:2]], + ], + }, + "month_first": { + "policy": {"order": "month_first"}, + "cases": [ + v("04/01/2024", "2024-04-01"), + v("2024/04/01", "2024-04-01"), + r("13/04/2024", "no month 13; never read day-first"), + ], + }, + "month_first_cells": { + "policy": {"order": "day_first", "date_cells": "swap_day_month"}, + "note": "the Stockport case: date cells were swapped by Excel, text cells are day-first as published", + "cases": [ + {"cell_type": "date", "stored": "2024-01-04", "outcome": "value", "value": "2024-04-01"}, + {"cell_type": "text", "text": "13/04/2024", "outcome": "value", "value": "2024-04-13"}, + ], + }, +} +STRING = { + "default": { + "policy": {}, + "cases": [ + v(" a b ", " a b "), + v("a\tb", "a\tb"), + v("\\path", "\\path"), + v("'0012", "'0012"), + v("line\nbreak", "line\nbreak"), + v("a b", "a b"), + *[b(t) for t in BLANKS], + ], + } +} +CATEGORISE = { + "note": "CATEGORISE with a boolean source term: does a cell count as a value?", + "cases": [{"text": t, "is_value": False} for t in BLANKS] + + [{"text": t, "is_value": True} for t in ["0", "N/A", "x", "a b"]], } -STRING = {"default": {"policy": {}, "cases": [ - v(" a b ", " a b "), v("a\tb", "a\tb"), v("\\path", "\\path"), v("'0012", "'0012"), v("line\nbreak", "line\nbreak"), - v("a b", "a b"), *[b(t) for t in BLANKS]]}} -CATEGORISE = {"note": "CATEGORISE with a boolean source term: does a cell count as a value?", - "cases": [{"text": t, "is_value": False} for t in BLANKS] + - [{"text": t, "is_value": True} for t in ["0", "N/A", "x", "a b"]]} def main(): assert not TARGET.exists(), f"{TARGET} exists; fixtures are never overwritten" TARGET.mkdir(parents=True) - key = {"generated": datetime.now().isoformat(timespec="seconds"), "design": "proposed typing", - "number": NUMBER, "integer": INTEGER, "boolean": BOOLEAN, "date": DATE, "string": STRING, "categorise": CATEGORISE} + key = { + "generated": datetime.now().isoformat(timespec="seconds"), + "design": "proposed typing", + "number": NUMBER, + "integer": INTEGER, + "boolean": BOOLEAN, + "date": DATE, + "string": STRING, + "categorise": CATEGORISE, + } (TARGET / "typing-key.json").write_text(json.dumps(key, indent=2, ensure_ascii=False), encoding="utf-8") count = sum(len(p["cases"]) for t in (NUMBER, INTEGER, BOOLEAN, DATE, STRING) for p in t.values()) print(count, "typing cases and", len(CATEGORISE["cases"]), "CATEGORISE cases written") diff --git a/tests/data/builders/layouts.py b/tests/data/builders/layouts.py index fa5d99c..4a77f83 100644 --- a/tests/data/builders/layouts.py +++ b/tests/data/builders/layouts.py @@ -35,25 +35,59 @@ # Start Date as text; in a workbook the first and third are date cells, as Excel leaves a day-first column it could # partly read (the Stockport case). Contact carries N/A, a lone space, an empty cell, and padding. RECORDS = [ - ["NN00101700004185", "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", "SBR3", "13250", - "04/01/2024", "N/A"], + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A", + ], ["NN03905740027005", "27-31, Meadow Road, Newbury, RG14 7AH", "SBRE", "10250", "13/04/2024", " "], - ["NN04105100003115", "At Station Road, Pangbourne, Reading, RG8 7AN", "RH0", "8600", "05/02/2024", - "Parish Clerk"], - ["NN0010170000401A", "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", "SBR1", "16250", - "25/12/2024", ""], - ["NN0010420000111A", "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", "MAN", - "5400", "01/02/2026", "N/A"], + ["NN04105100003115", "At Station Road, Pangbourne, Reading, RG8 7AN", "RH0", "8600", "05/02/2024", "Parish Clerk"], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "", + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A", + ], ["NN00201200001117", "The Bell Inn, Aldworth, Reading, RG8 9SE", "PU15", "24000", "30/01/2026", " Landlord "], ] DATE_CELLS = {0: date(2024, 1, 4), 2: date(2024, 2, 5)} -CODES = [["Code", "Description"], ["SBR3", "Small Business Rate Relief"], ["RH0", "Retail, Hospitality and Leisure"], - ["MAN", "Mandatory Charity Relief"]] +CODES = [ + ["Code", "Description"], + ["SBR3", "Small Business Rate Relief"], + ["RH0", "Retail, Hospitality and Leisure"], + ["MAN", "Mandatory Charity Relief"], +] FOOTNOTES = [["Source: business rates system, extracted 15/04/2026"], ["* R/V is the rateable value in pounds"]] # Each a name a script must still resolve to one field: quotes, brackets, comparison signs, a comma, padding, a line # break, an empty name, a repeat, one name inside another, and the row key's own name. -HOSTILE = ["Owner's name", 'The "Main" Rate', "Rate [2026]", "R/V > 0", "", "Code, Local", " padded ", - "line\nbreak", "", "Rate", "Rate", "Rate Relief", "__row__"] +HOSTILE = [ + "Owner's name", + 'The "Main" Rate', + "Rate [2026]", + "R/V > 0", + "", + "Code, Local", + " padded ", + "line\nbreak", + "", + "Rate", + "Rate", + "Rate Relief", + "__row__", +] MISSING_LIKE = ["N/A", "NA", "null", "NaN", "#N/A", "n/a", "-", "None"] @@ -82,66 +116,124 @@ def csv_cases(): cases = {} lines = [["Business rates reliefs, April 2026"], [], [], HEADER, *RECORDS] - cases["title-above.csv"] = (lines, None, { - "problem": "a title and two blank lines above the header", - "dialect": {"skipRows": 3}, "names": HEADER, "rows": RECORDS, - **expect({"blank-rows": {"rows": [2, 3]}})}) + cases["title-above.csv"] = ( + lines, + None, + { + "problem": "a title and two blank lines above the header", + "dialect": {"skipRows": 3}, + "names": HEADER, + "rows": RECORDS, + **expect({"blank-rows": {"rows": [2, 3]}}), + }, + ) lines = [HEADER, *RECORDS[:3], [], *RECORDS[3:]] - cases["blank-line-inside.csv"] = (lines, None, { - "problem": "a blank line inside the data, kept as a row and numbered", - "dialect": {}, "names": HEADER, "rows": padded([*RECORDS[:3], [], *RECORDS[3:]], width), - **expect({"blank-rows": {"rows": [5]}})}) + cases["blank-line-inside.csv"] = ( + lines, + None, + { + "problem": "a blank line inside the data, kept as a row and numbered", + "dialect": {}, + "names": HEADER, + "rows": padded([*RECORDS[:3], [], *RECORDS[3:]], width), + **expect({"blank-rows": {"rows": [5]}}), + }, + ) records = [r.copy() for r in RECORDS] records[1][1] = "27-31, Meadow Road\nNewbury, RG14 7AH" lines = [HEADER, *records] - cases["quoted-line-break.csv"] = (lines, None, { - "problem": "a quoted cell holding a line break, one record over two lines of the file", - "dialect": {}, "names": HEADER, "rows": records, **expect()}) + cases["quoted-line-break.csv"] = ( + lines, + None, + { + "problem": "a quoted cell holding a line break, one record over two lines of the file", + "dialect": {}, + "names": HEADER, + "rows": records, + **expect(), + }, + ) lines = [HEADER, *RECORDS] - cases["semicolon.csv"] = (lines, csv_text(lines, delimiter=";"), { - "problem": "`;` between cells, and commas inside the addresses unquoted", - "dialect": {"delimiter": ";"}, "names": HEADER, "rows": RECORDS, - **expect(proposals={"delimiter": ";"})}) + cases["semicolon.csv"] = ( + lines, + csv_text(lines, delimiter=";"), + { + "problem": "`;` between cells, and commas inside the addresses unquoted", + "dialect": {"delimiter": ";"}, + "names": HEADER, + "rows": RECORDS, + **expect(proposals={"delimiter": ";"}), + }, + ) # No cell is quoted, so the addresses lose their commas; one cell opens with a quote that never closes records = [[v.replace(",", "") for v in r] for r in RECORDS] records[2][1] = '"Old Mill Station Road Pangbourne' lines = [HEADER, *records] text = "\r\n".join(",".join(line) for line in lines) + "\r\n" - cases["stray-quote.csv"] = (lines, text, { - "problem": "a cell opening with a quote that never closes, which swallows the rest of the file if read as a " - "quote (1.x needed `quoting=csv.QUOTE_NONE`)", - "dialect": {"quoteChar": None}, "names": HEADER, "rows": records, - **expect({"csv-quote": {"rows": [4], "setting": "quoteChar"}}, {"quoteChar": None})}) + cases["stray-quote.csv"] = ( + lines, + text, + { + "problem": "a cell opening with a quote that never closes, which swallows the rest of the file if read as a " + "quote (1.x needed `quoting=csv.QUOTE_NONE`)", + "dialect": {"quoteChar": None}, + "names": HEADER, + "rows": records, + **expect({"csv-quote": {"rows": [4], "setting": "quoteChar"}}, {"quoteChar": None}), + }, + ) lines = [[*line, ""] for line in [HEADER, *RECORDS]] - cases["trailing-delimiter.csv"] = (lines, None, { - "problem": "every line ends with a delimiter, an empty last column with an empty header", - "dialect": {}, "names": HEADER, "rows": RECORDS, - **expect()}) + cases["trailing-delimiter.csv"] = ( + lines, + None, + { + "problem": "every line ends with a delimiter, an empty last column with an empty header", + "dialect": {}, + "names": HEADER, + "rows": RECORDS, + **expect(), + }, + ) # Each record over two lines of the file, as every line has the same fields: the second holds one value first = HEADER[:5] lines = [first, [HEADER[5], "", "", "", ""]] for r in RECORDS: lines += [r[:5], [r[5], "", "", "", ""]] - cases["two-line-records.csv"] = (lines, None, { - "problem": "each record, header included, over two lines: five fields, then one in the first field", - "dialect": {"rowsPerRecord": [5, 1]}, "names": HEADER, "rows": RECORDS, - **expect({"blank-rows": {"rows": [10]}}, {"rowsPerRecord": [5, 1]}), - "refuse": "a record without its second line, or a second line with values past its first field"}) + cases["two-line-records.csv"] = ( + lines, + None, + { + "problem": "each record, header included, over two lines: five fields, then one in the first field", + "dialect": {"rowsPerRecord": [5, 1]}, + "names": HEADER, + "rows": RECORDS, + **expect({"blank-rows": {"rows": [10]}}, {"rowsPerRecord": [5, 1]}), + "refuse": "a record without its second line, or a second line with values past its first field", + }, + ) # The same records with nothing after a record's one value: a lone empty value is a blank line lines = [first, [HEADER[5]]] for r in RECORDS: lines += [r[:5], [r[5]]] - cases["two-line-records-unpadded.csv"] = (lines, None, { - "problem": "each record over two lines, the second holding its one value and no delimiters, so an empty " - "value is a blank line", "dialect": {"rowsPerRecord": [5, 1]}, "names": HEADER, "rows": RECORDS, - **expect({"blank-rows": {"rows": [10]}}, {"rowsPerRecord": [5, 1]})}) + cases["two-line-records-unpadded.csv"] = ( + lines, + None, + { + "problem": "each record over two lines, the second holding its one value and no delimiters, so an empty " + "value is a blank line", + "dialect": {"rowsPerRecord": [5, 1]}, + "names": HEADER, + "rows": RECORDS, + **expect({"blank-rows": {"rows": [10]}}, {"rowsPerRecord": [5, 1]}), + }, + ) records = [r.copy() for r in RECORDS] records[2][1] = 'The "Old" Mill, Station Road, Pangbourne, Reading, RG8 7AN' @@ -153,48 +245,103 @@ def escaped(value): return value text = "".join(",".join(escaped(v) for v in line) + "\r\n" for line in lines) - cases["backslash-escape.csv"] = (lines, text, { - "problem": "a quote inside a quoted cell escaped by a backslash, not doubled", - "dialect": {"doubleQuote": False}, "names": HEADER, "rows": records, - **expect({"csv-quote": {"rows": [4], "setting": "doubleQuote"}}, {"doubleQuote": False})}) + cases["backslash-escape.csv"] = ( + lines, + text, + { + "problem": "a quote inside a quoted cell escaped by a backslash, not doubled", + "dialect": {"doubleQuote": False}, + "names": HEADER, + "rows": records, + **expect({"csv-quote": {"rows": [4], "setting": "doubleQuote"}}, {"doubleQuote": False}), + }, + ) lines = [HEADER, *RECORDS] - cases["cr-line-ends.csv"] = (lines, csv_text(lines, eol="\r"), { - "problem": "every line ending `\\r` alone, as old Mac files do", "dialect": {}, "names": HEADER, - "rows": RECORDS, **expect()}) + cases["cr-line-ends.csv"] = ( + lines, + csv_text(lines, eol="\r"), + { + "problem": "every line ending `\\r` alone, as old Mac files do", + "dialect": {}, + "names": HEADER, + "rows": RECORDS, + **expect(), + }, + ) lines = [HEADER[:5], *RECORDS] - cases["header-narrower.csv"] = (lines, None, { - "problem": "the header a field narrower than the data, so the last column has no name", - "dialect": {}, "names": [*HEADER[:5], "Unnamed: 5"], "rows": RECORDS, - "renamed": "the column past the header by position, as an empty name is", **expect()}) + cases["header-narrower.csv"] = ( + lines, + None, + { + "problem": "the header a field narrower than the data, so the last column has no name", + "dialect": {}, + "names": [*HEADER[:5], "Unnamed: 5"], + "rows": RECORDS, + "renamed": "the column past the header by position, as an empty name is", + **expect(), + }, + ) lines = [RECORDS[0][:5], *RECORDS[1:]] - cases["first-row-narrower.csv"] = (lines, None, { - "problem": "no header, and the first row a field narrower than the rest", - "dialect": {"headerRowCount": 0}, "names": [f"column_{i}" for i in range(width)], - "rows": padded(lines, width), **expect()}) + cases["first-row-narrower.csv"] = ( + lines, + None, + { + "problem": "no header, and the first row a field narrower than the rest", + "dialect": {"headerRowCount": 0}, + "names": [f"column_{i}" for i in range(width)], + "rows": padded(lines, width), + **expect(), + }, + ) lines = [HEADER, *RECORDS[:2], RECORDS[2][:4], *RECORDS[3:]] - cases["row-narrower.csv"] = (lines, None, { - "problem": "one row two fields narrower than the header, its last cells empty", - "dialect": {}, "names": HEADER, "rows": padded(lines[1:], width), **expect()}) + cases["row-narrower.csv"] = ( + lines, + None, + { + "problem": "one row two fields narrower than the header, its last cells empty", + "dialect": {}, + "names": HEADER, + "rows": padded(lines[1:], width), + **expect(), + }, + ) rows = [[f"v{i}{j}" for j in range(len(HOSTILE))] for i in range(2)] lines = [HOSTILE, *rows] - cases["hostile-headers.csv"] = (lines, None, { - "problem": "header names a script must still resolve: every name read exactly, none trimmed or joined", - "dialect": {}, "names_as_written": HOSTILE, "rows": rows, - "names": [f"Unnamed: {i}" if n == "" else n for i, n in enumerate(HOSTILE[:-3])] + ["Rate.1", "Rate Relief", - "__row__.1"], - "renamed": "the empty name by position, as stored schemas name it; the repeat and the row key's own name " - "by RowKeyParser.deconflict", **expect()}) + cases["hostile-headers.csv"] = ( + lines, + None, + { + "problem": "header names a script must still resolve: every name read exactly, none trimmed or joined", + "dialect": {}, + "names_as_written": HOSTILE, + "rows": rows, + "names": [f"Unnamed: {i}" if n == "" else n for i, n in enumerate(HOSTILE[:-3])] + + ["Rate.1", "Rate Relief", "__row__.1"], + "renamed": "the empty name by position, as stored schemas name it; the repeat and the row key's own name " + "by RowKeyParser.deconflict", + **expect(), + }, + ) rows = [[v, str(i)] for i, v in enumerate(MISSING_LIKE)] lines = [["Contact", "n"], *rows] - cases["missing-like-text.csv"] = (lines, None, { - "problem": "text pandas reads as missing by default; every one is text, and only the source schema can " - "say it means missing", "dialect": {}, "names": ["Contact", "n"], "rows": rows, **expect()}) + cases["missing-like-text.csv"] = ( + lines, + None, + { + "problem": "text pandas reads as missing by default; every one is text, and only the source schema can " + "say it means missing", + "dialect": {}, + "names": ["Contact", "n"], + "rows": rows, + **expect(), + }, + ) return cases @@ -237,9 +384,14 @@ def layouts_workbook(): ws.title = "Titled" ws["A1"], ws["A2"] = "West Berkshire Council", "Business rates reliefs, April 2026" dates = write_records(ws, top=4) - key["Titled"] = {"problem": "a title and subtitle, then a blank row, above the header on row 4", - "dialect": {"skipRows": 3}, "names": HEADER, "rows": workbook_rows(), "date_cells": dates, - **expect({"blank-rows": {"rows": [3]}})} + key["Titled"] = { + "problem": "a title and subtitle, then a blank row, above the header on row 4", + "dialect": {"skipRows": 3}, + "names": HEADER, + "rows": workbook_rows(), + "date_cells": dates, + **expect({"blank-rows": {"rows": [3]}}), + } ws = wb.create_sheet("MergedHeader") ws["A1"], ws["B1"], ws["D1"] = "Propref", "Property", "Relief" @@ -254,13 +406,15 @@ def layouts_workbook(): ws.cell(row=i, column=c, value=value) key["MergedHeader"] = { "problem": "a header over two rows: one name merged down both, two merged across two columns each", - "dialect": {"headerRowCount": 2}, "header_rows": [["Propref", "Property", None, "Relief", None], - [None, "Address", "Postcode", "Code", "R/V"]], - "merged": ["A1:A2", "B1:C1", "D1:E1"], "rows": records, + "dialect": {"headerRowCount": 2}, + "header_rows": [["Propref", "Property", None, "Relief", None], [None, "Address", "Postcode", "Code", "R/V"]], + "merged": ["A1:A2", "B1:C1", "D1:E1"], + "rows": records, "names": "supplied by the wrangler, one per column: a merged cell over two may be one title or a heading " - "over subheadings, so whyqd never builds a name from it", + "over subheadings, so whyqd never builds a name from it", "refuse": "headerRowCount 2 without names: the header rows and merged ranges named, and names asked for", - **expect({"merged-range": {"cells": ["A1:A2", "B1:C1", "D1:E1"]}})} + **expect({"merged-range": {"cells": ["A1:A2", "B1:C1", "D1:E1"]}}), + } ws = wb.create_sheet("BlankColumns") names = HEADER[:2] + [None] + HEADER[2:] @@ -275,9 +429,11 @@ def layouts_workbook(): ws.cell(row=i, column=c).border = border key["BlankColumns"] = { "problem": "an empty column inside the table, and three empty but formatted columns after it", - "dialect": {}, "names": [*HEADER[:2], "Unnamed: 2", *HEADER[2:]], + "dialect": {}, + "names": [*HEADER[:2], "Unnamed: 2", *HEADER[2:]], "rows": [[v if v != "" else None for v in r[:2] + [None] + r[2:]] for r in RECORDS], - **expect({"blank-columns": {"columns": [2]}})} + **expect({"blank-columns": {"columns": [2]}}), + } ws = wb.create_sheet("SubTables") write_records(ws, top=1, dates=False) @@ -290,8 +446,11 @@ def layouts_workbook(): rows = [*RECORDS, [""] * 6, *padded(CODES, 6), [""] * 6, *padded(FOOTNOTES, 6)] key["SubTables"] = { "problem": "a second table below a blank row, then footnotes; loaded as given, every row is in the table", - "dialect": {}, "names": HEADER, "rows": [[v if v != "" else None for v in r] for r in rows], - **expect({"blank-rows": {"rows": [len(RECORDS) + 2, len(RECORDS) + len(CODES) + 3]}})} + "dialect": {}, + "names": HEADER, + "rows": [[v if v != "" else None for v in r] for r in rows], + **expect({"blank-rows": {"rows": [len(RECORDS) + 2, len(RECORDS) + len(CODES) + 3]}}), + } ws = wb.create_sheet("Errors") write_records(ws, top=1, dates=False) @@ -302,17 +461,30 @@ def layouts_workbook(): rows = [[v if v != "" else None for v in r] for r in RECORDS] for coordinate, value in errors.items(): rows[int(coordinate[1:]) - 2][3] = value - key["Errors"] = {"problem": "error cells in a number column", "dialect": {}, "names": HEADER, "rows": rows, - "error_cells": errors, **expect({"error-cells": {"cells": list(errors)}})} + key["Errors"] = { + "problem": "error cells in a number column", + "dialect": {}, + "names": HEADER, + "rows": rows, + "error_cells": errors, + **expect({"error-cells": {"cells": list(errors)}}), + } ws = wb.create_sheet("Workings") ws.sheet_state = "hidden" ws["A1"], ws["A2"] = "Total R/V", "=SUM(Titled!D5:D10)" - key["Workings"] = {"problem": "a hidden sheet, read like any other and flagged", "state": "hidden", "dialect": {}, - "names": ["Total R/V"], "rows": [], - "formula_cells": {"A2": "a formula written without its cached value, so the row is empty " - "here and trimmed; the .xls, saved by LibreOffice, caches 0"}, - **expect({"hidden-sheet": {}})} + key["Workings"] = { + "problem": "a hidden sheet, read like any other and flagged", + "state": "hidden", + "dialect": {}, + "names": ["Total R/V"], + "rows": [], + "formula_cells": { + "A2": "a formula written without its cached value, so the row is empty " + "here and trimmed; the .xls, saved by LibreOffice, caches 0" + }, + **expect({"hidden-sheet": {}}), + } return wb, key @@ -325,8 +497,13 @@ def single_sheet_workbook(): for r, row in enumerate(CODES[1:], start=3): for c, value in enumerate(row, start=1): ws.cell(row=r, column=c, value=value) - key = {"problem": "the only sheet, with row 2 blank: 1.x's one-row probe judged it empty and dropped it", - "dialect": {}, "names": CODES[0], "rows": [[None, None], *CODES[1:]], **expect({"blank-rows": {"rows": [2]}})} + key = { + "problem": "the only sheet, with row 2 blank: 1.x's one-row probe judged it empty and dropped it", + "dialect": {}, + "names": CODES[0], + "rows": [[None, None], *CODES[1:]], + **expect({"blank-rows": {"rows": [2]}}), + } return wb, key @@ -366,12 +543,15 @@ def rule(ws): ws.column_dimensions["B"].width = 30 for row in ws.iter_rows(min_col=2, max_col=2): row[0].alignment = Alignment(wrap_text=True) - key["Ruled"] = {"problem": "a ruled grid, long addresses wrapped over several lines in their cells", - "names": HEADER, "rows": printed(RECORDS), - "note": "a date cell exports as its displayed text, dd/mm/yyyy, so the rows are the text", - "wrapped": "an address extracts with a line break at each wrap, and keeps it: a wrap and a line " - "break in the cell look the same in a PDF, so joining is a declared step, never the " - "reader's; the ruling keeps each wrapped cell in its one row"} + key["Ruled"] = { + "problem": "a ruled grid, long addresses wrapped over several lines in their cells", + "names": HEADER, + "rows": printed(RECORDS), + "note": "a date cell exports as its displayed text, dd/mm/yyyy, so the rows are the text", + "wrapped": "an address extracts with a line break at each wrap, and keeps it: a wrap and a line " + "break in the cell look the same in a PDF, so joining is a declared step, never the " + "reader's; the ruling keeps each wrapped cell in its one row", + } ws = wb.create_sheet("MergedHeader") ws["A1"], ws["B1"], ws["D1"] = "Propref", "Property", "Relief" @@ -385,10 +565,12 @@ def rule(ws): for c, value in enumerate(record, start=1): ws.cell(row=i, column=c, value=value) rule(ws) - key["MergedHeader"] = {"problem": "a ruled header over two rows with merged cells", - "header_rows": [["Propref", "Property", "", "Relief", ""], - ["", "Address", "Postcode", "Code", "R/V"]], "rows": records, - "names": "supplied by the wrangler, as layouts.xlsx § MergedHeader"} + key["MergedHeader"] = { + "problem": "a ruled header over two rows with merged cells", + "header_rows": [["Propref", "Property", "", "Relief", ""], ["", "Address", "Postcode", "Code", "R/V"]], + "rows": records, + "names": "supplied by the wrangler, as layouts.xlsx § MergedHeader", + } ws = wb.create_sheet("TwoTables") write_records(ws, top=1, dates=False) @@ -396,36 +578,40 @@ def rule(ws): for c, value in enumerate(row, start=1): ws.cell(row=r, column=c, value=value) fit(ws) - key["TwoTables"] = {"problem": "two tables on one page, unruled, two blank rows apart", - "tables": [{"names": HEADER, "rows": printed(RECORDS)}, - {"names": CODES[0], "rows": CODES[1:]}]} + key["TwoTables"] = { + "problem": "two tables on one page, unruled, two blank rows apart", + "tables": [{"names": HEADER, "rows": printed(RECORDS)}, {"names": CODES[0], "rows": CODES[1:]}], + } ws = wb.create_sheet("Landscape") wide = [*HEADER, "Liable From", "Account Holder", "Ward", "Band"] - extra = [["01/04/2010", "Orpheus Holdings Ltd", "Aldermaston", "B"], - ["01/04/2017", "Meadow Road Traders", "Newbury Central", "C"], - ["12/06/2019", "Pangbourne Station Ltd", "Pangbourne", "A"], - ["01/04/2023", "Andromeda Estates", "Aldermaston", "B"], - ["01/04/2005", "Aldermaston Parish Council", "Aldermaston", "A"], - ["30/09/2016", "The Bell Inn (Aldworth) Ltd", "Basildon", "B"]] + extra = [ + ["01/04/2010", "Orpheus Holdings Ltd", "Aldermaston", "B"], + ["01/04/2017", "Meadow Road Traders", "Newbury Central", "C"], + ["12/06/2019", "Pangbourne Station Ltd", "Pangbourne", "A"], + ["01/04/2023", "Andromeda Estates", "Aldermaston", "B"], + ["01/04/2005", "Aldermaston Parish Council", "Aldermaston", "A"], + ["30/09/2016", "The Bell Inn (Aldworth) Ltd", "Basildon", "B"], + ] wide_rows = [r + e for r, e in zip(RECORDS, extra, strict=True)] write_records(ws, top=1, records=wide_rows, header=wide, dates=False) ws.page_setup.orientation = "landscape" rule(ws) - key["Landscape"] = {"problem": "a wide table on a landscape page", "names": wide, - "rows": printed(wide_rows)} + key["Landscape"] = {"problem": "a wide table on a landscape page", "names": wide, "rows": printed(wide_rows)} - many = [[f"NN{900000000000 + i:014d}", r[1], r[2], r[3], r[4], r[5]] - for i, r in enumerate(RECORDS * 25)] + many = [[f"NN{900000000000 + i:014d}", r[1], r[2], r[3], r[4], r[5]] for i, r in enumerate(RECORDS * 25)] for title, repeat in (("RepeatedHeader", True), ("HeaderOnce", False)): ws = wb.create_sheet(title) write_records(ws, top=1, records=many, dates=False) if repeat: ws.print_title_rows = "1:1" rule(ws) - key[title] = {"problem": f"{len(many)} rows over several pages, the header " - + ("repeated on every page" if repeat else "on the first page only"), - "names": HEADER, "rows": printed(many)} + key[title] = { + "problem": f"{len(many)} rows over several pages, the header " + + ("repeated on every page" if repeat else "on the first page only"), + "names": HEADER, + "rows": printed(many), + } ws = wb.create_sheet("UnruledWrapped") write_records(ws, top=1) @@ -435,12 +621,14 @@ def rule(ws): row[0].alignment = Alignment(wrap_text=True) key["UnruledWrapped"] = { "problem": "no ruling, and long addresses wrapped in their cells: read by text position, each wrapped line " - "becomes a row of its own and words split across columns", - "names": HEADER, "rows": printed(RECORDS), + "becomes a row of its own and words split across columns", + "names": HEADER, + "rows": printed(RECORDS), "report": "loaded by the default reading, the rows that look like continuations (an empty first cell, text " - "only where a cell wraps) are shown, so the wrangler can declare how the table is read", + "only where a cell wraps) are shown, so the wrangler can declare how the table is read", "declared": "with the table's reading declared (its columns' bounds, which rows continue a record), the rows " - "above, each wrap a line break in its cell"} + "above, each wrap a line break in its cell", + } return wb, key @@ -462,12 +650,15 @@ def main(): key["workbook"]["single-sheet.xlsx"] = {"Reliefs": sheet} wb, sheets = pdf_workbook() save(wb, TARGET / "pdf-source.xlsx") - key["pdf"] = {"pdf-source.pdf": sheets, - "made": "pdf-source.xlsx exported in LibreOffice, every sheet, one PDF; each sheet's table " - "read across its pages"} - - (TARGET / "answer-key.json").write_text(json.dumps(key, indent=2, ensure_ascii=False, default=str), - encoding="utf-8") + key["pdf"] = { + "pdf-source.pdf": sheets, + "made": "pdf-source.xlsx exported in LibreOffice, every sheet, one PDF; each sheet's table " + "read across its pages", + } + + (TARGET / "answer-key.json").write_text( + json.dumps(key, indent=2, ensure_ascii=False, default=str), encoding="utf-8" + ) print(sorted(p.name for p in TARGET.iterdir())) diff --git a/tests/data/sample/voa/uk-englandwales-ndr-2010-listentries-compiled-epoch-0038-changeupdate-0001-csv.csv b/tests/data/sample/voa/uk-englandwales-ndr-2010-listentries-compiled-epoch-0038-changeupdate-0001-csv.csv new file mode 100644 index 0000000..f17b170 --- /dev/null +++ b/tests/data/sample/voa/uk-englandwales-ndr-2010-listentries-compiled-epoch-0038-changeupdate-0001-csv.csv @@ -0,0 +1,5 @@ +0660**N20056580000110*9103855000*BOOTHS SUPERMARKET, STANLEY ROAD, KNUTSFORD, CHESHIRE*WA16 0BS*CS9*SUPERSTORE AND PREMISES*760000*26-JUN-2013**25232493539*IPP**20*ALTERATION OF A NON DELETED ASSESSMENT*A*139S*2*27260837000*11-APR-2023*****BOOTHS SUPERMARKET*STANLEY ROAD**KNUTSFORD*CHESHIRE**11-APR-2023** +1445**046653*8791921000*25-26, ST JAMESS STREET, BRIGHTON*BN2 1RF*CS*SHOP AND PREMISES*35500*03-FEB-2014**24873740537*IPP**20*ALTERATION OF A NON DELETED ASSESSMENT*A*249G*4*27178750000*05-APR-2023*****25-26*ST JAMESS STREET**BRIGHTON***** +1445**046653*8791921000*25-26, ST JAMESS STREET, BRIGHTON*BN2 1RF*CS*SHOP AND PREMISES*37250*01-OCT-2015**36831856285*VOR**20*ALTERATION OF A NON DELETED ASSESSMENT**249G*4*27261682000*11-APR-2023*****25-26*ST JAMESS STREET**BRIGHTON***** +4225**50394699171651*4606914000*SOMERFIELD, DALE STREET, MILNROW, ROCHDALE, LANCS*OL16 4HS*CS*SHOP AND PREMISES*129000*02-MAR-2016**27218435539*IPP**20*ALTERATION OF A NON DELETED ASSESSMENT*A*152G*4*27260921000*11-APR-2023*****SOMERFIELD*DALE STREET*MILNROW*ROCHDALE*LANCS**** +4215**2744118*2602395000*SLUG & LETTUCE PT 2ND LEVEL HERON HOUSE 11-12, ALBERT SQUARE, MANCHESTER*M2 5HD*CL*PUBLIC HOUSE AND PREMISES*94500*01-APR-2015**36828015280*VOR**20*ALTERATION OF A NON DELETED ASSESSMENT**226G*2*27260177000*05-APR-2023*****SLUG & LETTUCE PT 2ND LEVEL HERON HOUSE 11-12*ALBERT SQUARE**MANCHESTER***05-APR-2023** diff --git a/tests/data/sample/voa/uk-englandwales-ndr-2010-summaryvaluations-compiled-epoch-0038-changeupdate-0001-csv.csv b/tests/data/sample/voa/uk-englandwales-ndr-2010-summaryvaluations-compiled-epoch-0038-changeupdate-0001-csv.csv new file mode 100644 index 0000000..b04db91 --- /dev/null +++ b/tests/data/sample/voa/uk-englandwales-ndr-2010-summaryvaluations-compiled-epoch-0038-changeupdate-0001-csv.csv @@ -0,0 +1,34 @@ +01*11736752000*5853519000*3420*The Occupier*UNIT 712***CENTRE 500*LOWFIELD DRIVE*WOLSTANTON*NEWCASTLE*STAFFS*ST5 0UU*81083*Workshop And Premises*352.70*19603*19603*19500*2010*Newcastle Under Lyme*01061300727009*13606080226*01-APR-2010**096*GIA*57.00 +02*1*Ground*Warehouse*352.70*55.58*19603 +01*9341856000*438200221*4605*The Occupier*1281****STRATFORD ROAD**BIRMINGHAM**B28 9AJ*80766*Shop And Premises*712.38*19733*20333*20000*2010*Birmingham*2969586*11984613221*01-APR-2010**249*NIA*145.00 +02*1*Ground*Warehouse*148.88*14.50*2159 +02*2*Ground*Office*15.16*14.50*220 +02*3*Ground*Internal Storage*94.97*14.50*1377 +02*4*First*Internal Storage*171.48*7.25*1243 +02*5*First*Internal Storage*23.33*7.25*169 +02*6*First*Kitchen*13.39*7.25*97 +02*7*First*Office*11.29*7.25*82 +02*8*First*Office*11.72*7.25*85 +02*9*First*Internal Storage*11.07*7.25*80 +02*10*First*Office*11.07*7.25*80 +02*11*Second*Internal Storage*44.42*5.80*258 +02*12*Ground*Retail Zone A*56.42*145.00*8181 +02*13*Ground*Retail Zone B*58.09*72.50*4212 +02*14*Ground*Retail Zone C*41.09*36.25*1490 +05*6*600**0*+600 +01*27260837000*9103855000*0660*The Occupier*BOOTHS SUPERMARKET****STANLEY ROAD**KNUTSFORD*CHESHIRE*WA16 0BS*114463*Superstore And Premises*5122.60*760394*760394*760000*2010*Cheshire East*N20056580000110*25232493539*11-APR-2023**139*GIA*189.00 +02*1*Basement*All Main Areas*2198.70*94.50*207777 +02*2*Ground*All Main Areas*2560.10*189.00*483859 +02*3*First*All Main Areas*363.80*189.00*68758 +01*27178750000*8791921000*1445**25-26****ST JAMESS STREET**BRIGHTON**BN2 1RF*114551*Shop And Premises*366.10*35525*35525*35500*2010*Brighton & Hove*046653*24873740537***249*NIA*275.00 +02*1*Ground*Retail Zone A*66.30*261.25*17321 +02*2*Ground*Retail Zone B*66.30*130.63*8661 +02*3*Ground*Retail Zone C*58.70*65.31*3834 +02*4*Ground*Remaining Retail Zone*174.80*32.66*5709 +01*27261682000*8791921000*1445**25-26****ST JAMESS STREET**BRIGHTON**BN2 1RF*114551*Shop And Premises*366.10*37395*37395*37250*2010*Brighton & Hove*046653*36831856285***249*NIA*275.00 +02*1*Ground*Retail Zone A*66.30*275.00*18233 +02*2*Ground*Retail Zone B*66.30*137.50*9116 +02*3*Ground*Retail Zone C*58.70*68.75*4036 +02*4*Ground*Remaining Retail Zone*174.80*34.38*6010 +01*27260921000*4606914000*4225*The Occupier*SOMERFIELD****DALE STREET*MILNROW*ROCHDALE*LANCS*OL16 4HS*86199*Shop And Premises*1134.10*129287*129287*129000*2010*Rochdale*50394699171651*27218435539***152*GIA*114.00 +02*1*Ground*All Main Areas*1134.10*114.00*129287 diff --git a/tests/test_datasource.py b/tests/test_datasource.py index 2724f70..a396a93 100644 --- a/tests/test_datasource.py +++ b/tests/test_datasource.py @@ -104,9 +104,7 @@ def test_coerce_keeps_rows_on_any_index(self): # Actions can leave a frame indexed by labels rather than 0..n; coercion must keep every value in its row schema = qd.SchemaDefinition() schema.set(schema={"name": "coercion"}) - schema.fields.add_multi( - terms=[{"name": "amount", "type": "number"}, {"name": "reliefs", "type": "array"}] - ) + schema.fields.add_multi(terms=[{"name": "amount", "type": "number"}, {"name": "reliefs", "type": "array"}]) df = pd.DataFrame( {"amount": ["80.0", "88.5"], "reliefs": [["80.0", None], ["88.5", "12.0"]]}, index=["7", "3"], @@ -117,7 +115,9 @@ def test_coerce_keeps_rows_on_any_index(self): def test_a_frames_columns_are_typed_as_1x_typed_them(self): # A date column is recorded as `string`, as 1.x recorded it; typing records dates - df = pd.DataFrame({"a": [1.5], "b": pd.array([1], dtype="Int64"), "c": pd.to_datetime(["2024-01-04"]), "d": ["x"]}) + df = pd.DataFrame( + {"a": [1.5], "b": pd.array([1], dtype="Int64"), "c": pd.to_datetime(["2024-01-04"]), "d": ["x"]} + ) columns = DataSourceParser().get_header_columns(df=df) assert [c.dtype for c in columns] == ["number", "number", "string", "string"] diff --git a/tests/test_diagnosis.py b/tests/test_diagnosis.py index efcbe53..c8b812b 100644 --- a/tests/test_diagnosis.py +++ b/tests/test_diagnosis.py @@ -28,6 +28,7 @@ IDS = [f"{name}-{dialect.get('sheet', '')}" for name, _, dialect, _ in CASES] SYNTHETIC = LAYOUTS.parent / "synthetic" FENLAND = LAYOUTS.parent / "sample" / "fenland" / "2026-10-25-Fenland District Council.csv" +ASTERISK = sorted((LAYOUTS.parent / "sample" / "voa").glob("*.csv")) def diagnose_source(source: Path, mimetype: str, dialect: dict, propose: bool = False) -> ReportModel: @@ -119,6 +120,38 @@ def test_a_clean_comma_file_with_semicolons_in_its_cells_is_not_second_guessed(t assert diagnose_source(source, "CSV", {}, propose=True).proposals == [] +def test_an_uneven_comma_file_with_a_character_on_every_line_is_not_second_guessed(tmp_path): + source = tmp_path / "uneven.csv" + source.write_text("a/b,c,d\n1/2,3,4\n5/6,7\n", encoding="utf-8") + assert diagnose_source(source, "CSV", {}, propose=True).proposals == [] + + +@pytest.mark.parametrize("source", ASTERISK, ids=[p.name[:40] for p in ASTERISK]) +def test_any_character_may_be_proposed_as_a_delimiter(source): + # One holds commas in its addresses and gives each line 34 cells; the other no comma, and several kinds of record + report = diagnose_source(source, "CSV", {}, propose=True) + assert [(p.setting, p.value) for p in report.proposals] == [("delimiter", "*")] + assert diagnose_source(source, "CSV", {"delimiter": "*"}, propose=True).proposals == [] + + +def test_a_delimiter_is_proposed_where_the_one_read_by_is_in_no_line(tmp_path): + source = tmp_path / "no-commas.csv" + source.write_text("a*b*c\n1*2*3\n4*5*6\n", encoding="utf-8") + assert [p.value for p in diagnose_source(source, "CSV", {}, propose=True).proposals] == ["*"] + + +def test_a_single_column_is_not_given_a_delimiter(tmp_path): + source = tmp_path / "single.csv" + source.write_text("name\nalpha\nbeta\n", encoding="utf-8") + assert diagnose_source(source, "CSV", {}, propose=True).proposals == [] + + +def test_two_characters_that_both_fit_leave_the_delimiter_unproposed(tmp_path): + source = tmp_path / "mixed.csv" + source.write_text("a*b;c\n1*2;3\n4*5;6\n", encoding="utf-8") + assert diagnose_source(source, "CSV", {}, propose=True).proposals == [] + + @pytest.mark.parametrize("name,mimetype,dialect,case", CASES, ids=IDS) def test_nothing_is_proposed_unasked(name, mimetype, dialect, case): assert diagnose(name, mimetype, dialect).proposals == [] diff --git a/tests/test_header.py b/tests/test_header.py index 1706dfd..2005ab9 100644 --- a/tests/test_header.py +++ b/tests/test_header.py @@ -115,7 +115,13 @@ def test_a_header_below_row_0_keeps_every_row(self): expected = V1ChecksumParser().read(source=model) df = DataSourceParser().get(source=model) assert df[ROW].tolist() == list(range(len(expected))) == list(range(268)) - assert values(df[["Country Name"]])[:5] == [["Data Source"], ["Last Updated Date"], [None], ["Country Name"], ["Aruba"]] + assert values(df[["Country Name"]])[:5] == [ + ["Data Source"], + ["Last Updated Date"], + [None], + ["Country Name"], + ["Aruba"], + ] assert df["Country Name"].fillna("").tolist() == expected["Country Name"].fillna("").tolist() def test_a_stored_crosswalk_gives_the_same_output(self): diff --git a/tests/test_known_faults.py b/tests/test_known_faults.py index 7cde804..2dd0a21 100644 --- a/tests/test_known_faults.py +++ b/tests/test_known_faults.py @@ -76,7 +76,9 @@ def test_pivot_longer_keeps_columns_in_source_order(self): def test_separate_passes_nulls_through(self): df = pd.DataFrame({"s": ["a;b", None, "c;d"]}) destination = [FieldModel(name="p", type="string"), FieldModel(name="q", type="string")] - df = Separate().transform(df=df, destination=destination, source=[FieldModel(name="s", type="string")], source_param=";") + df = Separate().transform( + df=df, destination=destination, source=[FieldModel(name="s", type="string")], source_param=";" + ) assert df[["p", "q"]].values.tolist() == [["a", "b"], [None, None], ["c", "d"]] @pytest.mark.xfail(strict=True, raises=KeyError, reason="a null header row raises KeyError") @@ -84,5 +86,8 @@ def test_pivot_categories_refuses_a_null_header(self): df = pd.DataFrame({"c": ["Cat1", "t2", None, "t4"]}) with pytest.raises(ValueError, match="2"): PivotCategories().transform( - df=df, destination=FieldModel(name="cat", type="string"), source=FieldModel(name="c", type="string"), assigned=[0, 2] + df=df, + destination=FieldModel(name="cat", type="string"), + source=FieldModel(name="c", type="string"), + assigned=[0, 2], ) diff --git a/tests/test_polars.py b/tests/test_polars.py index 47eb0d1..45cebc6 100644 --- a/tests/test_polars.py +++ b/tests/test_polars.py @@ -39,9 +39,7 @@ def lines(name: str) -> list[list[str]]: def polars_lines(name: str, **settings) -> list[list[str]]: # Each row as read: a field the row lacks is null, an empty field empty text - df = pl.read_csv( - LAYOUTS / name, has_header=False, infer_schema=False, empty_string_is_null=False, **settings - ) + df = pl.read_csv(LAYOUTS / name, has_header=False, infer_schema=False, empty_string_is_null=False, **settings) read = [] for row in df.rows(): row = list(row) @@ -60,7 +58,9 @@ def test_pandas_to_polars_and_back(self, dtype): assert cells(back) == HOSTILE def test_polars_to_pandas(self): - grid = pl.DataFrame([dict(zip(NAMES, row, strict=True)) for row in HOSTILE], schema={n: pl.String for n in NAMES}) + grid = pl.DataFrame( + [dict(zip(NAMES, row, strict=True)) for row in HOSTILE], schema={n: pl.String for n in NAMES} + ) df = grid.to_pandas() assert list(df.columns) == NAMES assert cells(df) == HOSTILE diff --git a/tests/test_tutorial_4.py b/tests/test_tutorial_4.py index 113b09f..7e755a4 100644 --- a/tests/test_tutorial_4.py +++ b/tests/test_tutorial_4.py @@ -22,7 +22,7 @@ "CATEGORISE > 'occupation_state_reliefs'::'retail' < 'AddRlfCd'::['RETDS3']", "CATEGORISE > 'occupation_state_reliefs'::'small_business' < 'SBRFlag'::['yes']", "CATEGORISE > 'occupation_state'::False < 'ChgType'::['V']", - "COLLATE > 'prop_ba_rates' < ['MandRlf', 'DiscRlf', 'AdditionalRlf', ~]" + "COLLATE > 'prop_ba_rates' < ['MandRlf', 'DiscRlf', 'AdditionalRlf', ~]", ] diff --git a/whyqd/__main__.py b/whyqd/__main__.py index 4373137..bc57de3 100644 --- a/whyqd/__main__.py +++ b/whyqd/__main__.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause """Data wrangling simplicity, complete audit transparency, and at speed.""" + import sys diff --git a/whyqd/config/settings.py b/whyqd/config/settings.py index a860c4f..1163fdc 100644 --- a/whyqd/config/settings.py +++ b/whyqd/config/settings.py @@ -38,9 +38,23 @@ class Settings(BaseSettings): "windows-1252": { "codec": "cp1252", "labels": ( - "ansi_x3.4-1968", "ascii", "cp1252", "cp819", "csisolatin1", "ibm819", "iso-8859-1", "iso-ir-100", - "iso8859-1", "iso88591", "iso_8859-1", "iso_8859-1:1987", "l1", "latin1", "us-ascii", - "windows-1252", "x-cp1252", + "ansi_x3.4-1968", + "ascii", + "cp1252", + "cp819", + "csisolatin1", + "ibm819", + "iso-8859-1", + "iso-ir-100", + "iso8859-1", + "iso88591", + "iso_8859-1", + "iso_8859-1:1987", + "l1", + "latin1", + "us-ascii", + "windows-1252", + "x-cp1252", ), }, "macintosh": {"codec": "mac_roman", "labels": ("csmacintosh", "mac", "macintosh", "x-mac-roman")}, diff --git a/whyqd/core/datasource.py b/whyqd/core/datasource.py index eab30f7..73da6bd 100644 --- a/whyqd/core/datasource.py +++ b/whyqd/core/datasource.py @@ -233,7 +233,9 @@ def get_citation(self) -> dict[str, str | dict[str, str]]: if not isinstance(self.model, list): return super().get_citation() return [ - m.citation.model_dump(by_alias=True, exclude_defaults=True, exclude_none=True) for m in self.model if m.citation + m.citation.model_dump(by_alias=True, exclude_defaults=True, exclude_none=True) + for m in self.model + if m.citation ] def set_citation(self, *, citation: CitationModel, index: int = None) -> None: diff --git a/whyqd/core/schema.py b/whyqd/core/schema.py index 4f26df7..fadadb9 100644 --- a/whyqd/core/schema.py +++ b/whyqd/core/schema.py @@ -51,7 +51,9 @@ def _refresh_model_fields(self) -> None: self.crud.add_multi(terms=self.model.fields) else: self.crud.add_multi( - terms=[f.model_dump(by_alias=True, exclude_defaults=True, exclude_none=True) for f in self.model.fields] + terms=[ + f.model_dump(by_alias=True, exclude_defaults=True, exclude_none=True) for f in self.model.fields + ] ) if self.model and not self.model.fields: self.crud.reset() diff --git a/whyqd/crosswalk/actions/__init__.py b/whyqd/crosswalk/actions/__init__.py index 3e3521a..04e393b 100644 --- a/whyqd/crosswalk/actions/__init__.py +++ b/whyqd/crosswalk/actions/__init__.py @@ -3,7 +3,7 @@ Actions ------- -Definitions and validation for actions anchoring structures in methods. Describes how an Action performs its roll. +Definitions and validation for actions anchoring structures in methods. Describes how an Action performs its roll. New transforms inherit from the core.BaseSchemaAction class and are automatically available as a plugin. Key functions are: @@ -32,16 +32,18 @@ ] Where the first term in the list if the action (as would appear in `action.settings`) and the subsequent fields -must conform to the structure defined in the action's structure, but with a format that can be accessed in the +must conform to the structure defined in the action's structure, but with a format that can be accessed in the `action.transform` function. Normally this would be a `name` and `type`, but any created value would have a `value` instead of a `name`. """ + import pkgutil import importlib.util as _importlib_util from pathlib import Path __action_path__ = str(Path(__file__).resolve().parent) + def load_dynamic(name, module_path): # https://bugs.python.org/issue43540 module_path = f"{module_path}/{name}" @@ -52,8 +54,8 @@ def load_dynamic(name, module_path): spec.loader.exec_module(module) return module.Action + actions = { - name.upper(): load_dynamic(name, finder.path) - for finder, name, _ispkg in pkgutil.iter_modules([__action_path__]) + name.upper(): load_dynamic(name, finder.path) for finder, name, _ispkg in pkgutil.iter_modules([__action_path__]) } default_actions = [a for a in [actn().settings for actn in actions.values()]] diff --git a/whyqd/crosswalk/actions/collate.py b/whyqd/crosswalk/actions/collate.py index d7b1a2b..db0fcf7 100644 --- a/whyqd/crosswalk/actions/collate.py +++ b/whyqd/crosswalk/actions/collate.py @@ -21,7 +21,7 @@ class Action(BaseSchemaAction): Where the order of collation is important and is used to create an array (list) of values from corresponding sources. Any missing values will be included as `None`. - This can be used as part of a process which will permit multiple corresponding columns (e.g. a `date`, `description`, + This can be used as part of a process which will permit multiple corresponding columns (e.g. a `date`, `description`, `value` sequence) to each be placed in an array, and then pivoted using `df.explode`. !!! example @@ -96,16 +96,11 @@ def transform( if isinstance(s, FieldModel): # It's not a spacer column source_column = df[s.name].tolist() - new_column.append( - [ - [x] if not isinstance(x, list) else x - for x in source_column - ] - ) + new_column.append([[x] if not isinstance(x, list) else x for x in source_column]) if len(new_column) > 1: # Permits reconstruction of datasets using array transformations # https://stackoverflow.com/a/18411610 - new_column = [[c for clist in x for c in clist] for x in zip(*new_column)] + new_column = [[c for clist in x for c in clist] for x in zip(*new_column, strict=False)] else: new_column = [[x] if not isinstance(x, list) else x for x in new_column[0]] df[destination.name] = new_column diff --git a/whyqd/crosswalk/base/category_base.py b/whyqd/crosswalk/base/category_base.py index 59206c8..3fba0d7 100644 --- a/whyqd/crosswalk/base/category_base.py +++ b/whyqd/crosswalk/base/category_base.py @@ -204,7 +204,7 @@ def transform( # Permits reconstruction of datasets using array transformations # https://stackoverflow.com/a/18411610 # new_column = [sorted(list(set([c for clist in x for c in clist])), key=lambda x: (x is None, x)) for x in zip(*new_column)] - new_column = [[c for clist in x for c in clist] for x in zip(*new_column)] + new_column = [[c for clist in x for c in clist] for x in zip(*new_column, strict=False)] else: new_column = [[x] if not isinstance(x, list) else x for x in new_column[0]] # for n in new_column: diff --git a/whyqd/models/action_category.py b/whyqd/models/action_category.py index 64ac42a..ad1b71c 100644 --- a/whyqd/models/action_category.py +++ b/whyqd/models/action_category.py @@ -20,6 +20,6 @@ class CategoryActionModel(BaseModel): @classmethod def check_valid_models(cls, v: Optional[List[str]]): for s in v: - if not (s in ["boolean", "unique"]): + if s not in ["boolean", "unique"]: raise ValueError(f"Structure ({s}) must be of either `boolean` or `unique`.") return v diff --git a/whyqd/models/action_morph.py b/whyqd/models/action_morph.py index aed6d08..ba513aa 100644 --- a/whyqd/models/action_morph.py +++ b/whyqd/models/action_morph.py @@ -20,6 +20,6 @@ class MorphActionModel(BaseModel): @classmethod def check_valid_models(cls, v: Optional[List[str]]): for s in v: - if not (s in ["fields", "rows", "source"]): + if s not in ["fields", "rows", "source"]: raise ValueError(f"Structure ({s}) must be of either `source`, `fields`, or `rows`.") return v diff --git a/whyqd/models/datasource.py b/whyqd/models/datasource.py index 2d280ce..136d673 100644 --- a/whyqd/models/datasource.py +++ b/whyqd/models/datasource.py @@ -35,7 +35,9 @@ class DataSourceModel(BaseModel): None, description="A complete description of the data source. Depending on how complex your work becomes, try and be as helpful as possible to 'future-you'. You'll thank yourself later.", ) - path: Annotated[str, StringConstraints(strip_whitespace=True)] = Field(..., description="Full path to valid source data file.") + path: Annotated[str, StringConstraints(strip_whitespace=True)] = Field( + ..., description="Full path to valid source data file." + ) mime: MimeType = Field(..., description="Mime type for source data. Automatically generated.") header: Optional[int] = Field( 0, description="Row (0-indexed) to use for the column labels of the parsed DataFrame. " @@ -61,7 +63,8 @@ class DataSourceModel(BaseModel): ) dialect: Optional[DialectModel] = Field(None, description="How the file is written, so it can be read.") hidden: Optional[bool] = Field( - None, description="The publisher hid this sheet of the workbook; it is read like any other. Absent where visible." + None, + description="The publisher hid this sheet of the workbook; it is read like any other. Absent where visible.", ) attributes: Optional[DataSourceAttributeModel] = Field( {}, diff --git a/whyqd/parsers/action.py b/whyqd/parsers/action.py index ea5e55b..49a0081 100644 --- a/whyqd/parsers/action.py +++ b/whyqd/parsers/action.py @@ -124,7 +124,7 @@ def parse( # extraction of the action, then replace the hash with a nested dictionary {action: , source: } # where each 'source' contains the input for that action, and is the input for the higher level. parsed_stack = stack - for txt, prsed, hx in parsed: + for txt, _, hx in parsed: # replace the txt with hx parsed_stack = parsed_stack.replace(f"[{txt}]", hx) i_prsed = [] @@ -133,7 +133,7 @@ def parse( if len(splt) == 1: i_prsed.extend(self.parser.get_listed_literal(text=s)) else: - for txt, prsed, hx in parsed: + for _, prsed, hx in parsed: if hx in s: prsed["action"] = self.parser.get_action_model(action=splt[0]) if not isinstance(prsed["action"], SchemaActionModel): diff --git a/whyqd/parsers/arrow_reader.py b/whyqd/parsers/arrow_reader.py index dd0deb1..299f428 100644 --- a/whyqd/parsers/arrow_reader.py +++ b/whyqd/parsers/arrow_reader.py @@ -53,5 +53,7 @@ def grid( columns[f"column_{i}"] = [name, *(None if c is None else c[1] for c in cells)] types[f"column_{i}"] = [CellType.STRING.value, *(None if c is None else c[0].value for c in cells)] schema = {c: pl.String for c in columns} - grid, types = self.grids.trim(grid=pl.DataFrame(columns, schema=schema), types=pl.DataFrame(types, schema=schema)) + grid, types = self.grids.trim( + grid=pl.DataFrame(columns, schema=schema), types=pl.DataFrame(types, schema=schema) + ) return grid, types, list(range(1, grid.height + 1)), dialect diff --git a/whyqd/parsers/category.py b/whyqd/parsers/category.py index 80ae061..3913001 100644 --- a/whyqd/parsers/category.py +++ b/whyqd/parsers/category.py @@ -185,9 +185,11 @@ def set_schema( self.schema_source = schema_source self.schema_destination = schema_destination - def get_schema_field_category(self, *, field: FieldModel, term: str, is_source: bool = True) -> CategoryModel | None: + def get_schema_field_category( + self, *, field: FieldModel, term: str, is_source: bool = True + ) -> CategoryModel | None: """ - Recover a field category model from a string. It is possible that source and destination schema category share + Recover a field category model from a string. It is possible that source and destination schema category share a common name, so we need to check all possibilities. Parameters @@ -287,12 +289,9 @@ def get_hexed_script(self, *, script: str) -> str: source_field = self.schema_source.fields.get(name=field_name(source_script[0])) if not source_field: raise ValueError(f'Source field not found: "{script}"') - hexing_terms.extend([ - [source_field], - hexing_categories(source_field) - ]) + hexing_terms.extend([[source_field], hexing_categories(source_field)]) hexed = [] - for x, f in zip(hexing_script, hexing_terms): + for x, f in zip(hexing_script, hexing_terms, strict=False): hexed.append(self.parser.get_hexed_script(script=x, fields=f)) if len(hexed) == 3: return f"{root_split[0]} > {hexed[0]} < {hexed[1]}::{hexed[2]}" @@ -302,4 +301,7 @@ def get_assigned_uniques(self, *, text: str) -> list[str]: terms = list(self.parser.generate_contents(text=text)) if len(terms) != 1: raise ValueError(f"Category assignment actions must not be nested. ({text}).") - return [self.parser.get_literal(text=t) for t in self.parser.get_split_terms(script=terms[0][1], by=",", maxsplit=-1)] + return [ + self.parser.get_literal(text=t) + for t in self.parser.get_split_terms(script=terms[0][1], by=",", maxsplit=-1) + ] diff --git a/whyqd/parsers/core.py b/whyqd/parsers/core.py index 3955af9..e08dd62 100644 --- a/whyqd/parsers/core.py +++ b/whyqd/parsers/core.py @@ -46,7 +46,7 @@ def chunks(self, *, lst: list, n: int) -> list: yield lst[i : i + n] def show_warning(self, message: str) -> None: - warnings.warn(message, UserWarning) + warnings.warn(message, UserWarning, stacklevel=2) ################################################################################################### ### MODEL UTILITIES @@ -198,9 +198,8 @@ def load_json(self, *, source: str) -> dict: with open(source, "r", encoding="utf-8") as f: try: return json.load(f) - except json.decoder.JSONDecodeError: - e = "File at `{}` not valid json.".format(source) - raise json.decoder.JSONDecodeError(e) + except json.decoder.JSONDecodeError as e: + raise json.decoder.JSONDecodeError(f"File at `{source}` not valid json. {e.msg}", e.doc, e.pos) from e def save_json(self, *, data: dict, source: str) -> bool: """ diff --git a/whyqd/parsers/csv_reader.py b/whyqd/parsers/csv_reader.py index 7dd9e9a..0d29f2e 100644 --- a/whyqd/parsers/csv_reader.py +++ b/whyqd/parsers/csv_reader.py @@ -27,8 +27,6 @@ class CsvParser: PLACE = "line" DIALECT = ("encoding", "encodingStep", "delimiter", "quoteChar", "doubleQuote") - # What a proposed delimiter is chosen among - DELIMITERS = (",", ";", "\t", "|") # What a `csv-quote` fault's setting takes, to read past it PAST_QUOTE = {"quoteChar": None, "doubleQuote": False} @@ -81,9 +79,12 @@ def proposals( """The settings that would read past this file's faults, each with its evidence: what lets the file read at all, or else a delimiter that fits its lines where the one read by does not. - A quote setting is proposed only once the whole file has read with it. A delimiter is proposed only where one - alone gives every line the same width, over one cell, and the one read by does not, so a sound file is never - second-guessed. Where the evidence is mixed, nothing is proposed. + A quote setting is proposed only once the whole file has read with it. A delimiter may be any character that is + not a letter, a digit, or a space (a tab may be), since a publisher may use any: it is proposed only where it + alone splits every line, the first included, and the one read by leaves them uneven or does not split them. + Where that one leaves most lines unsplit, the new one's widths may differ, as a file holding several kinds of + record does; otherwise they must not, and it must give as many cells as that one's widest row, so a sound file + is never second-guessed. Where the evidence is mixed, nothing is proposed. """ found = [] with warnings.catch_warnings(): @@ -111,26 +112,39 @@ def proposals( def delimiter(self, *, text: str, dialect: DialectModel) -> list[ProposalModel]: lines = [m.group() for m in islice(self.patterns.LINE.finditer(text), settings.REPORT_SAMPLE_LINES)] lines = [line for line in lines if line.strip("\r\n")] + if len(lines) < 2: + return [] - def width(delimiter: str) -> int | None: + def widths(delimiter: str) -> list[int]: try: - widths = {len(row) for row in csv.reader(lines, delimiter=delimiter) if row} + return [len(row) for row in csv.reader(lines, delimiter=delimiter) if row] except csv.Error: - return None - return widths.pop() if len(widths) == 1 else None + return [] - if width(dialect.delimiter) is not None: + read = widths(dialect.delimiter) + if len(set(read)) == 1 and read[0] != 1: return [] - fits = {d: w for d in self.DELIMITERS if d != dialect.delimiter and (w := width(d)) is not None and w > 1} + unsplit = sum(w == 1 for w in read) > len(read) / 2 + candidates = sorted( + c + for c in set(lines[0]) + if not c.isalnum() and (c == "\t" or not c.isspace()) and c not in (dialect.delimiter, dialect.quoteChar) + ) + fits = {} + for candidate in candidates: + found = set(widths(candidate)) + if found and min(found) > 1 and (unsplit or (len(found) == 1 and min(found) >= max(read))): + fits[candidate] = found if len(fits) != 1: return [] - delimiter, cells = fits.popitem() + delimiter, found = fits.popitem() + cells = " or ".join(str(w) for w in sorted(found)) + before = "leaves most of them unsplit" if unsplit else "gives them differing widths" return [ ProposalModel( setting="delimiter", value=delimiter, - evidence=f"It gives each of the first {len(lines)} lines {cells} cells; " - f"`{dialect.delimiter}` gives them differing widths.", + evidence=f"It gives each of the first {len(lines)} lines {cells} cells; `{dialect.delimiter}` {before}.", ) ] diff --git a/whyqd/parsers/datasource.py b/whyqd/parsers/datasource.py index 8aa4cd2..ccc797c 100644 --- a/whyqd/parsers/datasource.py +++ b/whyqd/parsers/datasource.py @@ -126,8 +126,8 @@ def set(self, *, df: pd.DataFrame, source: str | Path, mimetype: MimeType) -> No source = Path(source) try: source = source.with_suffix(f".{mimetype.name}") - except Exception: - raise FileExistsError(f"Save mimetype not supported, `{mimetype}`.") + except Exception as e: + raise FileExistsError(f"Save mimetype not supported, `{mimetype}`.") from e if mimetype in [MimeType.PRQ, MimeType.PARQUET]: df.to_parquet(path=source, engine="pyarrow", index=False) if mimetype in [MimeType.FTR, MimeType.FEATHER]: @@ -152,15 +152,19 @@ def coerce_to_schema(self, *, df: pd.DataFrame, schema: Type[SchemaDefinition]) validate["matched"].append(c.name) else: # Try coerce the column to the type - df[c.name] = self.coerce_column_to_dtype(column=pd.Series(df[c.name].to_numpy(), index=df.index), coerce=prospect.dtype) + df[c.name] = self.coerce_column_to_dtype( + column=pd.Series(df[c.name].to_numpy(), index=df.index), coerce=prospect.dtype + ) validate["coerced"].append(c.name) - df[c.name] = pd.Series(df[c.name].to_numpy(), index=df.index).astype(FieldType(prospect.dtype).astype) - except (TypeError, SyntaxError, ValueError): + df[c.name] = pd.Series(df[c.name].to_numpy(), index=df.index).astype( + FieldType(prospect.dtype).astype + ) + except (TypeError, SyntaxError, ValueError) as error: from_type = c.dtype if isinstance(df[c.name][0], list): from_type = "array" e = f"Column '{c.name}' in Data cannot be converted from type '{from_type}' to '{prospect.dtype}'." - raise TypeError(e) + raise TypeError(e) from error else: validate["unmatched"].append(c.name) if validate["unmatched"]: @@ -497,7 +501,9 @@ def diagnose( for sheet, hidden in sheets ] - def get_source_data_model(self, *, df: pd.DataFrame, source: Path | str, mimetype: str | MimeType) -> DataSourceModel: + def get_source_data_model( + self, *, df: pd.DataFrame, source: Path | str, mimetype: str | MimeType + ) -> DataSourceModel: """The data source of a transform's output, from the frame it was saved from, with its `table-1x` checksum. Parameters @@ -575,7 +581,7 @@ def _date_is_isoformat(self, x: str) -> bool: """ try: # `int` formatted dates will fail with `AttributeError` - datetime.fromisoformat(x.replace('Z', '+00:00')) + datetime.fromisoformat(x.replace("Z", "+00:00")) except (ValueError, TypeError, AttributeError): return False return True @@ -671,9 +677,9 @@ def _postprocess_parse_float(self, x: str) -> np.nan | float: """ # https://stackoverflow.com/a/71206446 if x.group(3): - return f"{x.group(1).replace(',','').replace('.','')}.{x.group(3)}" + return f"{x.group(1).replace(',', '').replace('.', '')}.{x.group(3)}" elif x.group(2): - return f"{x.group(1).replace(',','').replace('.','')}" + return f"{x.group(1).replace(',', '').replace('.', '')}" else: return np.nan diff --git a/whyqd/parsers/excel_reader.py b/whyqd/parsers/excel_reader.py index 3a67cd7..3bd4246 100644 --- a/whyqd/parsers/excel_reader.py +++ b/whyqd/parsers/excel_reader.py @@ -170,7 +170,9 @@ def grid( columns[f"column_{j}"].append(None if rendered is None else rendered[1]) types[f"column_{j}"].append(None if rendered is None else rendered[0].value) schema = {c: pl.String for c in columns} - grid, types = self.grids.trim(grid=pl.DataFrame(columns, schema=schema), types=pl.DataFrame(types, schema=schema)) + grid, types = self.grids.trim( + grid=pl.DataFrame(columns, schema=schema), types=pl.DataFrame(types, schema=schema) + ) return grid, types, list(range(1, grid.height + 1)), dialect.model_copy(update={"sheet": sheet}) def facts(self, *, source: Path, sheet: str, types: pl.DataFrame) -> dict[str, list[str]]: @@ -282,16 +284,16 @@ def parts(self, *, book: zipfile.ZipFile, path: str, where: str) -> dict[str, li for element in ElementTree.fromstring(book.read(path)).iter(): if self.local(element.tag) == "Relationship": target = element.get("Target") - target = target.lstrip("/") if target.startswith("/") else posixpath.normpath(posixpath.join(where, target)) + target = ( + target.lstrip("/") if target.startswith("/") else posixpath.normpath(posixpath.join(where, target)) + ) found.setdefault(element.get("Type").rsplit("/", 1)[-1], []).append((target, element.get("Id"))) return found def styles(self, *, book: zipfile.ZipFile, path: str) -> list[str]: """Each cell style's format code, by the style's index (`cellXfs`).""" root = ElementTree.fromstring(book.read(path)) - custom = { - int(e.get("numFmtId")): e.get("formatCode") for e in root.iter() if self.local(e.tag) == "numFmt" - } + custom = {int(e.get("numFmtId")): e.get("formatCode") for e in root.iter() if self.local(e.tag) == "numFmt"} cell_styles = next((e for e in root.iter() if self.local(e.tag) == "cellXfs"), []) codes = [] for xf in cell_styles: diff --git a/whyqd/parsers/pdf_reader.py b/whyqd/parsers/pdf_reader.py index 7bc26ca..78fb808 100644 --- a/whyqd/parsers/pdf_reader.py +++ b/whyqd/parsers/pdf_reader.py @@ -119,7 +119,9 @@ def lines(self, *, page, columns: list[float]) -> list[list[str | None]]: top = word["top"] column = max([0, *(i for i, edge in enumerate(columns) if word["x0"] >= edge)]) rows[-1][column].append(word) - return [[" ".join(w["text"] for w in sorted(cell, key=lambda w: w["x0"])) or None for cell in row] for row in rows] + return [ + [" ".join(w["text"] for w in sorted(cell, key=lambda w: w["x0"])) or None for cell in row] for row in rows + ] def unrepeated(self, *, rows: list[list[str | None]], dialect: DialectModel) -> list[list[str | None]]: """The rows without any equal to a header row after the header, each being that header atop a later page.""" diff --git a/whyqd/parsers/table.py b/whyqd/parsers/table.py index 56ad39c..4de5632 100644 --- a/whyqd/parsers/table.py +++ b/whyqd/parsers/table.py @@ -60,7 +60,9 @@ def table( """ beside = beside or [] if grid.width == 0: - return pl.DataFrame({self.row_key.ROW: []}, schema={self.row_key.ROW: pl.UInt32}), [pl.DataFrame()] * len(beside) + return pl.DataFrame({self.row_key.ROW: []}, schema={self.row_key.ROW: pl.UInt32}), [pl.DataFrame()] * len( + beside + ) frames, lines = [f.slice(dialect.skipRows) for f in [grid, *beside]], lines[dialect.skipRows :] if dialect.rowsPerRecord: self.fit(grid=frames[0], lines=lines, fields=dialect.rowsPerRecord, place=place) From 4990244eae0df5755d7ed199ee91de51f556ba1b Mon Sep 17 00:00:00 2001 From: Gavin Chait Date: Mon, 2 Nov 2026 19:53:05 +0100 Subject: [PATCH 6/7] Read names past a file's last column as empty columns --- tests/test_definition_slots.py | 7 +++++-- tests/test_header.py | 38 ++++++++++++++++++++++++++++++++++ whyqd/core/datasource.py | 2 +- whyqd/parsers/table.py | 8 +++++-- 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/tests/test_definition_slots.py b/tests/test_definition_slots.py index b52ff1f..3569f0b 100644 --- a/tests/test_definition_slots.py +++ b/tests/test_definition_slots.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause import json -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from pathlib import Path import pytest @@ -216,7 +216,10 @@ def transform() -> qd.TransformDefinition: class TestTransformRecord: def test_run_recorded(self, transform): model = transform.get - assert transform.before <= model.started <= model.ended <= datetime.now(timezone.utc) + # A machine's clock may step back by a second or more as it synchronises, so each is ordered within a margin + margin = timedelta(seconds=30) + assert transform.before - margin <= model.started <= model.ended + margin + assert model.ended - margin <= datetime.now(timezone.utc) assert model.whyqd == qd.__version__ def test_written_in_utc(self, transform): diff --git a/tests/test_header.py b/tests/test_header.py index 2005ab9..e59ebc0 100644 --- a/tests/test_header.py +++ b/tests/test_header.py @@ -79,6 +79,44 @@ def test_no_header_row_takes_the_names_given(self, tmp_path): assert datasource.get.dialect.headerRowCount == 0 assert df[["x", "y"]].values.tolist() == [["a", "b"], ["1", "2"]] + def test_names_past_the_grid_are_columns_the_file_leaves_empty(self, tmp_path): + # A format with a fixed set of fields, whose last are empty throughout this file, so the grid trims them + datasource = qd.DataSourceDefinition() + source = write(tmp_path, "a*b**\n1*2**\n") + dialect = {"delimiter": "*", "headerRowCount": 0} + datasource.derive_model(source=source, mimetype="CSV", dialect=dialect, names=["x", "y", "z", "w"]) + df = table(datasource) + assert [c.name for c in datasource.get.columns] == ["x", "y", "z", "w"] + assert datasource.get.columns[3].profile.blank == 2 + assert values(df[["x", "y", "z", "w"]]) == [["a", "b", None, None], ["1", "2", None, None]] + + def test_names_past_the_grid_leave_the_fingerprint_alone(self, tmp_path): + source = write(tmp_path, "a,b\n1,2\n") + dialect = {"headerRowCount": 0} + short, long = qd.DataSourceDefinition(), qd.DataSourceDefinition() + short.derive_model(source=source, mimetype="CSV", dialect=dialect) + long.derive_model(source=source, mimetype="CSV", dialect=dialect, names=["x", "y", "z"]) + assert long.get.get_checksum("fp0") == short.get.get_checksum("fp0") + + def test_every_file_of_a_fixed_format_reads_to_one_shape(self): + source = next((DATA / "sample" / "voa").glob("*listentries*")) + names = [f"field_{i}" for i in range(34)] + datasource = qd.DataSourceDefinition() + datasource.derive_model( + source=source, + mimetype="CSV", + dialect={"delimiter": "*", "quoteChar": None, "headerRowCount": 0}, + names=names, + ) + assert [c.name for c in datasource.get.columns] == names + assert table(datasource)["field_0"].tolist()[0] == "0660" + + def test_fewer_names_than_columns_are_refused(self, tmp_path): + datasource = qd.DataSourceDefinition() + source = write(tmp_path, "a,b,c\n1,2,3\n") + with pytest.raises(ValueError, match=r"\[whyqd:names-count\]"): + datasource.derive_model(source=source, mimetype="CSV", dialect={"headerRowCount": 0}, names=["x", "y"]) + def test_a_header_over_two_rows_takes_the_names_given(self, tmp_path): datasource = qd.DataSourceDefinition() source = write(tmp_path, "Rate,Rate\nfrom,to\n1,2\n") diff --git a/whyqd/core/datasource.py b/whyqd/core/datasource.py index 73da6bd..162353a 100644 --- a/whyqd/core/datasource.py +++ b/whyqd/core/datasource.py @@ -121,7 +121,7 @@ def derive_model( describes it. Undeclared, each setting takes its default; a CSV's encoding, undeclared, is settled by whyqd's ladder and recorded. For a workbook, a list, each naming its `sheet`, or one a sheet in order. names: One name per column, for a file with no header row or a header over several rows; for a workbook, a - mapping of sheet to names. + mapping of sheet to names. Names past the file's last filled column are added as empty columns. settings: Refused, naming the dialect: `header` and pandas' keywords are no longer read. Returns: diff --git a/whyqd/parsers/table.py b/whyqd/parsers/table.py index 4de5632..4266198 100644 --- a/whyqd/parsers/table.py +++ b/whyqd/parsers/table.py @@ -40,7 +40,9 @@ def table( Where in the file each grid row starts, counted from 1, for messages. dialect: DialectModel names: list[str] | None, default None - One name per column, in place of the header row's; required for a header over several rows. + One name per column, in place of the header row's; required for a header over several rows. Names past the + grid's last column are columns the file leaves empty, which the grid trims: each is added, empty, so every + file of a format with fixed fields reads to one shape. place: str, default "line" What `lines` counts, as the wrangler would look for it: a text file's `line`, a sheet's `row`. beside: list[pl.DataFrame] | None, default None @@ -83,6 +85,8 @@ def table( kept = ~frames[0].select(pl.all_horizontal(pl.all().is_null())).to_series() frames = [f.filter(kept) for f in frames] columns = self.names(header=header, names=names, dialect=dialect) + empty = [pl.lit(None, dtype=pl.String).alias(f"empty_{i}") for i in range(header.width, len(columns))] + frames = [f.with_columns(empty) for f in frames] for frame in frames: frame.columns = columns grid, *beside = frames @@ -91,7 +95,7 @@ def table( def names(self, *, header: pl.DataFrame, names: list[str] | None, dialect: DialectModel) -> list[str]: width = header.width if names is not None: - if len(names) != width: + if len(names) < width: raise ValueError( f"`names` gives {len(names)} names for a table of {width} columns. Give one per column, in order. " "[whyqd:names-count]" From c19466471d3ffc90d9f34ffe39871a67fd7ade5f Mon Sep 17 00:00:00 2001 From: Gavin Chait Date: Mon, 2 Nov 2026 20:00:41 +0100 Subject: [PATCH 7/7] Name columns past a short name list by position --- tests/test_header.py | 18 +++++++++++++++--- whyqd/core/datasource.py | 3 ++- whyqd/parsers/table.py | 14 ++++++++++---- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/tests/test_header.py b/tests/test_header.py index e59ebc0..98c13ea 100644 --- a/tests/test_header.py +++ b/tests/test_header.py @@ -111,11 +111,23 @@ def test_every_file_of_a_fixed_format_reads_to_one_shape(self): assert [c.name for c in datasource.get.columns] == names assert table(datasource)["field_0"].tolist()[0] == "0660" - def test_fewer_names_than_columns_are_refused(self, tmp_path): + def test_columns_past_the_names_are_named_by_position_and_nothing_shifts(self, tmp_path): datasource = qd.DataSourceDefinition() - source = write(tmp_path, "a,b,c\n1,2,3\n") - with pytest.raises(ValueError, match=r"\[whyqd:names-count\]"): + source = write(tmp_path, "a,b,c,d\n1,2,3,4\n") + with pytest.warns(UserWarning, match=r"\[whyqd:names-short\]"): datasource.derive_model(source=source, mimetype="CSV", dialect={"headerRowCount": 0}, names=["x", "y"]) + names = ["x", "y", "Unnamed: 2", "Unnamed: 3"] + assert [c.name for c in datasource.get.columns] == names + assert table(datasource)[names].values.tolist() == [["a", "b", "c", "d"], ["1", "2", "3", "4"]] + + def test_a_name_by_position_gives_way_to_a_name_given(self, tmp_path): + datasource = qd.DataSourceDefinition() + source = write(tmp_path, "a,b,c\n") + with pytest.warns(UserWarning, match=r"\[whyqd:names-short\]"): + datasource.derive_model( + source=source, mimetype="CSV", dialect={"headerRowCount": 0}, names=["Unnamed: 2", "y"] + ) + assert len({c.name for c in datasource.get.columns}) == 3 def test_a_header_over_two_rows_takes_the_names_given(self, tmp_path): datasource = qd.DataSourceDefinition() diff --git a/whyqd/core/datasource.py b/whyqd/core/datasource.py index 162353a..6287c01 100644 --- a/whyqd/core/datasource.py +++ b/whyqd/core/datasource.py @@ -121,7 +121,8 @@ def derive_model( describes it. Undeclared, each setting takes its default; a CSV's encoding, undeclared, is settled by whyqd's ladder and recorded. For a workbook, a list, each naming its `sheet`, or one a sheet in order. names: One name per column, for a file with no header row or a header over several rows; for a workbook, a - mapping of sheet to names. Names past the file's last filled column are added as empty columns. + mapping of sheet to names. Names past the file's last filled column are added as empty columns; + columns past the last name are named by position, with a warning. settings: Refused, naming the dialect: `header` and pandas' keywords are no longer read. Returns: diff --git a/whyqd/parsers/table.py b/whyqd/parsers/table.py index 4266198..9cd349c 100644 --- a/whyqd/parsers/table.py +++ b/whyqd/parsers/table.py @@ -1,4 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause +import warnings + import polars as pl from whyqd.models import DialectModel @@ -42,7 +44,8 @@ def table( names: list[str] | None, default None One name per column, in place of the header row's; required for a header over several rows. Names past the grid's last column are columns the file leaves empty, which the grid trims: each is added, empty, so every - file of a format with fixed fields reads to one shape. + file of a format with fixed fields reads to one shape. Columns past the last name are named by position, + with a warning, so no cell is lost and none moves. place: str, default "line" What `lines` counts, as the wrangler would look for it: a text file's `line`, a sheet's `row`. beside: list[pl.DataFrame] | None, default None @@ -96,10 +99,13 @@ def names(self, *, header: pl.DataFrame, names: list[str] | None, dialect: Diale width = header.width if names is not None: if len(names) < width: - raise ValueError( - f"`names` gives {len(names)} names for a table of {width} columns. Give one per column, in order. " - "[whyqd:names-count]" + warnings.warn( + f"`names` gives {len(names)} names for a table of {width} columns, so the rest are named by " + "position. Check the list, and give one per column, in order. [whyqd:names-short]", + UserWarning, + stacklevel=2, ) + names = [*names, *(f"Unnamed: {i}" for i in range(len(names), width))] elif dialect.headerRowCount > 1: raise ValueError( f"`headerRowCount` is {dialect.headerRowCount}, so declare the names in `names`, one for each of its "