diff --git a/.gitattributes b/.gitattributes index 1170ad1..05fef8c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -13,3 +13,8 @@ *.jpg binary *.jpeg binary *.ico binary +*.pdf binary +*.PDF binary + +# Test data keeps its bytes: line endings are among what the readers are tested on +tests/data/** -text diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index ddbe4dc..9824a94 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -32,7 +32,8 @@ jobs: sudo locale-gen --keep-existing locale -a | grep -qi '^de_DE\.utf8$' - name: Install dependencies - run: uv sync --locked + # every extra, so the tests of each one run + run: uv sync --locked --all-extras - name: Lint with ruff run: | # stop the build if there are Python syntax errors or undefined names diff --git a/.gitignore b/.gitignore index af757cd..6c5a47b 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,7 @@ __pycache__/ .pytest_cache/ variations/ sample/ +!tests/data/sample/ site/ .ipynb_checkpoints/ *.ipynb diff --git a/docs/strategies/crosswalk.md b/docs/strategies/crosswalk.md index 8644601..e3a1e01 100644 --- a/docs/strategies/crosswalk.md +++ b/docs/strategies/crosswalk.md @@ -145,14 +145,15 @@ could be used to indicate categories, or text for splitting. A script `modifier` specific. `row` terms are row numbers, meaning they must be in the source table or the script will fail. Rows are numbered from -`0` by their position in the source as read, and each sheet starts again at `0`. With the header at row `0` (the -default), row `0` is the first row after it; with `header=None`, another header row, or `names`, row `0` is the file's -first row. Deleting rows leaves the rest numbered as they were. - -!!! warning "Blank lines in a CSV are not numbered" - A blank line in a CSV is skipped at import and not counted, so each one moves every later row's number up by one. - Blank rows in an Excel sheet are kept and numbered. Take row numbers from the table **whyqd** shows you, not from - a text editor's line numbers. +`0` by their position in the table as read, and each sheet starts again at `0`. The header row, and any rows skipped +above it, are not in the table: row `0` is the first row after the header, or the file's first row where the data +source's `dialect` declares no header row (`headerRowCount` 0). Deleting rows leaves the rest numbered as they were. + +!!! warning "Row numbers are not line numbers" + A blank line in a CSV is a row, numbered like any other, and a cell holding a line break spans two lines of the file + in one row. Take row numbers from the table **whyqd** shows you, not from a text editor's line numbers. A data + source saved by **whyqd** 1.x keeps 1.x's numbering, in which a CSV's blank lines were skipped; where that changes a + row's number, reading it warns. !!! abstract "Tutorials" It is assumed that you're not working 'blind', that you're actually looking at your data while assigning actions - diff --git a/pyproject.toml b/pyproject.toml index 21849db..ee0a4b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,19 +22,23 @@ classifiers = [ "Topic :: Scientific/Engineering", ] dependencies = [ + "fastexcel>=0.21.0,<0.22", "numpy>=2.4.6,<3", - "openpyxl>=3.1.5,<4", "packaging>=26.3,<27", "pandas>=3.0.6,<4", + "polars>=2.0.0,<3", "pyarrow>=24,<27", "pydantic>=2.13.4,<3", "pydantic-settings>=2.14.1,<3", + "python-calamine>=0.9.0,<0.10", "randomname>=0.2.1,<0.3", "rfc8785>=0.1.4,<0.2", - "xlrd>=2.0.2,<3", ] [project.optional-dependencies] +pdf = [ + "pdfplumber>=0.11.10,<0.12", +] verify-1x = [ "openpyxl>=3.1.5,<4", "pandas>=3.0.6,<4", @@ -52,6 +56,7 @@ dev = [ "ruff~=0.14.0", "pytest>=7.2.2", "jsonschema>=4.26", + "openpyxl>=3.1.5,<4", ] notebooks = [ "jupyterlab>=3.6.1", diff --git a/tests/data/builders/fixtures.py b/tests/data/builders/fixtures.py index a93ce21..8090664 100644 --- a/tests/data/builders/fixtures.py +++ b/tests/data/builders/fixtures.py @@ -16,6 +16,7 @@ from openpyxl import Workbook from openpyxl.cell.cell import TYPE_ERROR +from workbook import save HERE = Path(__file__).resolve().parent TARGET = Path(sys.argv[1]).resolve() if len(sys.argv) > 1 else HERE.parent / "synthetic" @@ -135,7 +136,7 @@ def main(): key["workbook"]["Hidden"] = {"state": "hidden", "rows": [["secret"], ["kept"]]} wb.create_sheet("Empty") key["workbook"]["Empty"] = {"rows": []} - wb.save(TARGET / "fixture.xlsx") + save(wb, TARGET / "fixture.xlsx") key["workbook_file"] = "fixture.xlsx" (TARGET / "answer-key.json").write_text(json.dumps(key, indent=2, ensure_ascii=False, default=str), encoding="utf-8") diff --git a/tests/data/builders/layouts.py b/tests/data/builders/layouts.py new file mode 100644 index 0000000..fbce7e3 --- /dev/null +++ b/tests/data/builders/layouts.py @@ -0,0 +1,470 @@ +# SPDX-License-Identifier: BSD-3-Clause +"""Write the synthetic layout fixtures, and the answer key that says how each should load. + + python tests/data/builders/layouts.py [target] + +Target defaults to tests/data/synthetic-layouts, and the script refuses to overwrite it. Each file plants the layout +problems found in published spreadsheets, on records borrowed from tests/data/sample/pdf/west-berkshire-reliefs.csv +with the defects real files carry added (dates beside text, `N/A`, whitespace). + +The key is the contract. For each file or sheet: `lines` (CSV), each row's fields as written; `dialect`, the settings +that read it as intended, which a wrangler declares and diagnosis should propose; `names` and `rows`, the table those +settings give, each row padded to the names; and `propose`, what diagnosis should find beyond the dialect. Workbook +cells are given as stored, a date as its ISO text, with `date_cells` naming them. Two files are finished by hand in +LibreOffice: layouts.xlsx saved as layouts.xls, and pdf-source.xlsx exported as pdf-source.pdf, sheet by sheet. +""" + +import json +import sys +from datetime import date, datetime +from pathlib import Path + +from openpyxl import Workbook +from openpyxl.cell.cell import TYPE_ERROR +from openpyxl.styles import Alignment, Border, Side +from openpyxl.utils import get_column_letter +from openpyxl.worksheet.properties import PageSetupProperties +from workbook import save + +HERE = Path(__file__).resolve().parent +TARGET = Path(sys.argv[1]).resolve() if len(sys.argv) > 1 else HERE.parent / "synthetic-layouts" + +HEADER = ["Propref", "Property Address", "Ret Code", "R/V", "Start Date", "Contact"] +# Start Date as text; in a workbook the first and third are date cells, as Excel leaves a day-first column it could +# partly read (the Stockport case). Contact carries N/A, a lone space, an empty cell, and padding. +RECORDS = [ + ["NN00101700004185", "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", "SBR3", "13250", + "04/01/2024", "N/A"], + ["NN03905740027005", "27-31, Meadow Road, Newbury, RG14 7AH", "SBRE", "10250", "13/04/2024", " "], + ["NN04105100003115", "At Station Road, Pangbourne, Reading, RG8 7AN", "RH0", "8600", "05/02/2024", + "Parish Clerk"], + ["NN0010170000401A", "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", "SBR1", "16250", + "25/12/2024", ""], + ["NN0010420000111A", "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", "MAN", + "5400", "01/02/2026", "N/A"], + ["NN00201200001117", "The Bell Inn, Aldworth, Reading, RG8 9SE", "PU15", "24000", "30/01/2026", " Landlord "], +] +DATE_CELLS = {0: date(2024, 1, 4), 2: date(2024, 2, 5)} +CODES = [["Code", "Description"], ["SBR3", "Small Business Rate Relief"], ["RH0", "Retail, Hospitality and Leisure"], + ["MAN", "Mandatory Charity Relief"]] +FOOTNOTES = [["Source: business rates system, extracted 15/04/2026"], ["* R/V is the rateable value in pounds"]] +# Each a name a script must still resolve to one field: quotes, brackets, comparison signs, a comma, padding, a line +# break, an empty name, a repeat, one name inside another, and the row key's own name. +HOSTILE = ["Owner's name", 'The "Main" Rate', "Rate [2026]", "R/V > 0", "", "Code, Local", " padded ", + "line\nbreak", "", "Rate", "Rate", "Rate Relief", "__row__"] +MISSING_LIKE = ["N/A", "NA", "null", "NaN", "#N/A", "n/a", "-", "None"] + + +def field(value, delimiter=","): + value = "" if value is None else value + if any(c in value for c in f'{delimiter}"\n\r') or value != value.strip(): + return '"' + value.replace('"', '""') + '"' + return value + + +def csv_text(lines, delimiter=",", eol="\r\n"): + return eol.join(delimiter.join(field(v, delimiter) for v in line) for line in lines) + eol + + +def padded(rows, width): + return [row + [""] * (width - len(row)) for row in rows] + + +def csv_cases(): + """Each case: its file's lines, the text written (where it is not csv_text of the lines), and its key.""" + width = len(HEADER) + cases = {} + + lines = [["Business rates reliefs, April 2026"], [], [], HEADER, *RECORDS] + cases["title-above.csv"] = (lines, None, { + "problem": "a title and two blank lines above the header", + "dialect": {"skipRows": 3}, "names": HEADER, "rows": RECORDS, + "propose": {"skipRows": 3, "why": "the first line with as many fields as the data is line 3"}}) + + lines = [HEADER, *RECORDS[:3], [], *RECORDS[3:]] + cases["blank-line-inside.csv"] = (lines, None, { + "problem": "a blank line inside the data, kept as a row and numbered", + "dialect": {}, "names": HEADER, "rows": padded([*RECORDS[:3], [], *RECORDS[3:]], width), + "propose": {"blank_rows": [3]}}) + + records = [r.copy() for r in RECORDS] + records[1][1] = "27-31, Meadow Road\nNewbury, RG14 7AH" + lines = [HEADER, *records] + cases["quoted-line-break.csv"] = (lines, None, { + "problem": "a quoted cell holding a line break, one record over two lines of the file", + "dialect": {}, "names": HEADER, "rows": records}) + + lines = [HEADER, *RECORDS] + cases["semicolon.csv"] = (lines, csv_text(lines, delimiter=";"), { + "problem": "`;` between cells, and commas inside the addresses unquoted", + "dialect": {"delimiter": ";"}, "names": HEADER, "rows": RECORDS, + "propose": {"delimiter": ";"}}) + + # No cell is quoted, so the addresses lose their commas; one cell opens with a quote that never closes + records = [[v.replace(",", "") for v in r] for r in RECORDS] + records[2][1] = '"Old Mill Station Road Pangbourne' + lines = [HEADER, *records] + text = "\r\n".join(",".join(line) for line in lines) + "\r\n" + cases["stray-quote.csv"] = (lines, text, { + "problem": "a cell opening with a quote that never closes, which swallows the rest of the file if read as a " + "quote (1.x needed `quoting=csv.QUOTE_NONE`)", + "dialect": {"quoteChar": None}, "names": HEADER, "rows": records, + "propose": {"quoteChar": None, "why": "an opening quote with no close before the end of the file"}}) + + lines = [[*line, ""] for line in [HEADER, *RECORDS]] + cases["trailing-delimiter.csv"] = (lines, None, { + "problem": "every line ends with a delimiter, an empty last column with an empty header", + "dialect": {}, "names": HEADER, "rows": RECORDS, + "propose": {"why": "the empty last column is trimmed with the grid, as fp0 trims it"}}) + + # Each record over two lines of the file, as every line has the same fields: the second holds one value + first = HEADER[:5] + lines = [first, [HEADER[5], "", "", "", ""]] + for r in RECORDS: + lines += [r[:5], [r[5], "", "", "", ""]] + cases["two-line-records.csv"] = (lines, None, { + "problem": "each record, header included, over two lines: five fields, then one in the first field", + "dialect": {"rowsPerRecord": [5, 1]}, "names": HEADER, "rows": RECORDS, + "propose": {"rowsPerRecord": [5, 1], "why": "every second line has one value, in its first field"}, + "refuse": "a record without its second line, or a second line with values past its first field"}) + + # The same records with nothing after a record's one value: a lone empty value is a blank line + lines = [first, [HEADER[5]]] + for r in RECORDS: + lines += [r[:5], [r[5]]] + cases["two-line-records-unpadded.csv"] = (lines, None, { + "problem": "each record over two lines, the second holding its one value and no delimiters, so an empty " + "value is a blank line", "dialect": {"rowsPerRecord": [5, 1]}, "names": HEADER, "rows": RECORDS, + "propose": {"rowsPerRecord": [5, 1], "why": "every second line has at most one value"}}) + + records = [r.copy() for r in RECORDS] + records[2][1] = 'The "Old" Mill, Station Road, Pangbourne, Reading, RG8 7AN' + lines = [HEADER, *records] + + def escaped(value): + if any(c in value for c in ',"\n\r') or value != value.strip(): + return '"' + value.replace('"', '\\"') + '"' + return value + + text = "".join(",".join(escaped(v) for v in line) + "\r\n" for line in lines) + cases["backslash-escape.csv"] = (lines, text, { + "problem": "a quote inside a quoted cell escaped by a backslash, not doubled", + "dialect": {"doubleQuote": False}, "names": HEADER, "rows": records, + "propose": {"doubleQuote": False, "why": "a backslash before a quote inside a quoted cell"}}) + + lines = [HEADER, *RECORDS] + cases["cr-line-ends.csv"] = (lines, csv_text(lines, eol="\r"), { + "problem": "every line ending `\\r` alone, as old Mac files do", "dialect": {}, "names": HEADER, + "rows": RECORDS}) + + lines = [HEADER[:5], *RECORDS] + cases["header-narrower.csv"] = (lines, None, { + "problem": "the header a field narrower than the data, so the last column has no name", + "dialect": {}, "names": [*HEADER[:5], "Unnamed: 5"], "rows": RECORDS, + "renamed": "the column past the header by position, as an empty name is"}) + + lines = [RECORDS[0][:5], *RECORDS[1:]] + cases["first-row-narrower.csv"] = (lines, None, { + "problem": "no header, and the first row a field narrower than the rest", + "dialect": {"headerRowCount": 0}, "names": [f"column_{i}" for i in range(width)], + "rows": padded(lines, width)}) + + lines = [HEADER, *RECORDS[:2], RECORDS[2][:4], *RECORDS[3:]] + cases["row-narrower.csv"] = (lines, None, { + "problem": "one row two fields narrower than the header, its last cells empty", + "dialect": {}, "names": HEADER, "rows": padded(lines[1:], width)}) + + rows = [[f"v{i}{j}" for j in range(len(HOSTILE))] for i in range(2)] + lines = [HOSTILE, *rows] + cases["hostile-headers.csv"] = (lines, None, { + "problem": "header names a script must still resolve: every name read exactly, none trimmed or joined", + "dialect": {}, "names_as_written": HOSTILE, "rows": rows, + "names": [f"Unnamed: {i}" if n == "" else n for i, n in enumerate(HOSTILE[:-3])] + ["Rate.1", "Rate Relief", + "__row__.1"], + "renamed": "the empty name by position, as stored schemas name it; the repeat and the row key's own name " + "by RowKeyParser.deconflict"}) + + rows = [[v, str(i)] for i, v in enumerate(MISSING_LIKE)] + lines = [["Contact", "n"], *rows] + cases["missing-like-text.csv"] = (lines, None, { + "problem": "text pandas reads as missing by default; every one is text, and only the source schema can " + "say it means missing", "dialect": {}, "names": ["Contact", "n"], "rows": rows}) + return cases + + +def stored(value): + return value.isoformat() if isinstance(value, (date, datetime)) else value + + +def write_records(ws, top, records=RECORDS, header=HEADER, dates=True): + """Write a header and records from row `top`, and return the coordinates of the date cells.""" + date_cells = [] + for c, name in enumerate(header, start=1): + ws.cell(row=top, column=c, value=name) + for i, record in enumerate(records): + for c, value in enumerate(record, start=1): + if dates and c == 5 and i in DATE_CELLS: + cell = ws.cell(row=top + 1 + i, column=c, value=DATE_CELLS[i]) + cell.number_format = "dd/mm/yyyy" + date_cells.append(cell.coordinate) + else: + ws.cell(row=top + 1 + i, column=c, value=value if value != "" else None) + return date_cells + + +def workbook_rows(records=RECORDS): + """The records as a workbook stores them: two dates as date cells, empty text as an empty cell.""" + rows = [] + for i, record in enumerate(records): + row = [v if v != "" else None for v in record] + if i in DATE_CELLS: + row[4] = stored(DATE_CELLS[i]) + rows.append(row) + return rows + + +def layouts_workbook(): + wb = Workbook() + key = {} + + ws = wb.active + ws.title = "Titled" + ws["A1"], ws["A2"] = "West Berkshire Council", "Business rates reliefs, April 2026" + dates = write_records(ws, top=4) + key["Titled"] = {"problem": "a title and subtitle, then a blank row, above the header on row 4", + "dialect": {"skipRows": 3}, "names": HEADER, "rows": workbook_rows(), "date_cells": dates, + "propose": {"skipRows": 3}} + + ws = wb.create_sheet("MergedHeader") + ws["A1"], ws["B1"], ws["D1"] = "Propref", "Property", "Relief" + ws.merge_cells("A1:A2") + ws.merge_cells("B1:C1") + ws.merge_cells("D1:E1") + for c, name in enumerate(["Address", "Postcode", "Code", "R/V"], start=2): + ws.cell(row=2, column=c, value=name) + records = [[r[0], r[1].rsplit(", ", 1)[0], r[1].rsplit(", ", 1)[1], r[2], r[3]] for r in RECORDS] + for i, record in enumerate(records, start=3): + for c, value in enumerate(record, start=1): + ws.cell(row=i, column=c, value=value) + key["MergedHeader"] = { + "problem": "a header over two rows: one name merged down both, two merged across two columns each", + "dialect": {"headerRowCount": 2}, "header_rows": [["Propref", "Property", None, "Relief", None], + [None, "Address", "Postcode", "Code", "R/V"]], + "merged": ["A1:A2", "B1:C1", "D1:E1"], "rows": records, + "names": "supplied by the wrangler, one per column: a merged cell over two may be one title or a heading " + "over subheadings, so whyqd never builds a name from it", + "refuse": "headerRowCount 2 without names: the header rows and merged ranges named, and names asked for", + "propose": {"headerRowCount": 2, "why": "merged ranges in rows 1 and 2, and data from row 3"}} + + ws = wb.create_sheet("BlankColumns") + names = HEADER[:2] + [None] + HEADER[2:] + for c, name in enumerate(names, start=1): + ws.cell(row=1, column=c, value=name) + for i, record in enumerate(RECORDS, start=2): + for c, value in enumerate(record[:2] + [None] + record[2:], start=1): + ws.cell(row=i, column=c, value=value if value != "" else None) + border = Border(left=Side(style="thin"), right=Side(style="thin")) + for i in range(1, len(RECORDS) + 2): + for c in range(len(names) + 1, len(names) + 4): + ws.cell(row=i, column=c).border = border + key["BlankColumns"] = { + "problem": "an empty column inside the table, and three empty but formatted columns after it", + "dialect": {}, "names": [*HEADER[:2], "Unnamed: 2", *HEADER[2:]], + "rows": [[v if v != "" else None for v in r[:2] + [None] + r[2:]] for r in RECORDS], + "propose": {"empty_columns": [2], "why": "the trailing columns hold formatting only, and are trimmed"}} + + ws = wb.create_sheet("SubTables") + write_records(ws, top=1, dates=False) + for r, row in enumerate(CODES, start=len(RECORDS) + 3): + for c, value in enumerate(row, start=1): + ws.cell(row=r, column=c, value=value) + foot = len(RECORDS) + len(CODES) + 4 + for r, row in enumerate(FOOTNOTES, start=foot): + ws.cell(row=r, column=1, value=row[0]) + rows = [*RECORDS, [""] * 6, *padded(CODES, 6), [""] * 6, *padded(FOOTNOTES, 6)] + key["SubTables"] = { + "problem": "a second table below a blank row, then footnotes; loaded as given, every row is in the table", + "dialect": {}, "names": HEADER, "rows": [[v if v != "" else None for v in r] for r in rows], + "propose": {"table_rows": [0, len(RECORDS) - 1], + "second_table": {"header_row": len(RECORDS) + 1, "rows": len(CODES) - 1}, + "footnotes": [len(RECORDS) + len(CODES) + 2, len(rows) - 1]}} + + ws = wb.create_sheet("Errors") + write_records(ws, top=1, dates=False) + errors = {"D3": "#N/A", "D5": "#DIV/0!", "D7": "#REF!"} + for coordinate, value in errors.items(): + ws[coordinate] = value + ws[coordinate].data_type = TYPE_ERROR + rows = [[v if v != "" else None for v in r] for r in RECORDS] + for coordinate, value in errors.items(): + rows[int(coordinate[1:]) - 2][3] = value + key["Errors"] = {"problem": "error cells in a number column", "dialect": {}, "names": HEADER, "rows": rows, + "error_cells": errors} + + ws = wb.create_sheet("Workings") + ws.sheet_state = "hidden" + ws["A1"], ws["A2"] = "Total R/V", "=SUM(Titled!D5:D10)" + key["Workings"] = {"problem": "a hidden sheet, read like any other and flagged", "state": "hidden", "dialect": {}, + "names": ["Total R/V"], "rows": [], + "formula_cells": {"A2": "a formula written without its cached value, so the row is empty " + "here and trimmed; the .xls, saved by LibreOffice, caches 0"}} + return wb, key + + +def single_sheet_workbook(): + wb = Workbook() + ws = wb.active + ws.title = "Reliefs" + for c, name in enumerate(CODES[0], start=1): + ws.cell(row=1, column=c, value=name) + for r, row in enumerate(CODES[1:], start=3): + for c, value in enumerate(row, start=1): + ws.cell(row=r, column=c, value=value) + key = {"problem": "the only sheet, with row 2 blank: 1.x's one-row probe judged it empty and dropped it", + "dialect": {}, "names": CODES[0], "rows": [[None, None], *CODES[1:]], "propose": {"blank_rows": [0]}} + return wb, key + + +def printed(rows): + """Rows as a PDF prints them: a cell's spaces at either end are not kept.""" + return [[v.strip() for v in row] for row in rows] + + +def pdf_workbook(): + """Sheets for LibreOffice to export as PDF: the kinds of table the sample's PDFs lack.""" + wb = Workbook() + key = {} + thin = Side(style="thin") + grid = Border(left=thin, right=thin, top=thin, bottom=thin) + + def fit(ws): + # A PDF prints a cell only as wide as its column, so each column fits its longest value + for i, column in enumerate(ws.columns, start=1): + longest = max(len(str(c.value)) if c.value is not None else 0 for c in column) + ws.column_dimensions[get_column_letter(i)].width = max(longest, 10) + 2 + # One page wide, as many long as the rows need, so no table's columns are split across pages + ws.page_setup.fitToWidth = 1 + ws.page_setup.fitToHeight = 0 + ws.sheet_properties.pageSetUpPr = PageSetupProperties(fitToPage=True) + + def rule(ws): + fit(ws) + for row in ws.iter_rows(): + for cell in row: + if cell.value is not None or cell.coordinate in ws.merged_cells: + cell.border = grid + + ws = wb.active + ws.title = "Ruled" + write_records(ws, top=1) + rule(ws) + ws.column_dimensions["B"].width = 30 + for row in ws.iter_rows(min_col=2, max_col=2): + row[0].alignment = Alignment(wrap_text=True) + key["Ruled"] = {"problem": "a ruled grid, long addresses wrapped over several lines in their cells", + "names": HEADER, "rows": printed(RECORDS), + "note": "a date cell exports as its displayed text, dd/mm/yyyy, so the rows are the text", + "wrapped": "an address extracts with a line break at each wrap, and keeps it: a wrap and a line " + "break in the cell look the same in a PDF, so joining is a declared step, never the " + "reader's; the ruling keeps each wrapped cell in its one row"} + + ws = wb.create_sheet("MergedHeader") + ws["A1"], ws["B1"], ws["D1"] = "Propref", "Property", "Relief" + ws.merge_cells("A1:A2") + ws.merge_cells("B1:C1") + ws.merge_cells("D1:E1") + for c, name in enumerate(["Address", "Postcode", "Code", "R/V"], start=2): + ws.cell(row=2, column=c, value=name) + records = [[r[0], r[1].rsplit(", ", 1)[0], r[1].rsplit(", ", 1)[1], r[2], r[3]] for r in RECORDS] + for i, record in enumerate(records, start=3): + for c, value in enumerate(record, start=1): + ws.cell(row=i, column=c, value=value) + rule(ws) + key["MergedHeader"] = {"problem": "a ruled header over two rows with merged cells", + "header_rows": [["Propref", "Property", "", "Relief", ""], + ["", "Address", "Postcode", "Code", "R/V"]], "rows": records, + "names": "supplied by the wrangler, as layouts.xlsx § MergedHeader"} + + ws = wb.create_sheet("TwoTables") + write_records(ws, top=1, dates=False) + for r, row in enumerate(CODES, start=len(RECORDS) + 4): + for c, value in enumerate(row, start=1): + ws.cell(row=r, column=c, value=value) + fit(ws) + key["TwoTables"] = {"problem": "two tables on one page, unruled, two blank rows apart", + "tables": [{"names": HEADER, "rows": printed(RECORDS)}, + {"names": CODES[0], "rows": CODES[1:]}]} + + ws = wb.create_sheet("Landscape") + wide = [*HEADER, "Liable From", "Account Holder", "Ward", "Band"] + extra = [["01/04/2010", "Orpheus Holdings Ltd", "Aldermaston", "B"], + ["01/04/2017", "Meadow Road Traders", "Newbury Central", "C"], + ["12/06/2019", "Pangbourne Station Ltd", "Pangbourne", "A"], + ["01/04/2023", "Andromeda Estates", "Aldermaston", "B"], + ["01/04/2005", "Aldermaston Parish Council", "Aldermaston", "A"], + ["30/09/2016", "The Bell Inn (Aldworth) Ltd", "Basildon", "B"]] + wide_rows = [r + e for r, e in zip(RECORDS, extra, strict=True)] + write_records(ws, top=1, records=wide_rows, header=wide, dates=False) + ws.page_setup.orientation = "landscape" + rule(ws) + key["Landscape"] = {"problem": "a wide table on a landscape page", "names": wide, + "rows": printed(wide_rows)} + + many = [[f"NN{900000000000 + i:014d}", r[1], r[2], r[3], r[4], r[5]] + for i, r in enumerate(RECORDS * 25)] + for title, repeat in (("RepeatedHeader", True), ("HeaderOnce", False)): + ws = wb.create_sheet(title) + write_records(ws, top=1, records=many, dates=False) + if repeat: + ws.print_title_rows = "1:1" + rule(ws) + key[title] = {"problem": f"{len(many)} rows over several pages, the header " + + ("repeated on every page" if repeat else "on the first page only"), + "names": HEADER, "rows": printed(many)} + + ws = wb.create_sheet("UnruledWrapped") + write_records(ws, top=1) + fit(ws) + ws.column_dimensions["B"].width = 30 + for row in ws.iter_rows(min_col=2, max_col=2): + row[0].alignment = Alignment(wrap_text=True) + key["UnruledWrapped"] = { + "problem": "no ruling, and long addresses wrapped in their cells: read by text position, each wrapped line " + "becomes a row of its own and words split across columns", + "names": HEADER, "rows": printed(RECORDS), + "report": "loaded by the default reading, the rows that look like continuations (an empty first cell, text " + "only where a cell wraps) are shown, so the wrangler can declare how the table is read", + "declared": "with the table's reading declared (its columns' bounds, which rows continue a record), the rows " + "above, each wrap a line break in its cell"} + return wb, key + + +def main(): + assert not TARGET.exists(), f"{TARGET} exists; fixtures are never overwritten" + TARGET.mkdir(parents=True) + key = {"generated": datetime.now().isoformat(timespec="seconds"), "csv": {}, "workbook": {}} + + for name, (lines, text, entry) in csv_cases().items(): + (TARGET / name).write_bytes((text or csv_text(lines)).encode("utf-8")) + key["csv"][name] = {"lines": lines, **entry} + + wb, sheets = layouts_workbook() + save(wb, TARGET / "layouts.xlsx") + key["workbook"]["layouts.xlsx"] = sheets + key["workbook"]["layouts.xls"] = "layouts.xlsx saved as .xls in LibreOffice; read to the same key" + wb, sheet = single_sheet_workbook() + save(wb, TARGET / "single-sheet.xlsx") + key["workbook"]["single-sheet.xlsx"] = {"Reliefs": sheet} + wb, sheets = pdf_workbook() + save(wb, TARGET / "pdf-source.xlsx") + key["pdf"] = {"pdf-source.pdf": sheets, + "made": "pdf-source.xlsx exported in LibreOffice, every sheet, one PDF; each sheet's table " + "read across its pages"} + + (TARGET / "answer-key.json").write_text(json.dumps(key, indent=2, ensure_ascii=False, default=str), + encoding="utf-8") + print(sorted(p.name for p in TARGET.iterdir())) + + +if __name__ == "__main__": + main() diff --git a/tests/data/builders/workbook.py b/tests/data/builders/workbook.py new file mode 100644 index 0000000..c961060 --- /dev/null +++ b/tests/data/builders/workbook.py @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: BSD-3-Clause +"""Save a workbook as Excel and LibreOffice save one: text with leading or trailing whitespace marked to keep it. + +openpyxl writes ` ` where Excel writes ` `, and a reader may drop whitespace that +is not marked, as python-calamine does, so a fixture saved by openpyxl alone would test a file no spreadsheet writes. + + python tests/data/builders/workbook.py ... + +marks workbooks already saved, in place. +""" + +import re +import sys +import zipfile +from pathlib import Path + +UNMARKED = re.compile(rb"(\s[^<]*|[^<]*\s)") + + +def mark(path: Path) -> None: + """Mark the whitespace in a saved workbook's text, writing it anew beside it and then moving it into place.""" + staged = path.with_suffix(".marking") + with zipfile.ZipFile(path) as source, zipfile.ZipFile(staged, "w", zipfile.ZIP_DEFLATED) as marked: + for item in source.infolist(): + data = source.read(item) + if item.filename.startswith("xl/worksheets/") or item.filename == "xl/sharedStrings.xml": + data = UNMARKED.sub(rb'\1', data) + marked.writestr(item, data) + staged.replace(path) + + +def save(workbook, path: Path) -> None: + workbook.save(path) + mark(path) + + +if __name__ == "__main__": + for name in sys.argv[1:]: + mark(Path(name)) diff --git a/tests/data/sample/amber-valley/amber-valley-source.XLSX b/tests/data/sample/amber-valley/amber-valley-source.XLSX new file mode 100644 index 0000000..4f82842 Binary files /dev/null and b/tests/data/sample/amber-valley/amber-valley-source.XLSX differ diff --git a/tests/data/sample/amber-valley/amber-valley.TRANSFORM b/tests/data/sample/amber-valley/amber-valley.TRANSFORM new file mode 100644 index 0000000..6e5a137 --- /dev/null +++ b/tests/data/sample/amber-valley/amber-valley.TRANSFORM @@ -0,0 +1 @@ +{"uuid":"08f36e02-7cdd-452e-bc9d-96a732eaa388","name":"giant-vault","title":"Amber Valley","dataSource":{"uuid":"8bbbf551-fddb-44bf-91af-8db01c475861","title":"Amber Valley","path":"/app/working/references/41192e19-c6f5-4e72-afdc-1428ac2cf4b4.XLSX","mime":"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet","columns":[{"uuid":"1bb2c11d-583e-4474-8e2c-f29b3f972468","name":"Property address"},{"uuid":"8ae52698-8069-4802-bc12-2d2402443afe","name":"Liablefrom"},{"uuid":"6928fc05-16e0-4a9e-8da7-d96d6ee15d3f","name":"Emptyfrom"},{"uuid":"5ccf4f22-dee9-4b9b-8455-b5ac84996879","name":"Description"},{"uuid":"42733594-16d0-4e50-a571-cf7865b0da51","name":"Account holder address"},{"uuid":"bf1bca8e-aaa4-404f-9d78-beafdcf12593","name":"Charge amount"},{"uuid":"985eb97f-8d84-4076-b8f1-3a5fc8b0c4ab","name":"Rateable value"},{"uuid":"94b04048-0ead-427f-abda-bf48915bd6fa","name":"Reference"},{"uuid":"a314bf10-3a92-4ec2-afa6-c4a83ab75dc5","name":"Relief type"},{"uuid":"8197b51a-3fa4-4646-9061-6634ce33e7ef","name":"Relief value"},{"uuid":"384bd9fc-1e28-444b-91dd-ca77652be556","name":"Exemption type"},{"uuid":"dfa1bd0a-dde9-4d30-b3cc-e4db471eb1bb","name":"Exemption date"}],"preserve":["Property address","Liablefrom","Emptyfrom","Description","Account holder address","Charge amount","Rateable value","Reference","Relief type","Relief value","Exemption type","Exemption date"],"attributes":{},"checksum":"16561860ff0adef956d6c1cd28a73b245e0ae2ae262e3364895cc081ca2e1dc26acf61202a0397c2c6ad184a648e20754fb7048a0bed8b2b74df68c0e0de6d72","index":7372},"crosswalk":{"uuid":"57da6e7a-fb27-47b1-b41b-e2974cb90aca","name":"e07000032","title":"Amber Valley","schemaSource":{"uuid":"fa7df855-26ea-43b4-b535-5b4c47fc2291","name":"e07000032","title":"Amber Valley","fields":[{"uuid":"c0226c5e-3ccf-4f70-a324-dc4025c9f797","name":"Property address","title":"Property address","type":"string"},{"uuid":"a81bc2e7-4622-44d5-ab14-69ea222f3848","name":"Liablefrom","title":"Liablefrom","type":"string"},{"uuid":"054705b1-4d14-46b7-a14f-3c523cf9b4ba","name":"Emptyfrom","title":"Emptyfrom","type":"date","constraints":{"enum":[{"uuid":"b0ca9d40-fb22-4fec-8c43-3466c21afd16","name":true},{"uuid":"820346fb-4e9f-40d0-98b9-1f398f8294fb","name":false}]}},{"uuid":"240f411b-9bc9-4ef4-be55-c31939ad29d1","name":"Description","title":"Description","type":"string"},{"uuid":"9424495f-6e6c-4d5c-bb12-13e7af8e86aa","name":"Account holder address","title":"Account holder address","type":"string"},{"uuid":"8cff02db-326e-4542-b6ff-be4413842c5b","name":"Charge amount","title":"Charge amount","type":"string"},{"uuid":"45225b19-1563-4d8f-8d08-99c49db4d9c6","name":"Rateable value","title":"Rateable value","type":"string"},{"uuid":"8814e1d8-7aee-4917-9a63-5f1f45c107a0","name":"Reference","title":"Reference","type":"string"},{"uuid":"4653f456-f8f6-4746-b8b3-1ebb98d9b147","name":"Relief type","title":"Relief type","type":"string","constraints":{"enum":[{"uuid":"427c3384-e43b-42ad-a35e-f36f6697d195","name":"Trans Prem"},{"uuid":"bc0dc340-3c19-403b-a78b-42a9e5e72555","name":"Mandatory"},{"uuid":"1f43ec11-d4a3-4e5f-8c3a-9cfed68b1d73","name":"Discretionary"},{"uuid":"801e4543-f9b8-4aa7-a21b-e7a9f8bd6ee2","name":"SBRR"},{"uuid":"8a839dd5-baaf-4bfd-99ed-96da28d3d24e","name":"Transitional"}]}},{"uuid":"7e67bbe7-cacb-454f-91e5-3456667b6c34","name":"Relief value","title":"Relief value","type":"string"},{"uuid":"0a444276-0ddb-43e6-a236-7edc48a5cdc6","name":"Exemption type","title":"Exemption type","type":"string","constraints":{"enum":[{"uuid":"3956f29c-dce0-459a-824b-a942c0ae43a4","name":"Listed Buildings"},{"uuid":"86ee4eb7-3fef-4394-952e-e48c941e88c4","name":"Land"},{"uuid":"567563d0-a297-44a2-ad50-b7e85cf3147e","name":"Telecommunication Masts"},{"uuid":"3c7e8492-db7f-4627-8c2f-9c3c30c154a4","name":"Charities"},{"uuid":"5d757eb8-b783-4483-a0ab-4fd609bbaa18","name":"CASCs"}]}},{"uuid":"e7b74af6-dded-47a4-ab93-5db4cb6c256a","name":"Exemption date","title":"Exemption date","type":"string"}],"index":4819,"version":[{"name":"redgine04@gmail.com","description":"Initial save of SchemaModel.","updated":"2024-06-20T12:26:17.901487"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2024-06-21T03:07:17.143448"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2024-06-21T03:07:18.412815"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2024-06-21T03:08:08.053517"},{"name":"Gavin Chait ","description":"Update save of SchemaModel.","updated":"2025-07-26T15:30:11.762303"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2026-05-20T10:44:22.621099"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2026-05-20T10:44:24.092977"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2026-05-20T10:45:25.991498"}]},"schemaDestination":{"uuid":"507a0ccf-5735-4b3a-9b9e-2e7842921e4a","name":"england-and-wales-nondomestic-rates-schema","title":"England and Wales non-domestic rates schema","description":"Based on the The Open Data Collaboration group at Open Innovations (Open Innovations, Bradford Council, Calderdale Council, and Leeds Council) which started a project to visualise business rates data. Although each of the three councils publish open data about their business rates, they each have different field titles and don't always include the same fields. This project aimed to standardise the way the data was shared to make it more usable and to get value (within a council and externally) from the datasets.\nhttps://open-innovations.org/projects/business-rates/format.html","fields":[{"uuid":"e1b1d480-8481-4281-98bd-0e59f7fdb949","name":"localAuthorityCode","title":"Local authority code","description":"Unique reference identifier for each local authority. Added from the Task 'name' unique identifier field.","type":"string","constraints":{"default":{"uuid":"a989d748-ff9a-4585-be82-d1876032f6a2","name":"E07000032"}}},{"uuid":"f42d609b-c8a6-4017-aa9d-d55eb62bd63b","name":"localBillingReference","title":"Billing reference","description":"Local Authority hereditament billing reference used by both VOA & LA to correspond with each other in reference to a specific hereditament.","type":"string","example":"8577090904","constraints":{"required":true,"enum":[]}},{"uuid":"7774382b-7054-4100-8bf1-ff9741a47d5a","name":"occupierAccountHolderName","title":"Occupier account-holder name","description":"Account-holder's names may be split across multiple columns. They are to be joined into one for post-processing.","type":"string","example":"A.M.C (North East) Ltd"},{"uuid":"02347dbe-d491-47cb-bfd4-2a3c28c4c861","name":"occupierRecordDate","title":"Date each row entered the record","type":"date"},{"uuid":"090b1d3d-3be0-4d39-b24b-8c97969eeb8f","name":"occupierAccountStartDate","title":"Start-date for account-holder","description":"Date the Account Holder became responsible for the hereditament. Provide date only if provided in data release from LA. e.g. \"01/01/23\", \"BLANK\" or \"Not Supplied\".","type":"date"},{"uuid":"cb30a0df-ea14-4c67-b76a-1ab358f167bd","name":"occupierPropertyAddress","title":"Address of the rateable property","description":"Address may be split across multiple columns. They are to be joined into one for post-processing with the postcode at the end.","type":"string"},{"uuid":"3ed85652-0831-4cd3-a005-a353b6cc51a1","name":"occupierCorrespondenceAddress","title":"Correspondence address","description":"Account-holders correspondence address may be split across multiple columns. They are to be joined into one for post-processing with the postcode at the end. This may be different from the hereditament address.","type":"string"},{"uuid":"6305545d-7caf-40fb-b421-2e483c3fbd96","name":"occupierOccupationState","title":"Occupation state","description":"Occupational status of hereditament. Only include if provided in LA data release.\nOccupied = \"Occupied\", Void = \"Void\", No Information = \"BLANK\", no empty data provided by LA = \"Not Supplied\"","type":"string","constraints":{"enum":[{"uuid":"83cd608f-ae55-4090-836d-154c2ec256d7","name":"Occupied","description":"Occupied"},{"uuid":"de15f74d-90c3-4f38-8821-37e3178787bd","name":"Vacant","description":"Vacant"}]}},{"uuid":"17c261f9-e837-49d1-abe1-fc63f2a80f86","name":"occupierOccupationDate","title":"Occupation state date","description":"To be used in combination with 'occupierOccupationState'. If only a date is provided (e.g. for only date of void, or date of occupation), then categorise the column as a boolean and use this to specify the state and the date simultaneously.","type":"date"},{"uuid":"e325315f-5ec0-416d-a470-4facfb7bb759","name":"occupierRatesPayable","title":"Actual rates paid","description":"Rates paid, net of reliefs (if available) in pounds. Provide rates paid amount only if provided in data release from LA. e.g. \"£55,000\", \"BLANK\", \"not supplied\".","type":"number"},{"uuid":"1dba84d1-501a-427f-88ad-771e2393516a","name":"occupierReliefType","title":"Relief or Exemption Received","description":"Type of relief or exemption on rates paid received.\n\nNOTE: Type, Date and Amount are arrays and the position in the array must correspond between these three columns. This is for both 'wide' data (where there are multiple columns in the source data for each class of relief) and 'narrow' data (where there is only one set of columns). Narrow data must still be placed in an array (e.g. ['value'], with only one value).","type":"array","constraints":{"enum":[{"uuid":"60fd0916-8f55-4236-8a26-e09cbc9b3c25","name":"charity"},{"uuid":"4f6de275-e6af-4da5-adee-6e5ec60d0488","name":"covid_carf"},{"uuid":"dde9af58-e13c-4ca0-be93-4d46fadb760f","name":"discretionary"},{"uuid":"1b66abb2-3de7-4cfa-8a21-94be7abf8d0a","name":"empty"},{"uuid":"9ec6a8bd-2ee1-4061-bd1d-d454b57c167d","name":"enteprise_zone"},{"uuid":"4320d78d-4d6f-4c31-ba94-0ad2f20bc185","name":"exempt"},{"uuid":"e1895f58-8bc1-4c4c-9f4b-f9f4a507ce5d","name":"hardship"},{"uuid":"3887112a-d266-4e8d-9aa6-1771f2273c9c","name":"insolvency"},{"uuid":"82fe4da5-1e7c-4406-a076-3d8557d64b1b","name":"listed"},{"uuid":"53baf5cb-aff8-47e6-b9ab-f517365f5f7f","name":"mandatory"},{"uuid":"a8a4c3d6-ca77-4776-a754-5848a8415bf3","name":"retail","description":"Has differing definitions at different times"},{"uuid":"f50ade3e-2ea3-419e-86fe-dbb7f88d8e08","name":"rural"},{"uuid":"6ad19826-c4a0-4d55-9a47-d79012e356bd","name":"s44a_partially_empty"},{"uuid":"f87f1753-c7e7-4ab2-b57e-bd5611be4f7c","name":"small_business","description":"Includes SBRR, SBBR"},{"uuid":"2adbfa0c-0414-4827-911a-5e0203e43b34","name":"transitional"},{"uuid":"93fec68d-bcfe-42ff-a90e-0c6f663bbb33","name":"other"}]}},{"uuid":"fc7c378c-70db-4364-b8a2-5e050a5eceeb","name":"occupierReliefDate","title":"Relief or Exemption date awarded","description":"Date a relief or exemption is awarded","type":"array"},{"uuid":"af286083-171d-49da-b84b-814ee681e7b4","name":"occupierReliefAmount","title":"Relief or Exemption amount awarded","type":"array"},{"uuid":"80edf660-19c8-4a94-8da3-70b5c412dc4f","name":"occupierSurchargeType","title":"Surcharge or additional rates charge type","description":"NOTE: Type, Date and Amount are arrays and the position in the array must correspond between these three columns. This is for both 'wide' data (where there are multiple columns in the source data for each class of relief) and 'narrow' data (where there is only one set of columns). Narrow data must still be placed in an array (e.g. ['value'], with only one value).","type":"array","constraints":{"enum":[{"uuid":"317f3a54-58e6-4ad4-bc65-31881c2c5294","name":"bid_levy","description":"Business Improvement District, BID"},{"uuid":"0cc6d603-fc6f-4251-a825-ce7f44bf4cc0","name":"crossrail"},{"uuid":"74d1accc-31b1-4bb9-8c65-cff6d12f2383","name":"transitional_surcharge"},{"uuid":"0cb87dae-87f7-4a25-aa7d-0fc6c6c99ff5","name":"city_of_london","description":"If they publish"},{"uuid":"02a1859b-2b2a-4064-84a1-d808d2853926","name":"other"}]}},{"uuid":"92871604-fd9e-4f2f-b6f0-7d694bf6605e","name":"occupierSurchargeDate","title":"Surcharge award date","description":"Date surcharge and additional charges applied","type":"array"},{"uuid":"e3da11ad-d17b-4dda-9fd7-c2a829519ba9","name":"occupierSurchargeAmount","title":"Surcharge Amount Charged","type":"array"}],"version":[{"name":"gchait@whythawk.com","description":"Initial save of SchemaModel.","updated":"2024-01-18T09:45:02.887755"},{"name":"Gavin Chait ","description":"Update save of SchemaModel.","updated":"2024-01-18T14:50:55.745764"}]},"actions":[{"uuid":"11d5b3e7-9deb-4d1a-847c-44debd7beb3c","script":"NEW > 'localAuthorityCode' < ['E07000032']"},{"uuid":"2752898f-df71-4822-b78a-d91398ed5734","script":"RENAME > 'localBillingReference' < ['Reference']"},{"uuid":"342919d0-dfca-4b09-a001-d2fa2608cd4b","script":"RENAME > 'occupierPropertyAddress' < ['Property address']"},{"uuid":"1c996fb1-254c-4da8-948f-b856764ad9dc","script":"RENAME > 'occupierAccountStartDate' < ['Liablefrom']"},{"uuid":"4b7f683b-09bc-4566-b6ff-af7779ae9912","script":"CATEGORISE > 'occupierOccupationState'::'Vacant' < 'Emptyfrom'::[True]"},{"uuid":"b8edd740-cd1b-4800-9354-1203439a99e7","script":"CATEGORISE > 'occupierOccupationState'::'Occupied' < 'Emptyfrom'::[False]"},{"uuid":"60d655d0-5a14-4208-989e-06fb45d9e15f","script":"SELECT > 'occupierOccupationDate' < ['Emptyfrom']"},{"uuid":"9de060d5-24f9-4a90-a54a-9d78970aade6","script":"CATEGORISE > 'occupierReliefType'::'empty' < 'Emptyfrom'::[True]"},{"uuid":"403494f5-0faf-4436-a1cb-46f446cac65a","script":"CATEGORISE > 'occupierReliefType'::'mandatory' < 'Relief type'::['Mandatory']"},{"uuid":"9005b97f-bea6-4836-8efb-ede8a601a7b7","script":"CATEGORISE > 'occupierReliefType'::'discretionary' < 'Relief type'::['Discretionary']"},{"uuid":"fd4e34e4-92eb-4b18-a705-3e2ad11ad14b","script":"CATEGORISE > 'occupierReliefType'::'small_business' < 'Relief type'::['SBRR']"},{"uuid":"811029f9-58bb-4db1-bd8b-0026409c082c","script":"CATEGORISE > 'occupierReliefType'::'transitional' < 'Relief type'::['Trans Prem','Transitional']"},{"uuid":"129a9cb1-6a5a-4138-a6a0-63a4ebfe9a54","script":"CATEGORISE > 'occupierReliefType'::'listed' < 'Exemption type'::['Listed Buildings']"},{"uuid":"3a119a02-0236-4d6c-a7c4-f9c9dc7914f6","script":"CATEGORISE > 'occupierReliefType'::'empty' < 'Exemption type'::['CASCs','Charities','Land','Listed Buildings','Telecommunication Masts']"},{"uuid":"67b5d74f-3e3f-4bcc-a978-5e8c37583e75","script":"CATEGORISE > 'occupierReliefType'::'exempt' < 'Exemption type'::['CASCs','Charities','Land','Listed Buildings','Telecommunication Masts']"},{"uuid":"b2ad1f38-71dd-40e9-8c80-5adcd39964f6","script":"COLLATE > 'occupierReliefDate' < ['Emptyfrom',~,~,~,~,'Exemption date','Exemption date','Exemption date']"}]}} \ No newline at end of file diff --git a/tests/data/sample/fenland/2026-10-25-Fenland District Council.csv b/tests/data/sample/fenland/2026-10-25-Fenland District Council.csv new file mode 100644 index 0000000..e1541c5 --- /dev/null +++ b/tests/data/sample/fenland/2026-10-25-Fenland District Council.csv @@ -0,0 +1,102 @@ +Property Reference,Account Holder 1,Liable From,Property Address 1,Property Address 2,Property Address 3,Property Address 4,Property Post Code,Charge,RV,Multiplier,Description,Charity Y/N,Disc %,MAND %,Charity From,Rural Y/N,Disc %,MAND %,Rural From,SBRR Y/N,SBRR From,Exemption Y/N,Exemption From,Exemption Type,Empty Y/N,Empty From,S44a Y/N,S44a From,S49 Y/N,S49 From,Occupied Relief Y/N,Occupied Relief From,Transitional Relief Y/N,Transitional Relief From,Heat Network Relief Y/N,Heat Network Relief From,Property Improvement Relief Y/N,Property Improvement Relief From,Local Discount Y/N,Local Discount From,Local Code FR23,Local Code FR23 Amount,Local Code LDBRR,Local Code LDBRR Amount,Local Code PLM,Local Code PLM Amount +Local Code SSB26 Amount,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +50139050000,REDACTED,01.04.2010,50 DUNCOMBES ROAD,WHITTLESEY,PETERBOROUGH,,PE7 2DS,800,17500,Small Business RHL Multiplier,KENNELS AND PREMISES,N,0,0,,N,0,0,,,,N,, ,N,,N,,N,,N,,Y,01.04.2026,N,,N,,Y,01.04.2026,N,#0.00,N,#0.00, ,#0.00 +#-4963.45,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +50247013000,REDACTED,21.03.2026,13 MARKET STREET,WHITTLESEY,PETERBOROUGH,,PE7 1BA,0,11750,Small Business RHL Multiplier,SHOP AND PREMISES,N,0,0,,N,0,0,,Y,15.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +60061020001,ARTHUR RANK HOSPICE CHARITY,01.09.2023,20 BROAD STREET,MARCH,CAMBS,,PE15 8TG,2037.17,26500,Small Business RHL Multiplier,SHOP AND PREMISES,Y,0,80,01.04.2026,N,0,0,,N,,N,, ,N,,N,,N,,N,,Y,01.04.2026,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +60061MACK00,POUNDSTRETCHER LTD,12.06.2026,13-15 BROAD STREET,MARCH,CAMBS,,PE15 8TP,12272.29,39000,Small Business RHL Multiplier,SHOP AND PREMISES,N,0,0,,N,0,0,,N,,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +27022BARN00,PRINCIPLE CONSTRUCTION AND MAINTENANCE LTD,23.07.2024,CHAMBERLAIN FARM BARN,EUXIMOOR DROVE,CHRISTCHURCH,WISBECH CAMBS,PE14 9LT,4641,10500,Small Business Multiplier,STORE AND PREMISES,N,0,0,,N,0,0,,N,,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +40130BOOK00,REDACTED,12.05.2017,3 HIGH STREET,CHATTERIS,CAMBS,,PE16 6BE,0,2500,Small Business Multiplier,STORE AND PREMISES,N,0,0,,N,0,0,,N,,Y,01.04.2026,RV,Y,12.05.2017,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +10240007001,REDACTED,20.08.2024,7 HORSEFAIR,WISBECH,CAMBS,,PE13 1AR,0,9700,Small Business RHL Multiplier,SHOP AND PREMISES,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +10240BUSS00,FENLAND DISTRICT COUNCIL,01.04.2010,BUS STATION,HORSEFAIR,WISBECH,CAMBS,PE13 1AR,13260,30000,Small Business Multiplier,BUS STATION AND PREMISES,N,0,0,,N,0,0,,N,,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +10457000001,REDACTED,01.01.2020,11 SANDRINGHAM AVENUE,WISBECH,CAMBS,,PE13 3ED,0,1000,Small Business RHL Multiplier,SALON AND PREMISES,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +40130CARP01,FENLAND DISTRICT COUNCIL,01.04.2010,"FDC, FURROWFIELDS ROAD",CHATTERIS,CAMBS,,PE16 6DY,0,5600,Small Business Multiplier,PUBLIC CONVENIENCES,N,0,0,,N,0,0,,N,,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +10331WARE001,PORT OF WISBECH LTD,01.04.2010,PORT OF WISBECH,UNITS A & B NENE PARADE,WISBECH,CAMBS,PE13 3BB,42140,86000,Standard Multiplier,WAREHOUSE AND PREMISES,N,0,0,,N,0,0,,N,,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +10349088000,REDACTED,17.11.2023,88 NORFOLK STREET,WISBECH,CAMBS,,PE13 2LF,2519.4,5700,Small Business Multiplier,SHOP AND PREMISES,N,0,0,,N,0,0,,N,,N,, ,Y,17.11.2023,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +10349090002,BELLA BLOOM HAIR LTD,17.02.2026,89-90 NORFOLK STREET,WISBECH,CAMBS,,PE13 2LF,0,5300,Small Business RHL Multiplier,SHOP AND PREMISES,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +40184000U05,GRAHAM GLOBAL SERVICES LIMITED,01.05.2022,OFFICE U5,SOUTH FENS BUSINESS CENTRE,FENTON WAY,"CHATTERIS, CAMBS",PE16 6TT,0,2600,Small Business Multiplier,OFFICE,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +40184000U10,FENLAND DISTRICT COUNCIL,31.05.2023,OFFICE U10,SOUTH FENS BUSINESS CENTRE,FENTON WAY,"CHATTERIS, CAMBS",PE16 6TT,1878.5,4250,Small Business Multiplier,OFFICE,N,0,0,,N,0,0,,N,,N,, ,Y,31.05.2023,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +10310COBRA0,REDACTED,19.10.2020,COBRA,MILETREE LANE,WISBECH,CAMBS,PE13 4TR,0,11750,Small Business Multiplier,WORKSHOP AND PREMISES,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +50199000010,NATIONWIDE BUILDING SOCIETY,22.01.2024,1 ANGEL HOUSE,HIGH CAUSEWAY,WHITTLESEY,PETERBOROUGH,PE7 1SE,4243.2,9600,Small Business Multiplier,BANK AND PREMISES,N,0,0,,N,0,0,,N,,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +28144000001,REDACTED,01.04.2017,BROADGATE,HIGH ROAD,GUYHIRN,WISBECH CAMBS,PE13 4EQ,0,1275,Small Business RHL Multiplier,DOG GROOMING PARLOUR,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +10127UNIT06,THE NORFOLK HOSPICE,27.10.2017,UNIT 6 CROMWELL RETAIL PARK,SANDOWN ROAD,WISBECH,CAMBS,PE14 0SW,3469.2,44250,Small Business RHL Multiplier,RETAIL WAREHOUSE AND PREMISES,Y,0,80,01.04.2026,N,0,0,,N,,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +10154012013,AMBA DISTRIBUTION LIMITED,10.02.2023,11A GOLD LEAF INDUSTRIAL PARK,SANDALL ROAD,WISBECH,CAMBS,PE13 2GA,0,9400,Small Business Multiplier,WAREHOUSE AND PREMISES,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,Y,01.04.2026,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +22025000009,REDACTED,04.01.2021,UNIT 9 BEECHWOOD YARD,CATTLEDYKE,GOREFIELD,WISBECH CAMBS,PE13 4NR,4243.2,9600,Small Business Multiplier,WAREHOUSE AND PREMISES,N,0,0,,N,0,0,,N,,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +60061021000,HERON FOODS LTD,01.04.2010,21 BROAD STREET,MARCH,CAMBS,,PE15 8TP,15876,40500,Small Business RHL Multiplier,SHOP AND PREMISES,N,0,0,,N,0,0,,N,,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +28061GARA04,J.S. HOLMES LIMITED,12.07.2010,J S HOLMES,HIGH ROAD,WISBECH ST MARY,"WISBECH, CAMBS",PE13 4RQ,37912.05,100000,Standard RHL Multiplier,CAR SHOWROOM AND PREMISES,N,0,0,,N,0,0,,N,,N,, ,N,,N,,N,,N,,N,,N,,N,,Y,01.04.2026,N,#0.00,N,#0.00, ,#0.00 +#-6087.95,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +51148LARK00,ALLISON HOMES LIMITED,01.04.2022,LARKFLEET HOMES LTD,BASSENHALLY FARM,EASTREA ROAD,"WHITTLESEY, CAMBS",PE7 2AP,0,2650,Small Business Multiplier,CONTRACTORS HUTS,N,0,0,,N,0,0,,N,,Y,01.04.2026,RV,Y,30.09.2025,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +60451WORK01,REDACTED,01.04.2010,UNIT 2 COLESEED FARM,UPWELL ROAD,MARCH,CAMBS,PE15 0DP,0,2900,Small Business Multiplier,WORKSHOP AND PREMISES,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +28070SCHO02,DIAMOND LEARNING PARTNERSHIP TRUST,01.09.2017,MURROW PRIMARY ACADEMY SCHOOL,73 MURROW BANK,"MURROW, WISBECH",CAMBS,PE13 4HB,3226.6,36500,Small Business Multiplier,SCHOOL AND PREMISES,Y,0,80,01.04.2026,N,0,0,,N,,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +28118000003,PARAGON PRINT AND PACKAGING LIMITED,01.04.2017,BAYS 5-6 AT KEN THOMAS,THORNEY ROAD,GUYHIRN,WISBECH CAMBS,PE13 4AD,76440,156000,Standard Multiplier,WAREHOUSE AND PREMISES,N,0,0,,N,0,0,,N,,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +10088STOR13,REDACTED,01.01.2019,WISBECH MOBILITY,AT FENLAND HOUSE,CATTLE MARKET CHASE,WISBECH CAMBS,PE13 1RD,0,10750,Small Business RHL Multiplier,SHOWROOM AND PREMISES,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +10098CARP00,FENLAND DISTRICT COUNCIL,01.04.2010,CAR PARK,CHURCH TERRACE WISBECH,CAMBS,,PE13 1BL,42124.5,86000,Standard Multiplier,CAR PARK AND PREMISES,N,0,0,,N,0,0,,N,,N,, ,N,,N,,N,,N,,Y,01.04.2026,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +60182007C00,FREEMUNCE RENEWAL PARK LTD,22.12.2025,7C FOUNDRY WAY,MARCH,CAMBS,,PE15 0WR,2124.75,6200,Small Business Multiplier,WORKSHOP AND PREMISES,N,0,0,,N,0,0,,N,,N,, ,Y,22.12.2025,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +10275007000,REDACTED,07.12.2018,7/8 LITTLE CHURCH STREET,WISBECH,CAMBS,,PE13 1BG,0,6100,Small Business RHL Multiplier,SHOP AND PREMISES,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +60080007001,MARCH AMATEUR BOXING CLUB,01.04.2010,UNIT 7 CENTURY WAY,MARCH,CAMBS,,PE15 8QW,0,9000,Small Business Multiplier,WAREHOUSE AND PREMISES,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +10355CURT01,REDACTED,01.07.2024,THE CURTAIN & BLIND STUDIO,NORTH END,WISBECH,CAMBS,PE13 1PE,0,6900,Small Business Multiplier,WORKSHOP AND PREMISES,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +10507UKF000,STARLIGHT THEATRICAL LTD & PRESST LTD,20.02.2023,UNIT 2,ANGLIA WAY,WISBECH,CAMBS,PE13 2TY,2600,18250,Small Business Multiplier,FACTORY AND PREMISES,N,0,0,,N,0,0,,N,,N,, ,N,,N,,N,,N,,N,,N,,N,,Y,01.04.2026,N,#0.00,N,#0.00, ,#0.00 +#-5466.50,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +50370026001,REDACTED,01.04.2017,26-28 AND R/O 22,STATION ROAD,WHITTLESEY,PETERBOROUGH,PE7 1UA,0,7700,Small Business Multiplier,WORKSHOP AND PREMISES,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +10382SCRA01,MICK DENTON METAL RECYCLING & SKIP HIRE LTD,01.06.2010,MICK DENTON METAL RECYCLING,AND SKIP HIRE LTD,OSBORNE ROAD,"WISBECH, CAMBS",PE13 3JP,15691,35500,Small Business Multiplier,LAND USED FOR STORAGE AND PREMISES,N,0,0,,N,0,0,,N,,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +50028003501,J. REDDINGTON LTD,01.04.2026,YARD 1,35 BENWICK ROAD,WHITTLESEY,CAMBS,PE7 2HD,10387,23500,Small Business Multiplier,LAND USED FOR STORAGE AND PREMISES,N,0,0,,N,0,0,,N,,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +60436064000,REDACTED,02.12.2022,64 STATION ROAD,THE PRINTING HOUSE,MARCH,CAMBS,PE15 8NP,0,5500,Small Business RHL Multiplier,SHOP AND PREMISES,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,Y,01.04.2026,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +10118TEJA00,REDACTED,11.01.2016,TEJACEY KENNELS,COX'S LANE,WISBECH,CAMBS,PE13 4TB,0,4200,Small Business RHL Multiplier,KENNELS AND PREMISES,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,Y,01.04.2026,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +10331026002,REDACTED,06.04.2015,26A NENE PARADE,WISBECH,CAMBS,,PE13 3BY,0,4900,Small Business Multiplier,STORE AND PREMISES,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +40130076000,REDACTED,27.04.2019,R/O 76 HIGH STREET,CHATTERIS,CAMBS,,PE16 6NN,0,5600,Small Business Multiplier,WAREHOUSE AND PREMISES,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +50325001A00,CURRY COTTAGE Z H LTD,23.09.2025,1A QUEEN STREET,WHITTLESEY,PETERBOROUGH,,PE7 1AY,0,4800,Small Business RHL Multiplier,SHOP AND PREMISES,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +60061025A01,PHONE CAMPUS LTD,24.05.2022,25A BROAD STREET,MARCH,CAMBS,,PE15 8TP,800,12750,Small Business RHL Multiplier,SHOP AND PREMISES,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,Y,01.04.2026,N,#0.00,N,#0.00, ,#0.00 +#-449.50,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +27022CHER01,REDACTED,04.04.2010,CHERRY TREE FARM,EUXIMOOR DROVE,CHRISTCHURCH,WISBECH,PE14 9LT,0,2300,Small Business Multiplier,RECORDING STUDIO AND PREMISES,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +10301003001,TAN WORKS LIMITED,01.04.2012,3 MARKET PLACE,WISBECH,CAMBS,,PE13 1DT,0,3550,Small Business RHL Multiplier,SHOP AND PREMISES,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +10349108A00,REDACTED,28.02.2025,108A NORFOLK STREET,WISBECH,CAMBS,,PE13 2LD,0,6500,Small Business RHL Multiplier,SHOP AND PREMISES,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +600700SAL01,REDACTED,01.01.2014,HAIR SALON,8 BURROWMOOR ROAD,MARCH,CAMBS,PE15 9RD,0,520,Small Business RHL Multiplier,HAIRDRESSING SALON,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +10153GARA04,11/11 AESTHETICS CLINIC LIMITED,01.08.2020,UNIT 4 AT 12 ELM HIGH ROAD,WISBECH,CAMBS,,PE14 0DG,0,3500,Small Business RHL Multiplier,CLINIC AND PREMISES,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +40184000L22,NASSH HEALTHCARE LIMITED,11.04.2022,OFFICE L22,SOUTH FENS BUSINESS CENTRE,FENTON WAY,"CHATTERIS, CAMBS",PE16 6TT,2488.76,5700,Small Business Multiplier,OFFICE,N,0,0,,N,0,0,,N,,N,, ,Y,11.04.2022,N,,N,,N,,Y,01.04.2026,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +40277023001,WEST PARK STREET CHIPPIE LTD,01.04.2023,23 WEST PARK STREET,CHATTERIS,CAMBS,,PE16 6AL,0,8200,Small Business RHL Multiplier,SHOP AND PREMISES,N,0,0,,N,0,0,,Y,01.04.2026,N,, ,N,,N,,N,,N,,N,,N,,N,,N,,N,#0.00,N,#0.00, ,#0.00 +#0.00,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/tests/data/sample/guildford/guildford.TRANSFORM b/tests/data/sample/guildford/guildford.TRANSFORM new file mode 100644 index 0000000..e0d8d98 --- /dev/null +++ b/tests/data/sample/guildford/guildford.TRANSFORM @@ -0,0 +1 @@ +{"uuid":"4294c716-3817-4753-84a3-9949a08054b7","name":"late-owner","title":"Guildford","dataSource":{"uuid":"6b936a8f-09dd-4d98-98f2-9ad36fbf7069","title":"Guildford","path":"/app/working/references/580c55a2-e0f7-4e23-a72c-87ecfe05f6f6.XLSX","mime":"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet","columns":[{"uuid":"32e74caa-ed1c-4ce0-b18f-d10c10393c62","name":"Place Ref"},{"uuid":"d63140b7-32bf-4091-bf1d-531068e2aff1","name":"Property Addr 1"},{"uuid":"70320c80-4d7e-46e5-bda5-4bdc95d5ebf1","name":"Property Addr 2"},{"uuid":"3684c7b2-2e51-4b9f-8093-41720f9a9b39","name":"Property Addr 3"},{"uuid":"52fe9bfd-32d4-42be-af23-f4ff2343af09","name":"Property Addr 4"},{"uuid":"bc9e439f-8a17-41c4-a686-e4dc39cc1353","name":"Property Addr 5"},{"uuid":"3b7b0d69-0581-4cd5-854e-299d885620e5","name":"Property Post Code"},{"uuid":"ea5ac886-6d74-46f4-86b6-cd9ca26d64a1","name":"Property Description"},{"uuid":"b559adf5-4b10-4d78-88e2-ec9e663be0a8","name":"Annual Charge"},{"uuid":"7699dfde-d2eb-4a49-aaf0-cef6f38e5d44","name":"SBR Supplement in Annual Charge No Longer Relevant"},{"uuid":"2604aa7b-82e8-45db-b18b-fcf77076d2fc","name":"Transitional Reduction"},{"uuid":"454cd50c-26cc-49db-aa15-5464dce2d0b1","name":"Empty (Y/N) or Not Available (N/A)"},{"uuid":"db0b63e9-f326-47af-93d7-21add8e06be1","name":"Date Fell Empty"},{"uuid":"684ca6ed-0b84-4c35-98ba-808b05895af1","name":"Empty Exemption"},{"uuid":"ee231f92-f646-433e-8191-d079af2ba043","name":"Small Business Rate Relief (Y/N)"},{"uuid":"6f9218a9-f02e-49b8-98c1-3ec07e1eb787","name":"Mandatory Relief (Y/N)"},{"uuid":"3a6a50c8-e25c-46fd-a638-aaf2ced646b5","name":"Mandatory Relief %"},{"uuid":"eabf7b42-b00c-4f5a-8c16-e77c287e4576","name":"Discretionary Relief (Y/N)"},{"uuid":"52b6f564-1006-491f-aaf4-6b66610033e5","name":"Discretionary Relief %"},{"uuid":"e351c72f-06f0-42ff-a6fc-4b09840c313f","name":"Rural Rate Relief Y/N"},{"uuid":"bc19e9b7-dafb-42e9-b9fe-e20a60f34c51","name":"Supporting Small Business Relief"},{"uuid":"15121ee5-dd99-4fec-bdfe-b19ad3065248","name":"Retail Discount Y/N"},{"uuid":"e1229294-d0f1-4614-8c87-782a1be181be","name":"Name"},{"uuid":"52b580d7-565a-490f-a180-ce9361057752","name":"Correspondence Addr 1"},{"uuid":"bbf3bc86-75e1-4b16-85e5-5332a162a67a","name":"Correspondence Addr 2"},{"uuid":"4f78c3a9-c6b5-4b91-ba90-3ed0ce3f22a7","name":"Correspondence Addr 3"},{"uuid":"2977e39e-36b5-4ef6-bb27-d4c0067d31d3","name":"Correspondence Addr 4"},{"uuid":"7230bb6a-1908-4d12-bed2-477cb82ddbdf","name":"Correspondence Addr 5"},{"uuid":"324f2df7-59c1-4782-b763-909ef59bee98","name":"Correspondence Post Code"},{"uuid":"4e2c8622-fb2e-420d-b8c2-b75313fd88c1","name":"Other Reliefs"}],"preserve":["Place Ref","Property Addr 1","Property Addr 2","Property Addr 3","Property Addr 4","Property Addr 5","Property Post Code","Property Description","Annual Charge","SBR Supplement in Annual Charge No Longer Relevant","Transitional Reduction","Empty (Y/N) or Not Available (N/A)","Date Fell Empty","Empty Exemption","Small Business Rate Relief (Y/N)","Mandatory Relief (Y/N)","Mandatory Relief %","Discretionary Relief (Y/N)","Discretionary Relief %","Rural Rate Relief Y/N","Supporting Small Business Relief","Retail Discount Y/N","Name","Correspondence Addr 1","Correspondence Addr 2","Correspondence Addr 3","Correspondence Addr 4","Correspondence Addr 5","Correspondence Post Code","Other Reliefs"],"attributes":{},"checksum":"3bcb18e52b838940bccb7d37b3cc7e8d6402470a80a39557bd1a98d134e5a72ef312578a931b44e081b28dfc5abd0ad34e805c4dd339bb2a785cff1c18d0be54","index":4497},"crosswalk":{"uuid":"ba5b7a3e-e873-4bb3-8c5d-e546b611fe1b","name":"e07000209","title":"Guildford","schemaSource":{"uuid":"8bd37a74-cb2a-4a88-abba-c68671cc034a","name":"e07000209","title":"Guildford","fields":[{"uuid":"f3ffa3f1-af7c-4475-b56f-ebd39acaf18a","name":"Place Ref","title":"Place Ref","type":"string"},{"uuid":"b69737c4-3add-496e-9908-951c9f2bd14d","name":"Property Addr 1","title":"Property Addr 1","type":"string"},{"uuid":"730079e4-927e-4ca2-bfd9-a23483cfb988","name":"Property Addr 2","title":"Property Addr 2","type":"string"},{"uuid":"ef52c3f2-f1ca-468d-b0b5-b6aeb1cd3071","name":"Property Addr 3","title":"Property Addr 3","type":"string"},{"uuid":"6f19feb0-1c5b-4794-adfe-87f1d73aac62","name":"Property Addr 4","title":"Property Addr 4","type":"string"},{"uuid":"b4cc9a37-f0d2-4081-8a11-9741a3dc1ee4","name":"Property Addr 5","title":"Property Addr 5","type":"string"},{"uuid":"46ed3b52-a3de-4655-b7dd-39458943d377","name":"Property Post Code","title":"Property Post Code","type":"string"},{"uuid":"44f966c9-b27e-4a67-9886-8b777f4d2e68","name":"Property Description","title":"Property Description","type":"string"},{"uuid":"0f7d88cb-a469-4155-9289-d37936b62a7d","name":"Annual Charge","title":"Annual Charge","type":"string"},{"uuid":"ec0434c5-5d69-4abc-bcc6-bafa5224146c","name":"SBR Supplement in Annual Charge No Longer Relevant","title":"SBR Supplement in Annual Charge No Longer Relevant","type":"string"},{"uuid":"90e11636-5847-4114-848b-9a589a0d6d88","name":"Transitional Reduction","title":"Transitional Reduction","type":"number","constraints":{"enum":[{"uuid":"908febf1-a5d9-4a57-8782-5303c9aa40c1","name":true},{"uuid":"ac435202-60b1-41cb-a08a-0adcc7d0c2c4","name":false}]}},{"uuid":"7446189f-54f4-4a37-8de4-0d34242e74bd","name":"Empty (Y/N) or Not Available (N/A)","title":"Empty (Y/N) or Not Available (N/A)","type":"string","constraints":{"enum":[{"uuid":"2f3860f4-1d06-4602-8c87-464366f29b2a","name":"N"},{"uuid":"ec9e4f96-4b28-44b1-b12e-28d5e0c26ec0","name":"Y"}]}},{"uuid":"9a104f02-df38-4e69-aa0e-06ceed931485","name":"Date Fell Empty","title":"Date Fell Empty","type":"string"},{"uuid":"ac5e9a88-8bf4-4c5a-a85d-43c0bf3eae11","name":"Empty Exemption","title":"Empty Exemption","type":"string","constraints":{"enum":[{"uuid":"355f2698-2393-4550-a59f-b6f0ad9b9b12","name":"6 months"},{"uuid":"b4cc4286-2548-44f2-a25f-2233b9774744","name":"Other"},{"uuid":"6fac94d4-0fba-4d55-a9a5-85b29a6ad446","name":"Listed"},{"uuid":"64e0f0b0-82ea-4174-84c7-723de21ad451","name":"Under the rateable value threshold"},{"uuid":"b4bb8e11-8195-4a51-9571-f5720fe7711f","name":"3 months"}]}},{"uuid":"1a8c6d18-1503-4fb2-9fe3-2dbac116f765","name":"Small Business Rate Relief (Y/N)","title":"Small Business Rate Relief (Y/N)","type":"string","constraints":{"enum":[{"uuid":"909bd79f-2729-4d6a-8b7d-1a5c6374efe7","name":"Y"},{"uuid":"f3d1ba70-13c6-4556-a862-f67cbc570a46","name":"N"}]}},{"uuid":"46145d3a-5eb5-4084-9909-bd874fc04c91","name":"Mandatory Relief (Y/N)","title":"Mandatory Relief (Y/N)","type":"string","constraints":{"enum":[{"uuid":"50066df2-2833-4f13-afe3-72bbc600bbd8","name":"N"},{"uuid":"405369a5-b732-4a51-b3cf-5c6cb983580d","name":"Y"}]}},{"uuid":"4133d552-a2f3-4ae1-89a3-cec11488bf4b","name":"Mandatory Relief %","title":"Mandatory Relief %","type":"string"},{"uuid":"ef9f9725-39bd-4ec2-9054-d058e75def40","name":"Discretionary Relief (Y/N)","title":"Discretionary Relief (Y/N)","type":"string","constraints":{"enum":[{"uuid":"9a493693-0f1c-4bfa-b2b9-47c87b46ecbe","name":"N"},{"uuid":"fd62bf8d-c70c-45aa-9ac3-d7cae5309c6b","name":"Y"}]}},{"uuid":"fd45b7fa-78d6-4df3-947c-54eb9a44778b","name":"Discretionary Relief %","title":"Discretionary Relief %","type":"string"},{"uuid":"fd1a7c1d-983e-46ee-8107-cced7ed83bee","name":"Rural Rate Relief Y/N","title":"Rural Rate Relief Y/N","type":"string","constraints":{"enum":[{"uuid":"96358331-0f89-4881-95fe-004707d0e9cb","name":"N"},{"uuid":"05f753e0-2c6f-4482-b94a-986442d00cc9","name":"Y"}]}},{"uuid":"91ff95d9-64c0-4f2c-9d2c-95bf6304ffbd","name":"Supporting Small Business Relief","title":"Supporting Small Business Relief","type":"string","constraints":{"enum":[{"uuid":"000fd51b-5519-458f-8865-ee1525e7e3e1","name":"N"},{"uuid":"79f56505-7a84-4087-a458-f2dea8f1418d","name":"Y"}]}},{"uuid":"428dde3b-bc90-427c-83f8-b40e558bf1dc","name":"Retail Discount Y/N","title":"Retail Discount Y/N","type":"string","constraints":{"enum":[{"uuid":"b26c780f-82ef-48f8-a418-e2937ddf80bd","name":"N"}]}},{"uuid":"5bb354d6-0ab3-4178-bff1-9c615576eeef","name":"Name","title":"Name","type":"string"},{"uuid":"51473a36-7e51-40e8-afb0-0fd0b6400933","name":"Correspondence Addr 1","title":"Correspondence Addr 1","type":"string"},{"uuid":"075a77cb-f1e0-430a-ab69-70c9ce360b0b","name":"Correspondence Addr 2","title":"Correspondence Addr 2","type":"string"},{"uuid":"43c47c35-5617-4698-936c-cac6e4c7c64f","name":"Correspondence Addr 3","title":"Correspondence Addr 3","type":"string"},{"uuid":"5b84fd97-cc7d-49be-b8d7-8388ac689699","name":"Correspondence Addr 4","title":"Correspondence Addr 4","type":"string"},{"uuid":"01009e98-b76a-4977-8081-a2203d7bd65b","name":"Correspondence Addr 5","title":"Correspondence Addr 5","type":"string"},{"uuid":"33480221-46f9-448d-8f54-f5e4dfa004b9","name":"Correspondence Post Code","title":"Correspondence Post Code","type":"string"},{"uuid":"802d6ae5-b7cc-4645-8117-4c0121953e17","name":"Other Reliefs","title":"Other Reliefs","type":"string","constraints":{"enum":[]}}],"index":4497},"schemaDestination":{"uuid":"507a0ccf-5735-4b3a-9b9e-2e7842921e4a","name":"england-and-wales-nondomestic-rates-schema","title":"England and Wales non-domestic rates schema","description":"Based on the The Open Data Collaboration group at Open Innovations (Open Innovations, Bradford Council, Calderdale Council, and Leeds Council) which started a project to visualise business rates data. Although each of the three councils publish open data about their business rates, they each have different field titles and don't always include the same fields. This project aimed to standardise the way the data was shared to make it more usable and to get value (within a council and externally) from the datasets.\nhttps://open-innovations.org/projects/business-rates/format.html","fields":[{"uuid":"e1b1d480-8481-4281-98bd-0e59f7fdb949","name":"localAuthorityCode","title":"Local authority code","description":"Unique reference identifier for each local authority. Added from the Task 'name' unique identifier field.","type":"string","constraints":{"default":{"uuid":"f62e9a77-c153-4ff9-8eb3-b9963527f942","name":"E07000209"}}},{"uuid":"f42d609b-c8a6-4017-aa9d-d55eb62bd63b","name":"localBillingReference","title":"Billing reference","description":"Local Authority hereditament billing reference used by both VOA & LA to correspond with each other in reference to a specific hereditament.","type":"string","example":"8577090904","constraints":{"required":true,"enum":[]}},{"uuid":"7774382b-7054-4100-8bf1-ff9741a47d5a","name":"occupierAccountHolderName","title":"Occupier account-holder name","description":"Account-holder's names may be split across multiple columns. They are to be joined into one for post-processing.","type":"string","example":"A.M.C (North East) Ltd"},{"uuid":"02347dbe-d491-47cb-bfd4-2a3c28c4c861","name":"occupierRecordDate","title":"Date each row entered the record","type":"date"},{"uuid":"090b1d3d-3be0-4d39-b24b-8c97969eeb8f","name":"occupierAccountStartDate","title":"Start-date for account-holder","description":"Date the Account Holder became responsible for the hereditament. Provide date only if provided in data release from LA. e.g. \"01/01/23\", \"BLANK\" or \"Not Supplied\".","type":"date"},{"uuid":"cb30a0df-ea14-4c67-b76a-1ab358f167bd","name":"occupierPropertyAddress","title":"Address of the rateable property","description":"Address may be split across multiple columns. They are to be joined into one for post-processing with the postcode at the end.","type":"string"},{"uuid":"3ed85652-0831-4cd3-a005-a353b6cc51a1","name":"occupierCorrespondenceAddress","title":"Correspondence address","description":"Account-holders correspondence address may be split across multiple columns. They are to be joined into one for post-processing with the postcode at the end. This may be different from the hereditament address.","type":"string"},{"uuid":"6305545d-7caf-40fb-b421-2e483c3fbd96","name":"occupierOccupationState","title":"Occupation state","description":"Occupational status of hereditament. Only include if provided in LA data release.\nOccupied = \"Occupied\", Void = \"Void\", No Information = \"BLANK\", no empty data provided by LA = \"Not Supplied\"","type":"string","constraints":{"enum":[{"uuid":"83cd608f-ae55-4090-836d-154c2ec256d7","name":"Occupied","description":"Occupied"},{"uuid":"de15f74d-90c3-4f38-8821-37e3178787bd","name":"Vacant","description":"Vacant"}]}},{"uuid":"17c261f9-e837-49d1-abe1-fc63f2a80f86","name":"occupierOccupationDate","title":"Occupation state date","description":"To be used in combination with 'occupierOccupationState'. If only a date is provided (e.g. for only date of void, or date of occupation), then categorise the column as a boolean and use this to specify the state and the date simultaneously.","type":"date"},{"uuid":"e325315f-5ec0-416d-a470-4facfb7bb759","name":"occupierRatesPayable","title":"Actual rates paid","description":"Rates paid, net of reliefs (if available) in pounds. Provide rates paid amount only if provided in data release from LA. e.g. \"£55,000\", \"BLANK\", \"not supplied\".","type":"number"},{"uuid":"1dba84d1-501a-427f-88ad-771e2393516a","name":"occupierReliefType","title":"Relief or Exemption Received","description":"Type of relief or exemption on rates paid received.\n\nNOTE: Type, Date and Amount are arrays and the position in the array must correspond between these three columns. This is for both 'wide' data (where there are multiple columns in the source data for each class of relief) and 'narrow' data (where there is only one set of columns). Narrow data must still be placed in an array (e.g. ['value'], with only one value).","type":"array","constraints":{"enum":[{"uuid":"60fd0916-8f55-4236-8a26-e09cbc9b3c25","name":"charity"},{"uuid":"4f6de275-e6af-4da5-adee-6e5ec60d0488","name":"covid_carf"},{"uuid":"dde9af58-e13c-4ca0-be93-4d46fadb760f","name":"discretionary"},{"uuid":"1b66abb2-3de7-4cfa-8a21-94be7abf8d0a","name":"empty"},{"uuid":"9ec6a8bd-2ee1-4061-bd1d-d454b57c167d","name":"enteprise_zone"},{"uuid":"4320d78d-4d6f-4c31-ba94-0ad2f20bc185","name":"exempt"},{"uuid":"e1895f58-8bc1-4c4c-9f4b-f9f4a507ce5d","name":"hardship"},{"uuid":"3887112a-d266-4e8d-9aa6-1771f2273c9c","name":"insolvency"},{"uuid":"82fe4da5-1e7c-4406-a076-3d8557d64b1b","name":"listed"},{"uuid":"53baf5cb-aff8-47e6-b9ab-f517365f5f7f","name":"mandatory"},{"uuid":"a8a4c3d6-ca77-4776-a754-5848a8415bf3","name":"retail","description":"Has differing definitions at different times"},{"uuid":"f50ade3e-2ea3-419e-86fe-dbb7f88d8e08","name":"rural"},{"uuid":"6ad19826-c4a0-4d55-9a47-d79012e356bd","name":"s44a_partially_empty"},{"uuid":"f87f1753-c7e7-4ab2-b57e-bd5611be4f7c","name":"small_business","description":"Includes SBRR, SBBR"},{"uuid":"2adbfa0c-0414-4827-911a-5e0203e43b34","name":"transitional"},{"uuid":"93fec68d-bcfe-42ff-a90e-0c6f663bbb33","name":"other"},{"uuid":"407bf8b6-dbf9-4707-80df-4713f06b46c1","name":"ssb_one_year","description":"Supporting Small Business 1 Year"},{"uuid":"57e578f6-5f02-457b-b83a-cab6f6bfce3d","name":"ssb_three_year","description":"Supporting Small Business 3 Year"},{"uuid":"d0214f96-836f-4aad-b8da-a9f860beb03e","name":"ssb_unspecified","description":"Supporting Small Business"},{"uuid":"aac2de06-bcc9-44bc-86d0-d753fefa1b76","name":"improvement","description":"Improvement"},{"uuid":"ccefe7d1-9c1e-4cd1-b262-648dc7ef7d45","name":"heat_network","description":"Heat Network"},{"uuid":"c641779a-d8c8-4fda-970e-966e931198ae","name":"public_lavatories","description":"Public Lavatories"},{"uuid":"d7b8ab22-286f-4781-8be9-d461199fab7a","name":"freeport","description":"Freeport"},{"uuid":"47c4b03e-9d6a-41e3-a7ab-5f40e17121aa","name":"film_studio","description":"Film Studio"},{"uuid":"26268f7b-8ecb-4f20-b8d6-5d0db06c932b","name":"pubs_live_music","description":"Pubs and Live Music Venues"},{"uuid":"c8094a9f-33e9-4b3f-bc9b-2087296798b2","name":"ev_charging","description":"EV Charging Point"},{"uuid":"729c0681-1746-4539-a96c-2792bdfdde04","name":"local_newspaper","description":"Local Newspaper"}]}},{"uuid":"af286083-171d-49da-b84b-814ee681e7b4","name":"occupierReliefAmount","title":"Relief or Exemption amount awarded","type":"array"},{"uuid":"fc7c378c-70db-4364-b8a2-5e050a5eceeb","name":"occupierReliefDate","title":"Relief or Exemption date awarded","description":"Date a relief or exemption is awarded","type":"array"},{"uuid":"80edf660-19c8-4a94-8da3-70b5c412dc4f","name":"occupierSurchargeType","title":"Surcharge or additional rates charge type","description":"NOTE: Type, Date and Amount are arrays and the position in the array must correspond between these three columns. This is for both 'wide' data (where there are multiple columns in the source data for each class of relief) and 'narrow' data (where there is only one set of columns). Narrow data must still be placed in an array (e.g. ['value'], with only one value).","type":"array","constraints":{"enum":[{"uuid":"317f3a54-58e6-4ad4-bc65-31881c2c5294","name":"bid_levy","description":"Business Improvement District, BID"},{"uuid":"0cc6d603-fc6f-4251-a825-ce7f44bf4cc0","name":"crossrail"},{"uuid":"74d1accc-31b1-4bb9-8c65-cff6d12f2383","name":"transitional_surcharge"},{"uuid":"0cb87dae-87f7-4a25-aa7d-0fc6c6c99ff5","name":"city_of_london","description":"If they publish"},{"uuid":"02a1859b-2b2a-4064-84a1-d808d2853926","name":"other"}]}},{"uuid":"e3da11ad-d17b-4dda-9fd7-c2a829519ba9","name":"occupierSurchargeAmount","title":"Surcharge Amount Charged","type":"array"},{"uuid":"92871604-fd9e-4f2f-b6f0-7d694bf6605e","name":"occupierSurchargeDate","title":"Surcharge award date","description":"Date surcharge and additional charges applied","type":"array"}],"version":[{"name":"gchait@whythawk.com","description":"Initial save of SchemaModel.","updated":"2024-01-18T09:45:02.887755"},{"name":"Gavin Chait ","description":"Update save of SchemaModel.","updated":"2024-01-18T14:50:55.745764"},{"name":"Gavin Chait ","description":"Update save of SchemaModel.","updated":"2026-08-05T13:35:05.494165"},{"name":"Gavin Chait ","description":"Update save of SchemaModel.","updated":"2026-08-05T13:37:45.592489"},{"name":"Gavin Chait ","description":"Update save of SchemaModel.","updated":"2026-08-11T19:40:05.912166"}]},"actions":[{"uuid":"0a6bbd43-4f60-494a-810a-e5612b3c470c","script":"NEW > 'localAuthorityCode' < ['E07000209']"},{"uuid":"c578552b-64fa-4a9f-96cf-e76d141a725b","script":"RENAME > 'localBillingReference' < ['Place Ref']"},{"uuid":"0a129091-10da-4a53-8bda-4eb022a0c3dc","script":"RENAME > 'occupierAccountHolderName' < ['Name']"},{"uuid":"84708aa6-492e-4d35-928e-7dd198a46c31","script":"UNITE > 'occupierPropertyAddress' < ' '::['Property Addr 1','Property Addr 2','Property Addr 3','Property Addr 4','Property Addr 5','Property Post Code']"},{"uuid":"85683ad0-55fd-476d-919b-081fd1ca5211","script":"UNITE > 'occupierCorrespondenceAddress' < ' '::['Correspondence Addr 1','Correspondence Addr 2','Correspondence Addr 3','Correspondence Addr 4','Correspondence Addr 5','Correspondence Post Code']"},{"uuid":"0fa6699b-33cc-49af-8711-d660a8253ad0","script":"CATEGORISE > 'occupierOccupationState'::'Vacant' < 'Empty (Y/N) or Not Available (N/A)'::['Y']"},{"uuid":"3d2d5172-4aa2-4d8d-8c51-5b363bee7d58","script":"CATEGORISE > 'occupierOccupationState'::'Occupied' < 'Empty (Y/N) or Not Available (N/A)'::['N']"},{"uuid":"118f6521-5399-4766-bd90-ffb31a99500c","script":"SELECT > 'occupierOccupationDate' < ['Date Fell Empty']"},{"uuid":"03a79874-9acd-499e-93d1-bd1607bc41d1","script":"CATEGORISE > 'occupierReliefType'::'discretionary' < 'Transitional Reduction'::[True]"},{"uuid":"58a4673b-7063-4e62-8a22-1cfb3e78b74d","script":"CATEGORISE > 'occupierReliefType'::'transitional' < 'Transitional Reduction'::[True]"},{"uuid":"f9299d5c-c326-4d7e-94fc-fe4393425479","script":"CATEGORISE > 'occupierReliefType'::'empty' < 'Empty (Y/N) or Not Available (N/A)'::['Y']"},{"uuid":"987a13ec-8b68-4d37-9e76-d2fb23243f73","script":"CATEGORISE > 'occupierReliefType'::'empty' < 'Empty Exemption'::['3 months','6 months','Listed','Other','Under the rateable value threshold']"},{"uuid":"96798fcb-a2dc-4d78-a4ad-6da1b67b8af0","script":"CATEGORISE > 'occupierReliefType'::'exempt' < 'Empty Exemption'::['3 months','6 months','Listed','Other','Under the rateable value threshold']"},{"uuid":"f2967e6d-282c-477b-9545-2d2da073ea88","script":"CATEGORISE > 'occupierReliefType'::'listed' < 'Empty Exemption'::['Listed']"},{"uuid":"3773657b-762d-4529-8fc0-2e8bbddfe429","script":"CATEGORISE > 'occupierReliefType'::'small_business' < 'Small Business Rate Relief (Y/N)'::['Y']"},{"uuid":"ab2c50a4-b3a7-4756-8d03-eafa3edefdc8","script":"CATEGORISE > 'occupierReliefType'::'mandatory' < 'Mandatory Relief (Y/N)'::['Y']"},{"uuid":"afc251ca-ee8b-4047-ac06-48d3c527ffc9","script":"CATEGORISE > 'occupierReliefType'::'discretionary' < 'Discretionary Relief (Y/N)'::['Y']"},{"uuid":"7cd54b49-b4bf-43ff-842a-ca67da908923","script":"CATEGORISE > 'occupierReliefType'::'rural' < 'Rural Rate Relief Y/N'::['Y']"},{"uuid":"d2465f24-29fb-4a2b-af2a-4d9b9261d207","script":"CATEGORISE > 'occupierReliefType'::'ssb_unspecified' < 'Supporting Small Business Relief'::['Y']"},{"uuid":"edb2f8d4-e93a-41ea-94d7-2a23f9986f3e","script":"COLLATE > 'occupierReliefAmount' < ['Transitional Reduction','Transitional Reduction',~,~,~,~,~,~,~,~,~]"},{"uuid":"50621b60-80eb-4514-87a6-0cc67e3313d1","script":"COLLATE > 'occupierReliefDate' < [~,~,'Date Fell Empty',~,~,~,~,~,~,~,~]"}]}} \ No newline at end of file diff --git a/tests/data/sample/guildford/source-data.XLSX b/tests/data/sample/guildford/source-data.XLSX new file mode 100644 index 0000000..01f04ae Binary files /dev/null and b/tests/data/sample/guildford/source-data.XLSX differ diff --git a/tests/data/sample/pdf/2026-03-27-maldon-empty.CSV b/tests/data/sample/pdf/2026-03-27-maldon-empty.CSV new file mode 100644 index 0000000..59d4094 --- /dev/null +++ b/tests/data/sample/pdf/2026-03-27-maldon-empty.CSV @@ -0,0 +1,175 @@ +Addr1,Addr2,Addr3,Addr4,Postcode,Current Rv,Vo Descrip,Account Holder1 +OLD RECTORY COTTAGE,OFF LOWER ROAD,NORTH FAMBRIDGE,CHELMSFORD ESSEX,CM3 6LP,970,STABLES AND PREMISES, +MAYPOLE WOOD,MAYPOLE ROAD,LANGFORD,MALDON ESSEX,CM9 4SZ,7800,WORKSHOP AND PREMISES, +ESSEX BREEDING CENTRE,RECTORY LANE,LATCHINGDON,"CHELMSFORD, ESSEX",CM3 6HB,27750,WORKSHOP AND PREMISES,OLD OAK FARM PROPERTIES LIMITED +35 THE STREET,LATCHINGDON,CHELMSFORD,ESSEX,CM3 6JP,3400,SHOP AND PREMISES,PAUL MASON ASSOCIATES LTD +STORAGE LAND (CARGOBULL LTD),MAYFAIR INDUSTRIAL ESTATE,"MALDON ROAD, LATCHINGDON",CHELMSFORD ESSEX,CM3 6LF,11000,LAND USED FOR STORAGE AND PREMISES, +UNIT 6D MAYFAIR INDUSTRIAL ESTATE,MALDON ROAD,LATCHINGDON,CHELMSFORD ESSEX,CM3 6LF,1050,WORKSHOP AND PREMISES, +UNIT 18 MAPLEDEAN WORKS,MALDON ROAD,LATCHINGDON,CHELMSFORD ESSEX,CM3 6LG,10000,Warehouse and Premises,ALRIGHT LTD +MUNDON HALL,VICARAGE LANE,MUNDON,MALDON ESSEX,CM9 6PA,1975,Tea Rooms, +THE STUDIO,WHITE HOUSE FARM,NEW HALL LANE,MUNDON ESSEX,CM9 6PJ,7000,Workshop and Premises, +UNIT G4 WHITE HOUSE FARM,NEW HALL LANE,MUNDON,MALDON ESSEX,CM9 6PJ,1375,STORE AND PREMISES, +UNIT G3 WHITE HOUSE FARM,NEW HALL LANE,MUNDON,MALDON ESSEX,CM9 6PJ,1375,STORE AND PREMISES, +UNITS 1 & 2 HOOK FARM,MUNDON ROAD,MUNDON,MALDON ESSEX,CM9 6PN,3000,Workshop and Premises, +UNIT 3 HOOK FARM,MUNDON ROAD,MUNDON,MALDON ESSEX,CM9 6PN,1350,Workshop and Premises, +UNIT 9A HOOK FARM,MUNDON ROAD,MUNDON,MALDON ESSEX,CM9 6PN,2275,Workshop and Premises, +UNIT 12 AT HOOK FARM,MUNDON ROAD,MUNDON,MALDON ESSEX,CM9 6PN,1800,WORKSHOP AND PREMISES, +UNIT 4 OLD WHITMANS FARM,HACKMANS LANE,PURLEIGH,CHELMSFORD,CM3 6RW,7600,Workshop and Premises, +AT ATHERSTONE LODGE,FAMBRIDGE ROAD,MUNDON,"MALDON, ESSEX",CM9 6NL,5300,WORKSHOP AND PREMISES,BEN RIGBY GAME LTD +UNIT 3 THE BARN,FAMBRIDGE ROAD,MUNDON,MALDON ESSEX,CM9 6NL,2250,OFFICES AND PREMISES,I.D.CORCORAN BUILDING CONTRACTORS LTD +THE ROUND BUSH,ROUNDBUSH ROAD,MUNDON,"MALDON, ESSEX",CM9 6NN,9000,PUBLIC HOUSE AND PREMISES,GRAY AND SONS (CHELMSFORD) LTD +AT ATHERSTONE LODGE,FAMBRIDGE ROAD,MUNDON,MALDON ESSEX,CM9 6NW,13500,"Shop, Store and Premises",BEN RIGBY GAME LTD +UNIT 2 NEW FARM,STOW ROAD,STOW MARIES,CHELMSFORD ESSEX,CM3 6RS,10750,Warehouse and Premises,SUPREME INSTALLATION S.EAST LTD +SHOP AT STOW GARAGE,THE STREET,STOW MARIES,CHELMSFORD ESSEX,CM3 6RU,2700,Shop and Premises,LOCKHILL PROPERTIES LTD +GREAT HAYES FARM,LOWER ROAD,STOW MARIES,CHELMSFORD ESSEX,CM3 6SQ,4050,Workshop and Premises, +ROBINS,CROUCHMANS FARM ROAD,ULTING,"MALDON, ESSEX",CM9 6QS,2650,CATTERY AND PREMISES, +UNIT B27C LITTLE SMITHS BARN,BRYANTS LANE,WOODHAM MORTIMER,MALDON ESSEX,CM9 6TB,1625,Store and Premises, +2 PARK COTTAGES,CURLING TYE LANE,WOODHAM WALTER,MALDON ESSEX,CM9 6LT,2850,"KITCHEN, WORKSHOP AND PREMISES", +UNIT 3 CROMWELL FARM WORKS,CROMWELL LANE,MALDON,ESSEX,CM9 4LB,2175,Store and Premises,THE MALDON SMOKEHOUSE LTD +2ND FLOOR,FULLBRIDGE MILL,FULLBRIDGE,MALDON ESSEX,CM9 4LE,37250,OFFICES AND PREMISES,ELTIME LTD +LWR GROUND FLR,7 CHANDLERS QUAY,MALDON,ESSEX,CM9 4LF,1650,Workshop and Premises,CHANDLERS FREEHOLD LTD +CAR PARKING SPACES ADJ UNIT 3,MALDON TRADE PARK,THE CAUSEWAY,MALDON ESSEX,CM9 4LJ,600,CAR PARKING SPACES AND PREMISES,PETCHEY INDUSTRIAL PROPERTIES (NO.1) LTD +SALES OFF & FORECOURT AT TAVERN GARAGETHE CAUSEWAY,,MALDON,ESSEX,CM9 4LJ,11500,CAR SALES SITE AND PREMISES, +FIRST FLR NETWORK HOUSE,STATION ROAD,MALDON,ESSEX,CM9 4LQ,19000,OFFICES AND PREMISES,INTERNET ASSIST LTD +159 HIGH STREET,MALDON,,ESSEX,CM9 5BS,6500,SHOP AND PREMISES,DUCK NAIL STUDIOS LTD +125 HIGH STREET,MALDON,,ESSEX,CM9 5BS,7300,SHOP AND PREMISES,NEVER WHAT IF GROUP LTD +85 HIGH STREET,MALDON,,ESSEX,CM9 5EP,55000,SHOP AND PREMISES,DOVES GROUP LTD +69 HIGH STREET,MALDON,,ESSEX,CM9 5EP,41250,SHOP AND PREMISES,CRANE HOLDINGS LIMITED +107 HIGH STREET,MALDON,,ESSEX,CM9 5EP,9200,SHOP AND PREMISES, +ORANGE (ESX7126) MICROCELL AT,76 HIGH STREET,MALDON,ESSEX,CM9 5ET,4000,Communication Station and Premises,EE LTD +116 HIGH STREET,MALDON,ESSEX,,CM9 5ET,24250,RESTAURANT AND PREMISES, +QUAYSIDE INDUSTRIAL ESTATE,MALDON,ESSEX,,CM9 5FA,315000,WAREHOUSE AND PREMISES,CLOTHES POINT A +EAST ESSEX ADULT COMMUNITY COLLEGE,CARMELITE WAY,MALDON,ESSEX,CM9 5FJ,42750,COLLEGE AND PREMISES,ESSEX COUNTY COUNCIL +STORE,THE SAIL LOFTS,THE HYTHE,MALDON ESSEX,CM9 5HN,1375,Store and Premises, +132 MILL ROAD,,MALDON,ESSEX,CM9 5JA,8400,SHOP AND PREMSIES, +KIOSK 7 PROMENADE PARK,PARK DRIVE,MALDON,ESSEX,CM9 5JQ,11500,SHOP AND PREMISES,(NDR) MALDON DISTRICT COUNCIL +REAR OFFICE 1ST FLR,19 HIGH STREET,MALDON,ESSEX,CM9 5PE,3250,Offices and Premises, +2ND FLOOR,17 HIGH STREET,MALDON,ESSEX,CM9 5PE,2150,Offices and Premises, +3RD FLOOR,17 HIGH STREET,MALDON,ESSEX,CM9 5PE,1025,Offices and Premises, +8 HIGH STREET,MALDON,,ESSEX,CM9 5PJ,18250,RESTAURANT AND PREMISES,ELCROFT PITCH LTD +20 HIGH STREET,MALDON,,ESSEX,CM9 5PJ,17250,SHOP AND PREMISES,TGE GROUP +6A THE KINGS HEAD CENTRE,38 HIGH STREET,MALDON,ESSEX,CM9 5PN,6200,OFFICES AND PREMISES,THE KINGS HEAD CENTRE +OFFICES 1-4,40A HIGH STREET,MALDON,ESSEX,CM9 5PN,13750,Offices and Premises,PMW DEVELOPMENTS LIMITED +OFFICE 5 AT 40A HIGH STREET,MALDON,ESSEX,,CM9 5PN,6000,Offices and Premises,PMW DEVELOPMENTS LIMITED +OFFICE 4 THE KINGS HEAD CENTRE,38 HIGH STREET,MALDON,ESSEX,CM9 5PN,4750,OFFICES AND PREMISES,THE KINGS HEAD CENTRE +BASEMENT AT 1 KINGS HEAD CENTRE,38 HIGH STREET,MALDON,ESSEX,CM9 5PN,870,STORE AND PREMISES,THE KINGS HEAD CENTRE +OFFICE 2 & 5 THE KINGS HEAD CENTRE,38 HIGH STREET,MALDON,ESSEX,CM9 5PN,3450,OFFICES AND PREMISES,THE KINGS HEAD CENTRE +32 HIGH STREET,MALDON,,ESSEX,CM9 5PN,9200,SHOP AND PREMISES, +52A HIGH STREET,MALDON,,ESSEX,CM9 5PN,22250,SHOP AND PREMISES,TRYTHISCO LIMITED +2ND FLOOR,40A HIGH STREET,MALDON,ESSEX,CM9 5PN,16750,OFFICES AND PREMISES,PMW DEVELOPMENTS LIMITED +OFFICE 1,THE KINGS HEAD CENTRE,38 HIGH STREET,MALDON ESSEX,CM9 5PN,2100,OFFICES AND PREMISES,THE KINGS HEAD CENTRE +OFFICE 3,THE KINGS HEAD CENTRE,38 HIGH STREET,"MALDON, ESSEX",CM9 5PN,2700,OFFICES AND PREMISES,THE KINGS HEAD CENTRE +ROYAL MAIL (STORE),43/45 HIGH STREET,MALDON,ESSEX,CM9 5PQ,12250,STORE AND PREMISES,MERIDIAN REALISATIONS LTD +OFFICE AT,4A GATE STREET,MALDON,ESSEX,CM9 5QF,7500,Offices and Premises, +1ST FLOOR BLACKWATER HOUSE,FAMBRIDGE ROAD,MALDON,ESSEX,CM9 6AA,6300,OFFICES AND PREMISES, +SHOP B BLACKWATER HOUSE,FAMBRIDGE ROAD,MALDON,ESSEX,CM9 6AA,3750,SHOP AND PREMISES, +2ND FLOOR BLACKWATER HOUSE,FAMBRIDGE ROAD,MALDON,ESSEX,CM9 6AA,4050,Offices and Premises, +16 DRYDEN CLOSE,MALDON,,ESSEX,CM9 6DG,9600,SHOP AND PREMISES,BLACKSMITH STORE ESSEX LTD +81 WASHINGTON ROAD,MALDON,,ESSEX,CM9 6JE,8300,Shop and Premises,HOLMWOOD SECURITIES LTD +REAR OF 83/85 WASHINGTON ROAD,MALDON,ESSEX,,CM9 6JE,2250,STORE AND PREMISES,HOLMWOOD SECURITIES LTD +UNIT 6B WEST STATION YARD,SPITAL ROAD,MALDON,ESSEX,CM9 6TR,11750,Workshop and Premises,DELTA T SERVICES LTD +UNIT 6A WEST STATION YARD,SPITAL ROAD,MALDON,ESSEX,CM9 6TR,14500,WORKSHOP AND PREMISES,DELTA T SERVICES LTD +UNITS 27/28 WEST STATION YARD,SPITAL ROAD,MALDON,ESSEX,CM9 6TS,25250,Workshop and Premises,ATRIUM ASSET MANAGEMENT LIMITED +UNIT 8CDE WYCKE HILL BUSINESS PARK,WYCKE HILL,MALDON,ESSEX,CM9 6UZ,62500,WORKSHOP AND PREMISES,LAINDON TRADING LTD +KIT 2 LITTLE BRAXTED HALL,WITHAM ROAD,LITTLE BRAXTED,WITHAM ESSEX,CM8 3EU,22250,Warehouse and Premises, +LAURISTON BEACH HUT,JOYCES CHASE,GOLDHANGER,MALDON ESSEX,CM9 8AH,1300,BEACH HUT AND PREMISES, +CRICKETERS INN,33 CHURCH STREET,GOLDHANGER,MALDON ESSEX,CM9 8AR,7500,PUBLIC HOUSE AND PREMISES, +CRESTA HOUSE,2D CHURCH STREET,GOLDHANGER,MALDON ESSEX,CM9 8AR,1250,Camping Site and Premises, +GARAGE 1,69 HIGH STREET,TOLLESBURY,MALDON ESSEX,CM9 8QE,610,Store and Premises, +GARAGE 2,69 EAST STREET,TOLLESBURY,MALDON ESSEX,CM9 8QE,610,Store and Premises, +69 EAST STREET,TOLLESBURY,MALDON,ESSEX,CM9 8QE,10500,Offices and Premises, +10 HIGH STREET,TOLLESBURY,MALDON,ESSEX,CM9 8RG,7500,SHOP AND PREMISES, +8 HIGH STREET,TOLLESBURY,MALDON,ESSEX,CM9 8RG,7500,SHOP AND PREMISES, +19 WEST STREET,TOLLESBURY,,MALDON ESSEX,CM9 8RJ,5500,SHOP AND PREMISES, +SUITE 5 CUTTER HOUSE,WOODROLFE ROAD,TOLLESBURY,MALDON ESSEX,CM9 8SE,2850,OFFICES AND PREMISES,LEEDHOME LTD +SUITE B CUTTER HOUSE,WOODROLFE ROAD,TOLLESBURY,MALDON ESSEX,CM9 8SE,1450,Offices and Premises,LEEDHOME LTD +YACHT STORE,A5 WOODROLFE ROAD,TOLLESBURY,MALDON ESSEX,CM9 8TB,510,STORE AND PREMISES,FELLOWSHIP AFLOAT CHARITABLE +YACHT STORE,A6 WOODROLFE ROAD,TOLLESBURY,MALDON ESSEX,CM9 8TB,150,STORE AND PREMISES,FELLOWSHIP AFLOAT CHARITABLE +UNIT 5A HIGH HALL FARM,OXLEY HILL,TOLLESHUNT DARCY,MALDON ESSEX,CM9 8ES,3900,Offices and Premises, +QUEENS HEAD,15 NORTH STREET,TOLLESHUNT DARCY,MALDON ESSEX,CM9 8TF,25250,PUBLIC HOUSE AND PREMISES, +POLICE STATION,ADJ 31 TOLLESBURY ROAD,TOLLESHUNT D'ARCY,MALDON ESSEX,CM9 8UB,3200,POLICE STATION AND PREMISES,ESSEX POLICE AUTHORITY +FOUNDRY PLACE,WITHAM ROAD,TOLLESHUNT MAJOR,MALDON ESSEX,CM9 8JJ,16000,STORE AND PREMISES, +THE ESSEX BARN AT WICKS MANOR FARM,WITHAM ROAD,TOLLESHUNT MAJOR,MALDON ESSEX,CM9 8JU,8200,WORKSHOP AND PREMISES,HOWIES & SON LLP +KARENZA,BECKINGHAM STREET,TOLLESHUNT MAJOR,MALDON ESSEX,CM9 8LL,1850,Workshop and Premises, +UNIT H3 BECKINGHAM BUSINESS PARK,BECKINGHAM STREET,TOLLESHUNT MAJOR,MALDON ESSEX,CM9 8LQ,9800,WORKSHOP AND PREMISES,SIMPLY MEDS PRO LTD +UNIT 7B BECKINGHAM BUSINESS PARK,BECKINGHAM STREET,TOLLESHUNT MAJOR,MALDON,CM9 8LQ,7200,Store and Premises,SHAKESPEARE ENGINEERING +UNIT 7D BECKINGHAM BUSINESS PARK,BECKINGHAM STREET,TOLLESHUNT MAJOR,MALDON,CM9 8LQ,5900,Store and Premises,N B SHAKESPEARE LLP +UNIT 4 OAK FARM,LOAMY HILL ROAD,TOLLESHUNT MAJOR,MALDON ESSEX,CM9 8LS,2650,Workshop and Premises, +THE COMPASSES INN,12 COLCHESTER ROAD,GREAT TOTHAM,MALDON,CM9 8BZ,18000,PUBLIC HOUSE AND PREMISES, +UNIT 3-4 GREAT MOUNTAINS FARM,COLCHESTER ROAD,GREAT TOTHAM,MALDON ESSEX,CM9 8ED,12250,"STORE, OFFICES AND PREMISES",NAIO ENVIRONMENTAL LTD +CHASEFIELDS ESTATE SITE 11270,CATCHPOLE LANE,GREAT TOTHAM,MALDON,CM9 8PY,130,DOMESTIC FUEL INSTALLATION, +WORKSHOP AT SMITHS YARD,POST OFFICE LANE,LITTLE TOTHAM,MALDON ESSEX,CM9 8JL,2900,WORKSHOP AND PREMISES, +10 THE STREET,WICKHAM BISHOPS,WITHAM,ESSEX,CM8 3NN,14500,SHOP AND PREMISES, +ADVERTISING RIGHTS TO THE SIDE OF,38 THE STREET,WICKHAM BISHOPS,WITHAM ESSEX,CM8 3NN,500,ADVERTISING RIGHT AND PREMISES, +UNIT 1 MOAT LODGE INDUSTRIAL ESTATE,STOCK CHASE,HEYBRIDGE,MALDON,CM9 4AA,2550,WORKSHOP AND PREMISES, +ORANGE SITE ESX 0324,R/O BENTALLS INDUSTRIAL ESTATE,COLCHESTER ROAD,HEYBRIDGE MALDON ESSEX,CM9 4GD,10750,COMMUNICATION STATION AND PREMISES,EE LTD +ROOM I & K SUITE 3 3RD FLOOR,ICS HOUSE,HALL ROAD,HEYBRIDGE MALDON ESSEX,CM9 4LA,5800,OFFICES AND PREMISES,WATEROSE LIMITED +ROOM G SUITE3 3RD FLOOR,ICS HOUSE,HALL ROAD,HEYBRIDGE MALDON ESSEX,CM9 4LA,8800,OFFICES AND PREMISES,WATEROSE LIMITED +"ROOM A, SUITE 1, 3RD FLOOR",ICS HOUSE,HALL ROAD,HEYBRIDGE ESSEX,CM9 4LA,2500,OFFICES AND PREMISES,WATEROSE LIMITED +"ROOM M, SUITE 3, 3RD FLOOR",ICS HOUSE,HALL ROAD,HEYBRIDGE MALDON,CM9 4LA,3200,OFFICES AND PREMISES,WATEROSE LIMITED +SUITE 2 2ND FLOOR ICS HOUSE,HALL ROAD,HEYBRIDGE,MALDON ESSEX,CM9 4LA,20500,OFFICES AND PREMISES,WATEROSE LIMITED +ROOMS I & J SUITE 1 3RD FLOOR,ICS HOUSE,HALL ROAD,HEYBRIDGE MALDON ESSEX,CM9 4LA,8000,OFFICES AND PREMISES,WATEROSE LIMITED +ROOM H SUITE 1 3RD FLOOR,ICS HOUSE,HALL ROAD,HEYBRIDGE MALDON ESSEX,CM9 4LA,3200,OFFICE AND PREMISES,WATEROSE LIMITED +REMAINDER SUITE 1 3RD FLOOR,ICS HOUSE,HALL ROAD,HEYBRIDGE MALDON ESSEX,CM9 4LA,7500,OFFICES AND PREMISES,WATEROSE LIMITED +ROOMS C-L-N SUITE 3 3RD FLOOR,ICS HOUSE,HALL ROAD HEYBRIDGE,MALDON ESSEX,CM9 4LA,9400,Offices and Premises,WATEROSE LIMITED +ROOMS B C D & E SUITE 1 3RD FLOOR,ICS HOUSE HALL ROAD,HEYBRIDGE,MALDON ESSEX,CM9 4LA,13500,Offices and Premises,WATEROSE LIMITED +ROOM F SUITE 1 3RD FLOOR,ICS HOUSE HALL ROAD,HEYBRIDGE,ESSEX,CM9 4LA,4650,Offices and Premises,WATEROSE LIMITED +ROOMS A/B/C SUITE 2 THIRD FLOOR,ICS HOUSE,HALL ROAD,HEYBRIDGE ESSEX,CM9 4LA,19500,Offices and Premises,WATEROSE LIMITED +SUITE 1 2ND FLR ICS HOUSE,HALL ROAD,HEYBRIDGE,MALDON ESSEX,CM9 4LA,31250,Offices and Premises,WATEROSE LIMITED +ROOMS E & F 3RD FLR SUITE 3,ICS HOUSE,HALL ROAD,HEYBRIDGE ESSEX,CM9 4LA,8500,Offices and Premises,WATEROSE LIMITED +SUITE 3 2ND FLOOR,ICS HOUSE,HALL ROAD,HEYBRIDGE MALDON,CM9 4LA,32750,OFFICES AND PREMISES,WATEROSE LIMITED +STORE,3/5 ANCHOR LANE,HEYBRIDGE,MALDON ESSEX,CM9 4LS,10500,STORE AND PREMISES, +OFFICES AT THE BENTALLS IND COMPLEX,THE BENTALL COMPLEX,COLCHESTER ROAD,HEYBRIDGE ESSEX,CM9 4NW,128000,OFFICES AND PREMISES,PANTHER MALDON INDUSTRIAL LTD +FARLEIGH DAY HOSPICE,BENTALLS COMPLEX,COLCHESTER ROAD,HEYBRIDGE MALDON ESSEX,CM9 4NW,10750,DAY HOSPICE AND PREMISES,STEP START NURSERIES HEYBRIDGE +WORKSHOP 1A THE BOATYARD,BASIN ROAD,HEYBRIDGE BASIN,MALDON ESSEX,CM9 4RS,2150,WORKSHOP AND PREMISES,CRS MARINE LTD +WORKSHOP 2 BAY A,BASIN ROAD,HEYBRIDGE BASIN,MALDON ESSEX,CM9 4RS,1675,WORKSHOP AND PREMISES,CRS MARINE LTD +WORKSHOP 2 BAY B,BASIN ROAD,HEYBRIDGE BASIN,MALDON ESSEX,CM9 4RS,1625,WORKSHOP AND PREMISES,CRS MARINE LTD +WORKSHOP 2 BAY C,BASIN ROAD,HEYBRIDGE BASIN,MALDON ESSEX,CM9 4RS,1675,WORKSHOP AND PREMISES,CRS MARINE LTD +WORKSHOP 2 BAY D,BASIN ROAD,HEYBRIDGE BASIN,MALDON ESSEX,CM9 4RS,1675,WORKSHOP AND PREMISES,CRS MARINE LTD +WORKSHOP 2 BAY E,BASIN ROAD,HEYBRIDGE BASIN,MALDON ESSEX,CM9 4RS,2000,WORKSHOP AND PREMISES,CRS MARINE LTD +WORKSHOP 2 BAY F,BASIN ROAD,HEYBRIDGE BASIN,MALDON ESSEX,CM9 4RS,1675,WORKSHOP AND PREMISES,CRS MARINE LTD +WORKSHOP 1,BASIN ROAD,HEYBRIDGE BASIN,MALDON ESSEX,CM9 4RS,4400,WORKSHOP AND PREMISES,CRS MARINE LTD +HUT BASIN BEACH,THE BASIN,HEYBRIDGE BASIN,MALDON ESSEX,CM9 4RS,430,BEACH HUT AND PREMISES,(NDR) MALDON DISTRICT COUNCIL +OFFICES AT LEANNA HOUSE,LOCK HILL,HEYBRIDGE,MALDON ESSEX,CM9 4RX,10500,OFFICES AND PREMISES,APPLEBEE HOLDINGS LTD +18 CANAL BANK,HEYBRIDGE,MALDON,ESSEX,CM9 4SD,820,STORE AND PREMISES,ESSEX WATERWAYS LTD +THE STUDIO,CANAL BANK,HEYBRIDGE,MALDON ESSEX,CM9 4SD,3950,STORE AND PREMISES,ESSEX WATERWAYS LTD +RED SHED,LOCK HILL,HEYBRIDGE,MALDON ESSEX,CM9 6RA,4800,STORE AND PREMISES,MACRON GROUP LIMITED +POST OFFICE,BURNHAM ROAD,ALTHORNE,CHELMSFORD ESSEX,CM3 6BU,4450,SHOP AND PREMISES, +PLOT 86 SMUGGLERS CLUB GROUND,BRIDGEMARSH LANE,ALTHORNE,ESSEX,CM3 6DQ,890,SELF CATERING HOLIDAY UNIT AND PREMISES, +THE STORES AT WREKIN FARM,BURNHAM ROAD,ALTHORNE,ESSEX,CM3 6DT,1050,"STORE, WORKSHOP AND PREMISES", +ASHELDHAM YOUTH CHURCH,HALL ROAD,ASHELDHAM,SOUTHMINSTER ESSEX,CM0 7JF,3450,HALL AND PREMISES,CHELMSFORD DIOCESAN BOARD +OBSERVATORY ADJ ST PETERS COTTAGE,EAST END ROAD,BRADWELL ON SEA,SOUTHMINSTER ESSEX,CM0 7PN,2900,Observatory and Premises, +THE CRICKETERS INN,EAST END ROAD,BRADWELL ON SEA,SOUTHMINSTER ESSEX,CM0 7PT,2400,PUBLIC HOUSE AND PREMISES, +CAR PARK AT RUNWAY 2,BRADWELL AIRFIELD,BRADWELL ON SEA,SOUTHMINSTER ESSEX,CM0 7QF,66500,Land Used for Car Park and Premises,NUCLEAR DECOMMISSIONING +PORTABLE BUILDING BRADWELL MARINA,WATERSIDE,BRADWELL ON SEA,SOUTHMINSTER ESSEX,CM0 7RB,3000,Offices and Premises,PORT FLAIR LIMITED +RADAR INSTALLATION,HOCKLEY LANE,BRADWELL ON SEA,SOUTHMINSTER ESSEX,CM1 3BN,30250,COMMUNICATION STATION AND PREMISES,STRUTT & PARKER (FARMS) LTD +UNIT 2 MAYLAND INDUSTRIAL ESTATE,STEEPLE ROAD,MAYLAND,CHELMSFORD ESSEX,CM3 6AX,7200,WORKSHOP AND PREMISES, +28 STEEPLE ROAD,MAYLAND,CHELMSFORD,ESSEX,CM3 6BB,11250,VEHICLE REPAIR WORKSHOP AND PREMISES, +UNIT 9 MAYLAND INDUSTRIAL ESTATE,STEEPLE ROAD,MAYLAND,CHELMSFORD,CM3 6BB,7300,WORKSHOP AND PREMISES,ALLENGRAY KITCHENS LTD +PLOT A MAYFLOWER NURSERY,MILL ROAD,MAYLAND,ESSEX,CM3 6EQ,16250,LAND USED FOR STORAGE AND PREMISES,DOVE'S COMMERCIAL HOLDINGS LTD +YARD 1 MAYFLOWER NURSERY,MILL ROAD,MAYLAND,ESSEX,CM3 6EQ,9400,LAND USED FOR STORAGE AND PREMISES,DOVE'S COMMERCIAL HOLDINGS LTD +UNIT 2 JEAGOR FARM,STEEPLE ROAD,MAYLAND,CHELMSFORD,CM3 6RR,5500,WORKSHOP AND PREMISES, +UNIT 1 JEAGOR FARM,STEEPLE ROAD,MAYLAND,CHELMSFORD,CM3 6RR,4800,WORKSHOP AND PREMISES, +POLICE STATION,QUEENBOROUGH ROAD,SOUTHMINSTER,ESSEX,CM0 7AB,20000,POLICE STATION AND PREMISES,STOCKVALE LTD +SOUTHMINSTER LIBRARY,QUEENBOROUGH ROAD,SOUTHMINSTER,ESSEX,CM0 7AB,4700,LIBRARY AND PREMISES,STOCKVALE LIMITED +HIGHLANDS YARD,QUEENBOROUGH ROAD,SOUTHMINSTER,ESSEX,CM0 7AD,22250,LAND USED FOR STORAGE AND PREMISES, +MODULAR BUILDING - CAR PARK,HIGH STREET,SOUTHMINSTER,ESSEX,CM0 7AY,3950,Offices and Premises, +UNIT 4 WITCHARDS FARM,SCOTTS HILL,SOUTHMINSTER,ESSEX,CM0 7BG,5000,Workshop and Premises,K - WOOD PROPERTIES LTD +CAKE HOUSE AT NEW MOORE FARM,TILLINGHAM ROAD,SOUTHMINSTER,ESSEX,CM0 7DS,5100,WORKSHOP AND PREMISES, +STORE & PREMISES,STATION ROAD,SOUTHMINSTER,ESSEX,CM0 7EW,11250,WORKSHOP AND PREMISES, +NEWHOUSE FARM,MALDON ROAD,STEEPLE,SOUTHMINSTER ESSEX,CM0 7BD,3300,WORKSHOP AND PREMISES, +PART BARN AT BATTS FARM,BATTS ROAD,STEEPLE,SOUTHMINSTER ESSEX,CM0 7LE,4500,WORKSHOP AND PREMISES, +R/O VILLAGE HALL,VICARAGE LANE,"TILLINGHAM, SOUTHMINSTER",ESSEX,CM0 7TW,1825,CLUB HOUSE AND PREMISES,TILLINGHAM YOUTH & HOBBIES CLUB +80 HIGH STREET,BURNHAM ON CROUCH,ESSEX,,CM0 8AA,19000,RESTAURANT AND PREMISES,TCO PROPERTIES LTD +36 HIGH STREET,BURNHAM-ON-CROUCH,,ESSEX,CM0 8AA,7800,OFFICES AND PREMISES,TRALAND LLP +38 HIGH STREET,BURNHAM-ON-CROUCH,,ESSEX,CM0 8AA,14250,SHOP AND PREMISES,MINERVA SIPP +41 HIGH STREET,BURNHAM-ON-CROUCH,,ESSEX,CM0 8AG,5600,SHOP AND PREMISES, +UNIT 1 MILDMAY HOUSE,FOUNDRY LANE,BURNHAM ON CROUCH,ESSEX,CM0 8BL,3050,OFFICES AND PREMISES, +GROUND FLOOR,THE CHANDLERY,FOUNDRY LANE,BURNHAM ON CROUCH,CM0 8BL,19500,SHOP AND PREMISES,BURNHAM YACHT HARBOUR +161/161A STATION ROAD,BURNHAM-ON-CROUCH,,ESSEX,CM0 8HJ,7800,SHOP AND PREMISES,PEABERRIES LTD +REAR OF,34 HIGH STREET,BURNHAM ON CROUCH,ESSEX,CM0 8JU,7000,STORE AND PREMISES,TRALAND LLP +UNIT 3A WICK ROAD INDUSTRIAL SITE,WICK ROAD,BURNHAM ON CROUCH,ESSEX,CM0 8LT,4050,Store and Premises,BURNHAM STORAGE LTD +UNIT 20 DAMMERWICK FARM,MARSH ROAD,BURNHAM ON CROUCH,ESSEX,CM0 8NB,3800,WORKSHOP AND PREMISES,STRUTT & PARKER (FARMS) LTD +WEST WICK,MARSH ROAD,THE MARSHES,BURNHAM ON CROUCH ESSEX,CM0 8NE,3950,Workshop and Premises, +UNIT Z MILDMAY INDUSTRIAL ESTATE,FOUNDRY LANE,BURNHAM ON CROUCH,ESSEX,CM0 8SH,9200,WAREHOUSE AND PREMISES, +UNIT 2 MILDMAY HOUSE,FOUNDRY LANE,BURNHAM ON CROUCH,ESSEX,CM0 8UN,4150,SHOP AND PREMISES, diff --git a/tests/data/sample/pdf/NDR_Empty_Property_List___March_2026.pdf b/tests/data/sample/pdf/NDR_Empty_Property_List___March_2026.pdf new file mode 100644 index 0000000..b50478f Binary files /dev/null and b/tests/data/sample/pdf/NDR_Empty_Property_List___March_2026.pdf differ diff --git a/tests/data/sample/pdf/NNDR EXEMPTIONS.pdf b/tests/data/sample/pdf/NNDR EXEMPTIONS.pdf new file mode 100644 index 0000000..c2dea44 Binary files /dev/null and b/tests/data/sample/pdf/NNDR EXEMPTIONS.pdf differ diff --git a/tests/data/sample/pdf/NNDR RELIEFS 15 04 2026 pages 1-3.pdf b/tests/data/sample/pdf/NNDR RELIEFS 15 04 2026 pages 1-3.pdf new file mode 100644 index 0000000..0515a4d Binary files /dev/null and b/tests/data/sample/pdf/NNDR RELIEFS 15 04 2026 pages 1-3.pdf differ diff --git a/tests/data/sample/pdf/maldon-page-1-scanned.pdf b/tests/data/sample/pdf/maldon-page-1-scanned.pdf new file mode 100644 index 0000000..1f611db Binary files /dev/null and b/tests/data/sample/pdf/maldon-page-1-scanned.pdf differ diff --git a/tests/data/sample/pdf/west-berkshire-exemptions.csv b/tests/data/sample/pdf/west-berkshire-exemptions.csv new file mode 100644 index 0000000..adb0013 --- /dev/null +++ b/tests/data/sample/pdf/west-berkshire-exemptions.csv @@ -0,0 +1,260 @@ +Propref,Pee Code,Description,Prop Exemption Start Date,Prop Exemption End Date +NN03905470021016,UNOC 3MTHS,Unoccupied for 3 months,30-JAN-2026,29-APR-2026 +NN03905470019001,UNOC 3MTHS,Unoccupied for 3 months,01-FEB-2026,30-APR-2026 +NN05203830001004,UNOC 3MTHS,Unoccupied for 3 months,20-JAN-2026,19-APR-2026 +NN03905350004253,UNOC 3MTHS,Unoccupied for 3 months,27-FEB-2026,26-MAY-2026 +NN05300601600088,UNOC 3MTHS,Unoccupied for 3 months,01-FEB-2026,30-APR-2026 +NN05300601600049,UNOC 3MTHS,Unoccupied for 3 months,01-FEB-2026,30-APR-2026 +NN03905350013027,UNOC 3MTHS,Unoccupied for 3 months,25-MAR-2026,02-MAY-2026 +NN03300780030062,UNOC 3MTHS,Unoccupied for 3 months,30-JAN-2026,26-APR-2026 +NN05303400011389,UNOC 3MTHS,Unoccupied for 3 months,03-MAR-2026,02-JUN-2026 +NN06104400025114,UNOC 3MTHS,Unoccupied for 3 months,17-JAN-2026,16-APR-2026 +NN03302600005005,LISTED B,Listed Building,01-APR-2013, +NN01304900033001,LISTED B,Listed Building,28-FEB-2011, +NN03604800006123,LISTED B,Listed Building,01-APR-2017, +NN03905350283009,LISTED B,Listed Building,13-NOV-2017, +NN05505800031121,LISTED B,Listed Building,28-SEP-2018, +NN02904510001050,LISTED B,Listed Building,01-APR-2017, +NN02904510001049,LISTED B,Listed Building,01-APR-2017, +NN02904510001048,LISTED B,Listed Building,01-APR-2017, +NN02904510001029,LISTED B,Listed Building,01-APR-2017, +NN02904510001030,LISTED B,Listed Building,01-APR-2017, +NN02904510001038,LISTED B,Listed Building,01-APR-2017, +NN02904510001037,LISTED B,Listed Building,01-APR-2017, +NN02904510001036,LISTED B,Listed Building,01-APR-2017, +NN02904510001035,LISTED B,Listed Building,01-APR-2017, +NN02904510001056,LISTED B,Listed Building,01-APR-2017, +NN02904510001055,LISTED B,Listed Building,01-APR-2017, +NN02904510001054,LISTED B,Listed Building,01-APR-2017, +NN02904510001045,LISTED B,Listed Building,01-APR-2017, +NN02904510001044,LISTED B,Listed Building,01-APR-2017, +NN02904510001043,LISTED B,Listed Building,01-APR-2017, +NN02904510001042,LISTED B,Listed Building,01-APR-2017, +NN02904510001041,LISTED B,Listed Building,01-APR-2017, +NN02904510001040,LISTED B,Listed Building,01-APR-2017, +NN02904510001053,LISTED B,Listed Building,01-APR-2017, +NN03902060006020,LISTED B,Listed Building,11-AUG-2020, +NN03902060006019,LISTED B,Listed Building,11-AUG-2020, +NN03902060006021,LISTED B,Listed Building,11-AUG-2020, +NN04104950001003,LISTED B,Listed Building,08-JUL-2021, +NN03604800006124,LISTED B,Listed Building,19-SEP-2021, +NN05201300081018,LISTED B,Listed Building,30-NOV-2017, +NN03302600019344,LISTED B,Listed Building,30-SEP-2021, +NN03302600019062,LISTED B,Listed Building,17-DEC-2021, +NN03904130030134,LISTED B,Listed Building,05-JAN-2022, +NN0390413003012A,LISTED B,Listed Building,05-JAN-2022, +NN03905350070011,LISTED B,Listed Building,01-APR-2022, +NN03901440024024,LISTED B,Listed Building,24-DEC-2021, +NN0390144002403A,LISTED B,Listed Building,24-DEC-2021, +NN03901440024023,LISTED B,Listed Building,24-DEC-2021, +NN0390144002402A,LISTED B,Listed Building,24-DEC-2021, +NN03901440024022,LISTED B,Listed Building,24-DEC-2021, +NN0390785000100A,LISTED B,Listed Building,02-JAN-2020, +NN03907850004012,LISTED B,Listed Building,29-JUN-2021, +NN03907850007114,LISTED B,Listed Building,31-MAY-2018, +NN0330260000800B,LISTED B,Listed Building,04-JUL-2022, +NN03905350044104,LISTED B,Listed Building,01-JAN-2023, +NN03905350044117,LISTED B,Listed Building,01-JAN-2023, +NN03906680027037,LISTED B,Listed Building,26-SEP-2022, +NN0520135001840B,LISTED B,Listed Building,15-MAY-2023, +NN03106400003110,LISTED B,Listed Building,02-JUN-2023, +NN03300500020005,LISTED B,Listed Building,02-OCT-2023, +NN02904510001051,LISTED B,Listed Building,11-NOV-2020, +NN03905350009031,LISTED B,Listed Building,13-SEP-2024, +NN03300800004001,LISTED B,Listed Building,05-AUG-2024, +NN03302600024240,LISTED B,Listed Building,21-MAR-2024, +NN03302600024238,LISTED B,Listed Building,06-JUN-2024, +NN0390559000403B,LISTED B,Listed Building,04-DEC-2024, +NN02904510001064,LISTED B,Listed Building,01-APR-2023, +NN02904510001063,LISTED B,Listed Building,01-APR-2023, +NN0390559000405A,LISTED B,Listed Building,25-FEB-2025, +NN03902060041006,LISTED B,Listed Building,31-OCT-2024, +NN03900720072024,LISTED B,Listed Building,01-AUG-2024, +NN03302600035012,LISTED B,Listed Building,01-JAN-2024, +NN03909550001124,LISTED B,Listed Building,15-JUL-2025, +NN02904500274347,LISTED B,Listed Building,06-NOV-2025, +NN03900720072023,LISTED B,Listed Building,01-AUG-2024, +NN03900720072020,LISTED B,Listed Building,01-MAY-2025, +NN02904500274351,LISTED B,Listed Building,06-NOV-2025, +NN03905590004040,LISTED B,Listed Building,05-MAR-2025, +NN03302600128001,LISTED B,Listed Building,17-DEC-2025, +NN03905350069014,LISTED B,Listed Building,25-JUL-2024, +NN03900720044121,LISTED B,Listed Building,29-AUG-2024, +NN03302600024164,LISTED B,Listed Building,12-JUL-2025, +NN03300500004002,LISTED B,Listed Building,13-JAN-2024, +NN0390072015314B,LISTED B,Listed Building,15-JUL-2024, +NN03905350069029,LISTED B,Listed Building,27-JAN-2024, +NN03900720072016,LISTED B,Listed Building,01-MAR-2025, +NN0290450027427A,LISTED B,Listed Building,12-APR-2025, +NN03906680039028,LISTED B,Listed Building,01-JAN-2026, +NN03900720074023,LISTED B,Listed Building,01-SEP-2025, +NN05200850050031,LISTED B,Listed Building,29-SEP-2023, +NN03106400003004,QUAL IND,Qualifying Industrial,01-DEC-2025,31-MAY-2026 +NN05305600057117,QUAL IND,Qualifying Industrial,21-OCT-2025,20-APR-2026 +NN03306500030409,QUAL IND,Qualifying Industrial,19-NOV-2025,18-MAY-2026 +NN00101700008129,QUAL IND,Qualifying Industrial,03-DEC-2025,02-JUN-2026 +NN08301650064147,QUAL IND,Qualifying Industrial,01-MAR-2026,31-AUG-2026 +NN00102200002179,QUAL IND,Qualifying Industrial,01-NOV-2025,30-APR-2026 +NN05205900018029,QUAL IND,Qualifying Industrial,05-DEC-2025,04-JUN-2026 +NN08104950001004,QUAL IND,Qualifying Industrial,31-JAN-2026,30-JUL-2026 +NN02901350005112,QUAL IND,Qualifying Industrial,24-OCT-2025,23-APR-2026 +NN00104200025200,OWNERLIQ,Owner is a Liquidator,19-DEC-2024, +NN0290320002011A,OWNERLIQ,Owner is a Liquidator,30-AUG-2024, +NN05201500013032,WUP,Winding up Proceedings,15-OCT-2018, +NN0520150019117,WUP,Winding up Proceedings,15-MAR-2019, +NN0520150019118,WUP,Winding up Proceedings,15-MAR-2019, +NN00101700007185,WUP,Winding up Proceedings,23-NOV-2022, +NN05206350193166,WUP,Winding up Proceedings,30-NOV-2022, +NN04104200019032,WUP,Winding up Proceedings,31-OCT-2022, +NN05205270002116,WUP,Winding up Proceedings,23-DEC-2024, +NN03905650023005,WUP,Winding up Proceedings,08-JUL-2025, +NN03207200014119,WUP,Winding up Proceedings,08-APR-2025, +NN05300601310025,WUP,Winding up Proceedings,21-AUG-2025, +NN05300601310023,WUP,Winding up Proceedings,21-AUG-2025, +NN05300601310027,WUP,Winding up Proceedings,21-AUG-2025, +NN08100460384110,WUP,Winding up Proceedings,29-AUG-2025, +NN0520086000500B,WUP,Winding up Proceedings,04-SEP-2025, +NN05300601310026,WUP,Winding up Proceedings,21-AUG-2025, +NN00703200033119,PEND TOR,Pending TOR,19-MAR-2025, +NN0520445001210C,PEND TOR,Pending TOR,17-MAR-2026, +NN03300500002007,PEND DOM,Pending Domestic Assess,12-JUN-2025, +NN01701800072112,PEND DOM,Pending Domestic Assess,15-AUG-2025, +NN08301650042134,PEND DOM,Pending Domestic Assess,09-MAR-2026, +NN04306000030002,PEND DOM,Pending Domestic Assess,15-DEC-2025, +NN08205800032014,PEND DOM,Pending Domestic Assess,21-MAY-2025, +NN05204360004005,PEND DOM,Pending Domestic Assess,20-FEB-2026, +NN03907070003106,PEND DOM,Pending Domestic Assess,31-JAN-2026, +NN03902060056001,PEND DOM,Pending Domestic Assess,13-FEB-2026, +NN01302050078160,PEND DOM,Pending Domestic Assess,06-FEB-2026, +NN04403900080123,PEND DOM,Pending Domestic Assess,27-MAR-2026, +NN03902060048003,PEND DOM,Pending Domestic Assess,07-NOV-2025, +NN05206300009003,PEND DOM,Pending Domestic Assess,27-AUG-2025, +NN0390702000100A,EMPLAND,Empty Land,30-JUN-2012, +NN0390024014700B,EMPLAND,Empty Land,31-MAY-2013, +NN04301400001001,EMPLAND,Empty Land,25-JUN-2013, +NN02904600069002,EMPLAND,Empty Land,25-JUN-2013, +NN05202100020002,EMPLAND,Empty Land,18-OCT-2010, +NN05505300078004,EMPLAND,Empty Land,26-MAR-2013, +NN05504320040006,EMPLAND,Empty Land,05-JUN-2013, +NN05205150108111,EMPLAND,Empty Land,01-APR-2010, +NN08405100011111,EMPLAND,Empty Land,05-OCT-2017, +NN05300601600165,EMPLAND,Empty Land,04-MAR-2019, +NN05300601600166,EMPLAND,Empty Land,04-MAR-2019, +NN05300601600163,EMPLAND,Empty Land,04-MAR-2019, +NN05301700100114,EMPLAND,Empty Land,28-AUG-2020, +NN00603600080161,EMPLAND,Empty Land,02-APR-2020, +NN00104200070005,EMPLAND,Empty Land,22-JUL-2019, +NN05305601330025,EMPLAND,Empty Land,10-JUL-2020, +NN05303400019256,EMPLAND,Empty Land,13-SEP-2021, +NN03905350034001,EMPLAND,Empty Land,03-DEC-2021, +NN02904510001033,EMPLAND,Empty Land,01-APR-2022, +NN03203200001024,EMPLAND,Empty Land,21-JUN-2022, +NN00104200028109,EMPLAND,Empty Land,01-APR-2022, +NN00104200028110,EMPLAND,Empty Land,01-APR-2022, +NN03904130005135,EMPLAND,Empty Land,08-DEC-2017, +NN05300601510032,EMPLAND,Empty Land,14-NOV-2022, +NN05300601510052,EMPLAND,Empty Land,14-NOV-2022, +NN0130440000615A,EMPLAND,Empty Land,31-AUG-2024, +NN02904510001032,EMPLAND,Empty Land,01-JAN-2023, +NN0170810000222B,EMPLAND,Empty Land,03-SEP-2019, +NN03306500010115,EMPLAND,Empty Land,25-NOV-2022, +NN02904510001061,EMPLAND,Empty Land,06-JUN-2025, +NN08104950054128,EMPLAND,Empty Land,01-JAN-2024, +NN02904500104024,EMPLAND,Empty Land,21-MAY-2025, +NN05300601600199,EMPLAND,Empty Land,10-MAR-2023, +NN00105000035057,EMPLAND,Empty Land,01-FEB-2025, +NN08104460050606,EMPLAND,Empty Land,24-JUN-2025, +NN00704400011137,EMPLAND,Empty Land,14-MAY-2025, +NN02904500139004,EMPLAND,Empty Land,01-APR-2024, +NN0530340006511A,RV<2900,Rv Less Than £2900,01-APR-2017, +NN06201200013125,RV<2900,Rv Less Than £2900,04-APR-2017, +NN03108400028112,RV<2900,Rv Less Than £2900,01-AUG-2019, +NN05305601330010,RV<2900,Rv Less Than £2900,25-FEB-2020, +NN05300601230077,RV<2900,Rv Less Than £2900,25-FEB-2020, +NN05300601230067,RV<2900,Rv Less Than £2900,25-FEB-2020, +NN05300601320005,RV<2900,Rv Less Than £2900,25-FEB-2020, +NN05300601220071,RV<2900,Rv Less Than £2900,25-FEB-2020, +NN05300601220077,RV<2900,Rv Less Than £2900,25-FEB-2020, +NN05300601220078,RV<2900,Rv Less Than £2900,25-FEB-2020, +NN05300601220083,RV<2900,Rv Less Than £2900,25-FEB-2020, +NN05300601220070,RV<2900,Rv Less Than £2900,25-FEB-2020, +NN02001800006012,RV<2900,Rv Less Than £2900,15-JUN-2020, +NN02001800006111,RV<2900,Rv Less Than £2900,15-JUN-2020, +NN02001800006110,RV<2900,Rv Less Than £2900,15-JUN-2020, +NN02001800006119,RV<2900,Rv Less Than £2900,15-JUN-2020, +NN05305601330018,RV<2900,Rv Less Than £2900,25-FEB-2020, +NN05300601220073,RV<2900,Rv Less Than £2900,25-FEB-2020, +NN05300601410005,RV<2900,Rv Less Than £2900,25-FEB-2020, +NN00101700065145,RV<2900,Rv Less Than £2900,01-AUG-2020, +NN05300601430006,RV<2900,Rv Less Than £2900,14-JAN-2021, +NN05300601220084,RV<2900,Rv Less Than £2900,25-FEB-2020, +NN05300601230059,RV<2900,Rv Less Than £2900,26-FEB-2020, +NN05300601410036,RV<2900,Rv Less Than £2900,25-FEB-2020, +NN05300601220086,RV<2900,Rv Less Than £2900,25-FEB-2020, +NN05300601220069,RV<2900,Rv Less Than £2900,14-NOV-2021, +NN05300601220068,RV<2900,Rv Less Than £2900,14-NOV-2021, +NN05300601220067,RV<2900,Rv Less Than £2900,14-NOV-2021, +NN05300601410038,RV<2900,Rv Less Than £2900,16-JAN-2021, +NN0390830002511A,RV<2900,Rv Less Than £2900,16-FEB-2022, +NN05300601600105,RV<2900,Rv Less Than £2900,25-JUN-2021, +NN01208100010228,RV<2900,Rv Less Than £2900,01-NOV-2022, +NN03904880003121,RV<2900,Rv Less Than £2900,27-MAR-2023, +NN05300601410031,RV<2900,Rv Less Than £2900,24-AUG-2023, +NN05300601410026,RV<2900,Rv Less Than £2900,24-AUG-2023, +NN05300601410028,RV<2900,Rv Less Than £2900,24-AUG-2023, +NN05300601410024,RV<2900,Rv Less Than £2900,31-AUG-2023, +NN05300601410037,RV<2900,Rv Less Than £2900,31-AUG-2023, +NN05300601410022,RV<2900,Rv Less Than £2900,31-AUG-2023, +NN01306500014326,RV<2900,Rv Less Than £2900,15-NOV-2024, +NN05300601310022,RV<2900,Rv Less Than £2900,09-OCT-2024, +NN05305601330012,RV<2900,Rv Less Than £2900,03-SEP-2024, +NN05300601310013,RV<2900,Rv Less Than £2900,14-OCT-2023, +NN05300601310012,RV<2900,Rv Less Than £2900,14-OCT-2023, +NN05300601310020,RV<2900,Rv Less Than £2900,24-JAN-2024, +NN05300601410042,RV<2900,Rv Less Than £2900,01-APR-2024, +NN05300601430005,RV<2900,Rv Less Than £2900,21-NOV-2023, +NN05300601310024,RV<2900,Rv Less Than £2900,09-OCT-2024, +NN05300601420218,RV<2900,Rv Less Than £2900,01-APR-2023, +NN05300601430047,RV<2900,Rv Less Than £2900,01-APR-2024, +NN05300601430003,RV<2900,Rv Less Than £2900,21-NOV-2023, +NN05305601600096,RV<2900,Rv Less Than £2900,21-NOV-2023, +NN05300601430002,RV<2900,Rv Less Than £2900,21-NOV-2023, +NN05300601430007,RV<2900,Rv Less Than £2900,21-NOV-2023, +NN05305601600006,RV<2900,Rv Less Than £2900,01-APR-2023, +NN05204350030111,RV<2900,Rv Less Than £2900,30-SEP-2023, +NN05305601330022,RV<2900,Rv Less Than £2900,03-SEP-2024, +NN05300601600001,RV<2900,Rv Less Than £2900,01-JUN-2025, +NN03906710010012,RV<2900,Rv Less Than £2900,01-MAY-2024, +NN05300601420215,RV<2900,Rv Less Than £2900,06-JUL-2022, +NN02904500274438,RV<2900,Rv Less Than £2900,01-DEC-2025, +NN05300601310033,RV<2900,Rv Less Than £2900,07-AUG-2024, +NN05300601600009,RV<2900,Rv Less Than £2900,01-MAR-2025, +NN03906710010001,RV<2900,Rv Less Than £2900,01-NOV-2025, +NN05300601600087,RV<2900,Rv Less Than £2900,01-MAY-2024, +NN03905050001002,RV<2900,Rv Less Than £2900,01-OCT-2025, +NN05300601430009,RV<2900,Rv Less Than £2900,21-NOV-2023, +NN05300601600085,RV<2900,Rv Less Than £2900,01-FEB-2026, +NN01206750025115,RV<2900,Rv Less Than £2900,01-NOV-2024, +NN05300601310015,RV<2900,Rv Less Than £2900,24-JAN-2024, +NN05300601600099,RV<2900,Rv Less Than £2900,01-DEC-2025, +NN05201500007017,RV<2900,Rv Less Than £2900,17-MAY-2023, +NN05300601230047,RV<2900,Rv Less Than £2900,01-APR-2025, +NN03906710010010,RV<2900,Rv Less Than £2900,01-NOV-2025, +NN05300601600090,RV<2900,Rv Less Than £2900,01-FEB-2024, +NN05300601310018,RV<2900,Rv Less Than £2900,24-JAN-2024, +NN05300601310016,RV<2900,Rv Less Than £2900,24-JAN-2024, +NN05300601310035,RV<2900,Rv Less Than £2900,07-AUG-2024, +NN05300601310028,RV<2900,Rv Less Than £2900,07-AUG-2024, +NN03906310032020,RV<2900,Rv Less Than £2900,20-AUG-2024, +NN03108400028111,RV<2900,Rv Less Than £2900,01-APR-2023, +NN0290450019219A,RV<2900,Rv Less Than £2900,01-FEB-2026, +NN05300601420216,RV<2900,Rv Less Than £2900,07-JUL-2022, +NN05300601420211,RV<2900,Rv Less Than £2900,06-JUL-2022, +NN05300601600091,RV<2900,Rv Less Than £2900,01-JUN-2025, +NN05300601600097,RV<2900,Rv Less Than £2900,01-MAR-2025, +NN01304400006161,RV<2900,Rv Less Than £2900,06-FEB-2026, +NN0390671001009A,RV<2900,Rv Less Than £2900,03-DEC-2025, +NN05300601310032,RV<2900,Rv Less Than £2900,07-AUG-2024, +NN00304000003005,RV<2900,Rv Less Than £2900,01-NOV-2024, +NN03906310032019,RV<2900,Rv Less Than £2900,24-SEP-2025, +NN08100460383087,RV<2900,Rv Less Than £2900,01-NOV-2025, diff --git a/tests/data/sample/pdf/west-berkshire-reliefs.csv b/tests/data/sample/pdf/west-berkshire-reliefs.csv new file mode 100644 index 0000000..b64b0fa --- /dev/null +++ b/tests/data/sample/pdf/west-berkshire-reliefs.csv @@ -0,0 +1,75 @@ +Propref,Property Address,Ret Code,R/V +NN00101700004185,"Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA",SBR3,13250 +NN03905740027005,"27-31, Meadow Road, Newbury, RG14 7AH",SBRE,10250 +NN04105100003115,"At Station Road, Pangbourne, Reading, RG8 7AN",RH0,8600 +NN04105100003115,"At Station Road, Pangbourne, Reading, RG8 7AN",SBR3,8600 +NN0010170000401A,"1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN",SBR1,16250 +NN00101700016144,"4, Titan House, Calleva Park, Aldermaston, Reading, Berkshire, RG7 8AA",SBRE,4900 +NN0010420000111A,"Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW",MAN,5400 +NN0010420000111A,"Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW",RH0,5400 +NN00104200047005,"Unit 47, Youngs Industrial Estate, Paices Hill, Aldermaston, Reading, RG7 4PW",RH0,56500 +NN00104200047005,"Unit 47, Youngs Industrial Estate, Paices Hill, Aldermaston, Reading, RG7 4PW",SBR3,56500 +NN00106600007112,"The Old Mill, Station Road, Aldermaston, Reading, Berkshire, RG7 4LB",RH0,78500 +NN00106600007112,"The Old Mill, Station Road, Aldermaston, Reading, Berkshire, RG7 4LB",SBR3,78500 +NN00201200001117,"The Bell Inn, Aldworth, Reading, RG8 9SE",PU15,24000 +NN00201200001117,"The Bell Inn, Aldworth, Reading, RG8 9SE",RH0,24000 +NN00201200001117,"The Bell Inn, Aldworth, Reading, RG8 9SE",SBR3,24000 +NN00301800006112,"Ashampstead Village Hall, Ashampstead, Reading, Berkshire, RG8 8SW",MAN,3600 +NN00301800006112,"Ashampstead Village Hall, Ashampstead, Reading, Berkshire, RG8 8SW",RH0,3600 +NN00404200072116,"Church Farm, The Pavilion, Lower, Basildon, Berkshire, RG8 9NH",MAN,147000 +NN00404200072116,"Church Farm, The Pavilion, Lower, Basildon, Berkshire, RG8 9NH",RH0,147000 +NN00600800121119,"Village Hall, Oxford Road, Beedon, Newbury, Berkshire, RG20 8SH",MAN,2950 +NN00600800121119,"Village Hall, Oxford Road, Beedon, Newbury, Berkshire, RG20 8SH",RH0,2950 +NN0070680000111B,"Clubroom And Prems, Beenham, Reading, RG7 5NN",SBRE,4100 +NN00706800003116,"The Beenham Victory Hall, Picklepythe Lane, Beenham, Reading, RG7 5NT",MAN,4950 +NN00902100033112,"Bradfield Church Of England Primary School, Cock Lane, Bradfield Southend, Reading, RG7 6HR",MAN,40500 +NN00906300052118,"The Garage, Southend Road, Bradfield Southend, Reading, RG7 6ES",RH0,13500 +NN00906300052118,"The Garage, Southend Road, Bradfield Southend, Reading, RG7 6ES",SBR3,13500 +NN00906300054113,"Bradstone House, Southend Road, Southend, Reading, Berkshire, RG7 6EU",RH0,2950 +NN00906300054113,"Bradstone House, Southend Road, Southend, Reading, Berkshire, RG7 6EU",SBRE,2950 +NN00906500055119,"Bradfield Riding Centre, Bradfield, Reading, RG7 6AJ",RH0,7100 +NN00906500055119,"Bradfield Riding Centre, Bradfield, Reading, RG7 6AJ",SBRE,7100 +NN00906750004116,"Bradfield Social Club, Union Road, Bradfield, Reading, Berkshire, RG7 6AE",SBRE,10250 +NN01001200007119,"Brightwalton C Of E Primary School, Brightwalton, Newbury, RG20 7BN",MAN,29000 +NN01001200010111,"The Village Hall, Common Lane, Brightwalton, Newbury, Berkshire, RG20 7BN",MAN,5800 +NN01001200010111,"The Village Hall, Common Lane, Brightwalton, Newbury, Berkshire, RG20 7BN",RH0,5800 +NN01102100017113,"Brimpton Village Hall, Crookham Road, Brimpton, Reading, Berkshire, RG7 4ST",MAN,2650 +NN01102100017113,"Brimpton Village Hall, Crookham Road, Brimpton, Reading, Berkshire, RG7 4ST",RH0,2650 +NN0120120005911B,"Village Hall, Broad Lane, Upper Bucklebury, Reading, RG7 6QH",MAN,13500 +NN0120120005911B,"Village Hall, Broad Lane, Upper Bucklebury, Reading, RG7 6QH",RH0,13500 +NN01201950062114,"Scout Hut At Cemetery Road, Bucklebury Common, Reading, RG7 6RY",MAN,890 +NN01201950062114,"Scout Hut At Cemetery Road, Bucklebury Common, Reading, RG7 6RY",RH0,890 +NN01208400025115,"The Victory Room, The Village, Bucklebury, Reading, Berkshire,",MAN,8700 +NN01301800061112,"Iverne House, Burghfield Bridge, Burghfield, Reading, RG30 3RD",RH0,6100 +NN01301800061112,"Iverne House, Burghfield Bridge, Burghfield, Reading, RG30 3RD",SBRE,6100 +NN01303000005112,"Nr Burghfield Bridge, Green Lane, Burghfield, Reading, RG30 3XN",RH0,6400 +NN01303000005112,"Nr Burghfield Bridge, Green Lane, Burghfield, Reading, RG30 3XN",SBRE,6400 +NN01304200009126,"Store Hill Farm, Manns Hill, Burghfield Common, Reading, RG7 3BD",RH0,7700 +NN01304200009126,"Store Hill Farm, Manns Hill, Burghfield Common, Reading, RG7 3BD",SBRE,7700 +NN01305100068119,"Scout Hut, Reading Road, Burghfield Common, Reading, RG7 3BG",MAN,8700 +NN01305100068119,"Scout Hut, Reading Road, Burghfield Common, Reading, RG7 3BG",RH0,8700 +NN01305200021119,"Burghfield Village Hall, Recreation Road, Burghfield Common, Reading, RG7 3EN",RH0,9000 +NN01305200021119,"Burghfield Village Hall, Recreation Road, Burghfield Common, Reading, RG7 3EN",SBRE,9000 +NN0150720005011A,"West Berkshire Golf Course, Chaddleworth, Newbury, Berkshire, RG20 7DU",RH0,45000 +NN0150720005011A,"West Berkshire Golf Course, Chaddleworth, Newbury, Berkshire, RG20 7DU",SBR3,45000 +NN01508500060116,"Chaddleworth Village Hall, Chaddleworth, Newbury, RG20 7DY",MAN,3150 +NN01508500060116,"Chaddleworth Village Hall, Chaddleworth, Newbury, RG20 7DY",RH0,3150 +NN01601200030117,"Curridge Womens Institute Hall, Curridge, Thatcham, RG18 9DZ",MAN,2750 +NN0160300003911A,"Chieveley Recreation Centre, High Street, Chieveley, Newbury, Berkshire, RG20 8TE",MAN,19000 +NN0160300003911A,"Chieveley Recreation Centre, High Street, Chieveley, Newbury, Berkshire, RG20 8TE",RH0,19000 +NN01605100180112,"Oxford Road, Oxford Road, Chieveley, Newbury, RG20 8RU",SBR3,32750 +NN01605400200118,"At, Priors Court Road, Hermitage, Thatcham, Berkshire, RG18 9QZ",MAN,73500 +NN01605400200118,"At, Priors Court Road, Hermitage, Thatcham, Berkshire, RG18 9QZ",RH0,73500 +NN01706000031118,"St Finians Catholic Primary School, The Ridge, Cold Ash, Thatcham, Berkshire, RG18 9HU",MAN,38750 +NN01900600019015,"Compton Village Hall, Burrell Road, Compton, Newbury, RG20 6NP",MAN,8600 +NN01900600019015,"Compton Village Hall, Burrell Road, Compton, Newbury, RG20 6NP",RH0,8600 +NN01901300001126,"The Scout And Guide Headquarters, Warnham Lane, Compton, Newbury, RG20 7PL",MAN,5400 +NN01901300001126,"The Scout And Guide Headquarters, Warnham Lane, Compton, Newbury, RG20 7PL",RH0,5400 +NN0200030001211B,"12, Back Street, East Garston, Hungerford, Berkshire, RG17 7EX",MAN,7100 +NN0200030001211B,"12, Back Street, East Garston, Hungerford, Berkshire, RG17 7EX",RH0,7100 +NN02000300025002,"Adj College Farm, Back Street, East Garston, Hungerford, Berkshire, RG17 7EX",RH0,27250 +NN02000300025002,"Adj College Farm, Back Street, East Garston, Hungerford, Berkshire, RG17 7EX",SBR3,27250 +NN0220300001011A,"Enborne C E School, Crockham Heath, Enborne, Newbury, Berkshire, RG20 0JU",MAN,26750 +NN0230500004411B,"Englefield C E School, The Street, Englefield, Reading, RG7 5ER",MAN,28500 +NN02501000034114,"Club Room, North Fawley, Fawley, Wantage, OX12 9NJ",RH0,3400 +NN02501000034114,"Club Room, North Fawley, Fawley, Wantage, OX12 9NJ",SBRE,3400 diff --git a/tests/data/sample/reliefs-key/Reliefs Key.xlsx b/tests/data/sample/reliefs-key/Reliefs Key.xlsx new file mode 100644 index 0000000..bf4933b Binary files /dev/null and b/tests/data/sample/reliefs-key/Reliefs Key.xlsx differ diff --git a/tests/data/sample/south-hams/south-hams-source.XLSX b/tests/data/sample/south-hams/south-hams-source.XLSX new file mode 100644 index 0000000..c4d13f9 Binary files /dev/null and b/tests/data/sample/south-hams/south-hams-source.XLSX differ diff --git a/tests/data/sample/south-hams/south-hams.TRANSFORM b/tests/data/sample/south-hams/south-hams.TRANSFORM new file mode 100644 index 0000000..db37173 --- /dev/null +++ b/tests/data/sample/south-hams/south-hams.TRANSFORM @@ -0,0 +1 @@ +{"uuid":"eb624412-fded-44b8-b493-eec74c0658f8","name":"mute-comment","title":"South Hams","dataSource":{"uuid":"82b8db2d-dc44-4a71-afb1-023da329a69c","title":"South Hams","path":"/app/working/references/329cde2e-a5c2-43be-b5cd-086fd8fe6296.XLSX","mime":"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet","columns":[{"uuid":"c83dc1a4-280a-42cc-86b6-502c013eb605","name":"Authority"},{"uuid":"21ea0458-5f90-4e9e-98e1-6d30f452d523","name":"Account Name"},{"uuid":"375e8938-1fc7-4de9-8e82-1a2bc980c03b","name":"Full Property Address"},{"uuid":"effa8f66-46af-4ea9-955b-4432a23ee0b2","name":"Party Type"},{"uuid":"851c05a9-13d5-4bc1-b2aa-518c68474c73","name":"Company Type"},{"uuid":"3606bff0-9345-4303-8411-4c29a3257d4d","name":"Billing Authority Reference"},{"uuid":"b48e2867-9511-4579-a0ed-e52166487e67","name":"Current Rateable Value"},{"uuid":"497424d5-9364-49e4-a134-70c0a3a92d91","name":"Previous List Rateable Value"},{"uuid":"7ff814e0-515a-4b7a-8844-589786d398c3","name":"Current Charge"},{"uuid":"94251695-908f-4d57-b15b-f0d42d8a7e51","name":"Description"},{"uuid":"299cfeee-1288-4844-8b3d-25ce879b4055","name":"Relief Type 1"},{"uuid":"5401f191-40c1-4670-bac9-e146810ed99c","name":"Date Applied 1"},{"uuid":"52af3a18-280b-4565-9a4e-7b8228e9f3ba","name":"Value 1"},{"uuid":"51d4a28b-9168-4f93-a8c8-3c6414c053e9","name":"Relief Type 2"},{"uuid":"1a62ebce-295a-488e-8178-6b516faf8c09","name":"Date Applied 2"},{"uuid":"c4031817-6213-43ea-bf17-1c59fd534854","name":"Value 2"},{"uuid":"0975a9fb-2885-48fb-ad4a-0824dd4f8ea5","name":"Relief Type 3"},{"uuid":"8afb2865-dc9a-4eb6-b01a-f29e92e40689","name":"Date Applied 3"},{"uuid":"3447fb93-0dd3-4ccb-846a-8c3fdcdbd85b","name":"Value 3"},{"uuid":"c8cf9774-ffee-4cc3-b2b6-e701a51961b7","name":"Occupied / Empty Status"},{"uuid":"9e82eed6-1e29-44e7-b9c0-163cb438fdb2","name":"Transactional Total"},{"uuid":"edb6253a-84a7-459b-8a26-225689b37c8a","name":"Transitional Relief"},{"uuid":"d97d3639-a3b8-40b3-b731-dd5c4a239d10","name":"Exemption Type"},{"uuid":"0209865e-d367-4f3a-bea7-296d6328423b","name":"Date Applied"},{"uuid":"4ecc79c0-3fd6-4aa4-b7db-9d7af1b92e63","name":"Value"},{"uuid":"f6c8e838-c399-452b-a05c-55bc0d06302b","name":"Account Start date"}],"preserve":["Authority","Account Name","Full Property Address","Party Type","Company Type","Billing Authority Reference","Current Rateable Value","Previous List Rateable Value","Current Charge","Description","Relief Type 1","Date Applied 1","Value 1","Relief Type 2","Date Applied 2","Value 2","Relief Type 3","Date Applied 3","Value 3","Occupied / Empty Status","Transactional Total","Transitional Relief","Exemption Type","Date Applied","Value","Account Start date"],"attributes":{},"checksum":"c048b0cab2e9252f32107f77283f52563df55f0f411f2ff3652b26a3314930745205a02425b1f3808330adf367eafae0c28be9c2cb972d11ed12fe1709471f03","index":3459},"crosswalk":{"uuid":"b14f938f-26d7-4d2c-ac04-9af785e3df14","name":"e07000044","title":"South Hams","schemaSource":{"uuid":"5c7c194d-9da1-4b06-8aa5-5b7a44fa5bb1","name":"e07000044","title":"South Hams","fields":[{"uuid":"92f89713-ff63-424c-9e3f-819e22263d8b","name":"Authority","title":"Authority","type":"string"},{"uuid":"69ba9f9e-0518-43b9-9fef-711d1821a790","name":"Account Name","title":"Account Name","type":"string"},{"uuid":"b7723053-d5b3-4bf2-9320-dc88d9b802d2","name":"Full Property Address","title":"Full Property Address","type":"string"},{"uuid":"673fa825-bf81-48a7-8963-cc53e647242c","name":"Party Type","title":"Party Type","type":"string"},{"uuid":"3c9afd33-ab42-4784-b3bc-e40c8904a7bc","name":"Company Type","title":"Company Type","type":"string"},{"uuid":"62eece4d-8047-4d53-a94a-f8a74a830c1c","name":"Billing Authority Reference","title":"Billing Authority Reference","type":"string"},{"uuid":"0ccac28e-63cf-4fb9-8023-2bf0d444e4ec","name":"Current Rateable Value","title":"Current Rateable Value","type":"string"},{"uuid":"71f3376b-f790-4411-976a-fd98f68d5ca3","name":"Previous List Rateable Value","title":"Previous List Rateable Value","type":"string"},{"uuid":"0ef8125f-5c15-4f2d-93a8-8ceb8039c729","name":"Current Charge","title":"Current Charge","type":"string"},{"uuid":"449b1029-3e11-416d-9af6-1abf0cc300a9","name":"Description","title":"Description","type":"string"},{"uuid":"27f4238e-a80b-49f9-8e34-811aeb113a56","name":"Relief Type 1","title":"Relief Type 1","type":"string","constraints":{"enum":[{"uuid":"26720785-3642-459e-a546-8d5a8023531e","name":"Retail Discount 2024"},{"uuid":"89d80987-ff5c-4267-b540-e75089620b49","name":"Empty Property Rate: Industrial"},{"uuid":"ed43e243-b9ef-48a0-8a7f-0019658340a7","name":" "},{"uuid":"d88c946d-1c25-4371-934e-09500b73f0ff","name":"Small Business Relief England"},{"uuid":"7cd61ebb-d4f9-4518-bb76-3fda51471845","name":"Empty Property Rate: Non Industrial"},{"uuid":"e59b4be7-ab06-4304-b5e3-e94cb84d9b57","name":"Discretionary Relief (Up To 20%)"},{"uuid":"ab971fa6-aa5f-42a1-9f77-3da10fa14f23","name":"Mandatory Charitable Relief 80%"},{"uuid":"71e13e8b-a705-4186-aac5-de062c25c781","name":"Retail Discount 2023"},{"uuid":"ee80837b-0266-4cff-b844-cb6a59b9f134","name":"Supporting Small Business Relief 2023"},{"uuid":"e29e200c-7a0d-41ef-88be-2b30a84a9c50","name":"Empty Property Rate : Industrial"},{"uuid":"f3817ec0-c85e-460b-aa37-292ad59ba351","name":"Empty Property Rate : Non-Industrial"},{"uuid":"08a94065-7e84-4bc6-a9b8-9f6bcbc4c0f8","name":"Discretionary Rate Relief (Up To 20%) "},{"uuid":"bf81cd21-578d-4ed7-9196-67c389c4d79e","name":"Empty Property Rate : Charitable"},{"uuid":"d128eaaf-0142-4028-b835-cf2aa51cd09f","name":"Public Convenience Rate Relief"},{"uuid":"033177e4-795e-4f20-a55b-9b884f5f0a5f","name":"Supporting Small Business Relief 23 1y"},{"uuid":"00daa8aa-7fc5-4939-ae05-b2802847831a","name":"Small Business Relief 1 Year Extension"},{"uuid":"73e87a60-adbf-4c23-9572-feddbc35a024","name":"Mandatory Rural Rate Relief"},{"uuid":"ed78bcde-8477-4704-9e48-d2a05b622dda","name":"Rural Rate Relief - 100%"},{"uuid":"ea64400a-f854-471b-b919-6b015d4267a7","name":"Retail Relief 50% To 31.03.2023"},{"uuid":"6c1fbbe0-6c2c-49ed-9c3b-18734c741a7e","name":"Mandatory relief for CASCS"},{"uuid":"91255a22-5d0d-4b8f-8a6e-21a256670a76","name":"Discretionary Relief For Cascs"},{"uuid":"b63b2129-91eb-4660-89a2-d0b6d800cd1c","name":"Newspaper Relief Up To £1,500"}]}},{"uuid":"1cdacbfb-426c-4e26-8a48-24ecbd73f890","name":"Date Applied 1","title":"Date Applied 1","type":"date"},{"uuid":"d83e5347-c822-4292-8e37-3a4aece41c10","name":"Value 1","title":"Value 1","type":"string"},{"uuid":"69f00a51-bdce-438a-8495-425ef9874741","name":"Relief Type 2","title":"Relief Type 2","type":"string","constraints":{"enum":[{"uuid":"f6bef465-592b-4be1-aa32-a5e09b635c03","name":" "},{"uuid":"cbd0f68e-3711-4880-b560-3a041ff3e0ef","name":"Mandatory Charitable Relief 80%"},{"uuid":"2f21e47b-8b9b-4a86-80b7-0dd760108f21","name":"Small Business Relief England"},{"uuid":"0f2c1d49-aef6-4a72-afa8-3e0da2c2f819","name":"Retail Discount 2023"},{"uuid":"8346a9bf-badb-438d-b5f1-48f6ce989dac","name":"Discretionary Rate Relief (Up To 20%) "},{"uuid":"cdaf036d-d6f2-439e-9076-6c516db318f1","name":"Supporting Small Business Relief 2023"},{"uuid":"7b874eed-47c8-46f2-b9eb-fe7b84a7b8ae","name":"Discretionary Rate Relief"},{"uuid":"b92d2b3c-c795-4a77-b9cf-8fc3c3efc6fa","name":"Retail Discount 2024"},{"uuid":"4c00c66a-ec6e-42f3-a2df-2c7feb44dcd1","name":"Mandatory Rural Rate Relief"},{"uuid":"40bac094-0bd1-44f0-b7ee-c68e3d7e889f","name":"Mandatory relief for CASCS"},{"uuid":"1e5fdd9f-51d0-4829-8684-aa1576f03e52","name":"Discretionary Rural Relief (50% Top Up)"}]}},{"uuid":"ef33c6c9-ed20-4eaa-a35e-645d55c3e007","name":"Date Applied 2","title":"Date Applied 2","type":"date"},{"uuid":"d4f0031f-63bf-4c26-8525-6c8005b40515","name":"Value 2","title":"Value 2","type":"string"},{"uuid":"41e56bfa-f2f3-4c3a-a3d8-ebd1169375fe","name":"Relief Type 3","title":"Relief Type 3","type":"string","constraints":{"enum":[{"uuid":"e77e3a51-66ac-458a-b969-c1a0f728a9b6","name":" "},{"uuid":"7542efb0-0fa6-4c6e-a516-cac73f012c4d","name":"Small Business Relief England"},{"uuid":"ca6c8c28-4394-42a0-83c1-3110dba15285","name":"Mandatory Charitable Relief 80%"},{"uuid":"64374b12-7c47-44b5-a341-399607df75d4","name":"Discretionary Rural Relief (50% Top Up)"},{"uuid":"50e4d7c3-3fb1-4382-8a01-837c97d14ba6","name":"Supporting Small Business Relief 2023"}]}},{"uuid":"28006102-1027-4d3f-907e-49e5e4cdd66e","name":"Date Applied 3","title":"Date Applied 3","type":"date"},{"uuid":"cd4c2c84-da71-411f-9658-698825553f84","name":"Value 3","title":"Value 3","type":"string"},{"uuid":"52448a9b-b75e-495b-a3d0-36dabb867268","name":"Occupied / Empty Status","title":"Occupied / Empty Status","type":"string","constraints":{"enum":[{"uuid":"449e1c9e-fb09-4312-9e5a-f87dfa57fc83","name":"N"},{"uuid":"8ad1ed53-098f-4224-92f0-8f3cfec157a7","name":"Y"}]}},{"uuid":"64788a1c-f8c7-43a3-98db-a1bef361df3b","name":"Transactional Total","title":"Transactional Total","type":"string"},{"uuid":"12e4d0ed-bd19-45a6-b3fa-a8b01ea83d9f","name":"Transitional Relief","title":"Transitional Relief","type":"string","constraints":{"enum":[{"uuid":"216059eb-6fdd-49c1-b0cb-0b28b49f6963","name":true},{"uuid":"fe89c802-e077-4ee0-9956-1d2af4b7c3e0","name":false}]}},{"uuid":"952d9be7-0ca1-47a5-be6c-7bb88578f102","name":"Exemption Type","title":"Exemption Type","type":"string","constraints":{"enum":[{"uuid":"e20aa8a6-afab-41d1-9c33-70d119d31e00","name":"0"},{"uuid":"e78a7581-e5f6-49dc-8491-eb2c91de721e","name":"Rv Less Than £2,900"},{"uuid":"5a27f977-db14-4061-8a60-f9a1b1f437f3","name":"-10.96"},{"uuid":"1b52d60a-917d-4191-8acb-2e3ceb89b5ab","name":"-15.34"},{"uuid":"9d5063ec-6e75-4c0d-882f-21f08d399b9f","name":"-2424.14"},{"uuid":"df18599c-0eca-4c9d-87aa-c52068923f41","name":"-366.76"},{"uuid":"9fb4a99b-df4a-48b3-b3f1-1bc5f74b35aa","name":"Decommissioned Mast"},{"uuid":"e936975b-7515-4973-954d-478c90ed4f7c","name":"-2575.46"},{"uuid":"6a3263b9-5900-4cf8-a9f3-a9e4e2ea55fa","name":"Listed Property"},{"uuid":"19843568-35d1-4ada-8bda-b2c5f48cb097","name":"Exempt Empty Rate Land"},{"uuid":"1f3e164c-271d-4e90-a61f-c425340b2f61","name":"-109.6"},{"uuid":"cb8d43ec-19a2-4476-90aa-0c317049ab29","name":"Prohibited By Law"},{"uuid":"6c720c8a-3c78-42fb-997a-25aca515e973","name":"Owner Subject To Winding Up"},{"uuid":"95f8c249-04e4-4405-8f88-846dfbd5b35c","name":" "},{"uuid":"550d4913-7135-4e62-89c4-2ea43945076a","name":"-713.2"},{"uuid":"d5373ac0-88b9-49a0-84f6-742ef901b7d1","name":"-19.46"},{"uuid":"6becb995-afaf-4fcf-ae5b-10d0b1974a07","name":"-10.85"},{"uuid":"3bb68b59-6cbb-4c1b-8302-155af6f21c78","name":"-271.71"},{"uuid":"0ab6cfe1-647a-4073-a24e-0d1b8e63c0e8","name":"-348.43"},{"uuid":"b74ddfa0-2223-46e4-b12f-590ad2710263","name":"-85.7"},{"uuid":"56381561-cc6a-4b8b-b3a5-5aeb1b082f96","name":"-14.53"},{"uuid":"0f997745-e91e-4aaa-805c-b747f4c17d9d","name":"-325.85"},{"uuid":"4a3119b2-7333-42a3-bb75-28f444dfcc1e","name":"-251.5"},{"uuid":"99242e25-2806-46b6-a88a-900cabea84cf","name":"-454.71"},{"uuid":"6490e23c-533b-4a9b-94db-742042831936","name":"-336.58"},{"uuid":"3b37ae67-9656-4dfc-9d5c-298cc3a66392","name":"-143.84"},{"uuid":"33374664-d913-43ff-8f65-f22ce5ba80b3","name":"Company In Administration"},{"uuid":"2dbe2925-5a22-4376-9807-e14eab6b8d23","name":"-2603.53"},{"uuid":"e9945bf3-44dd-4edc-9a6b-d554a10d4b3e","name":"-895.7"},{"uuid":"b2e82049-0105-4ba2-b942-8c5c478daeca","name":"-376.5"},{"uuid":"d67d79b9-f7b3-4de6-9977-28b20af1fab8","name":"-90.57"},{"uuid":"76f7f06c-13cd-40f8-a1ef-a5f306d1f85b","name":"-740.64"},{"uuid":"9c6063c9-74b9-447d-bfb3-8ddd31877055","name":"-87.7"},{"uuid":"e17e8974-4d38-4b4f-92cf-bf74c5a53400","name":"-60.88"},{"uuid":"1688e107-4554-4236-aab3-a3b938f38f9b","name":"-3827.33"},{"uuid":"51c99931-1c92-4b6d-9005-8ee36aabad21","name":"-85.58"},{"uuid":"b3d92f96-210f-4a20-b8e3-7607572e7de1","name":"-131.89"},{"uuid":"55087abe-eaf5-400d-b66d-e9d5a49d1ae8","name":"-342.81"},{"uuid":"0b5e4531-2f28-4244-80ff-9ee7217deb0b","name":"-268.96"},{"uuid":"358e376a-6e9d-4a13-a3da-e38b02736296","name":"-749.02"},{"uuid":"9f2ed50f-63d9-4db4-8c8d-f09cd24048fb","name":"-810.13"},{"uuid":"836c7cfe-ae71-441d-97d1-9930d254f4f7","name":"-116.52"},{"uuid":"6f33da83-d058-4948-8dc4-1cd8ab1bd78b","name":"-388.9"},{"uuid":"1b7a1f34-9982-4ca6-8a9e-10c00f41a540","name":"-46.91"},{"uuid":"3bdde678-5bdc-4380-8a19-841dbaf555ec","name":"-466.94"},{"uuid":"210f6d4f-de1d-42fc-b934-4577f6dc7e8b","name":"-42.08"},{"uuid":"af40e8d3-8559-4dfc-89f6-40e78b2e8b87","name":"-41.79"},{"uuid":"ee22961c-f2e4-4dfc-a2d5-a7ba5e06312c","name":"-153.18"},{"uuid":"16b5af7c-85ce-4280-94f0-ab9d144096d7","name":"-1224.8"},{"uuid":"46dbbaca-08ba-4a16-9629-77e5c54bee5c","name":"-799.65"},{"uuid":"dfb64acf-464d-4016-88ef-853932c077fd","name":"695.16"},{"uuid":"c1d3b899-b1e4-4c7f-b3fd-6f1af8b51f91","name":"-711.82"},{"uuid":"99f1320b-27d6-443b-838a-103837cb495d","name":"-3854.77"},{"uuid":"9ba46432-543f-490b-a437-50d6e5dc3555","name":"-6004.72"},{"uuid":"eb7978f0-464a-4b81-ba22-841d0758affd","name":"-493.36"},{"uuid":"c6ee1f6e-d7c2-4e69-b3c7-9032d2afcde0","name":"-1489.14"},{"uuid":"e25c5873-4c91-44c1-8e25-8c7549640574","name":"-762.22"},{"uuid":"6c915a28-a8e3-431c-aafc-afa0359b809d","name":"Action taken by Crown"},{"uuid":"5a440b9e-b341-4712-b06c-2d3aed2a12e2","name":"-1169.53"},{"uuid":"41e608e8-2171-4156-a7ed-f7f9197cc1b9","name":"Empty Land (No Buildings)"},{"uuid":"7e36550f-c790-433b-9606-f4544c38345c","name":"-23.26"},{"uuid":"9ae39f4c-6158-4a78-b1eb-e3b59ab87fb3","name":"-896.66"},{"uuid":"5bc28684-a34f-41bd-94f1-d02e7d1c55b5","name":"-326.72"},{"uuid":"baf5e86b-e6bb-4292-a104-5d6f6f38e427","name":"-552.27"},{"uuid":"66dfe960-37b4-4196-b6fb-a7bb6ff64173","name":"-178.39"},{"uuid":"f054bcc5-8a0a-4919-ad05-7a2d765b9da7","name":"-114.95"},{"uuid":"fa6d7817-ab89-4e5a-ba61-4ced43665e97","name":"-409.68"},{"uuid":"be9db7f6-cb42-493a-9a90-821b6073592c","name":"Owner subject to winding up"},{"uuid":"4cad7ec6-fbf2-4e00-b025-e7eef79879ff","name":"-19.89"},{"uuid":"5a0f8004-d536-472e-bf38-7c96fcedf58a","name":"-1394.21"},{"uuid":"bd9e996a-ba82-4723-b852-6cad55b950e2","name":"-22.19"},{"uuid":"c73dc6dc-55fb-4b6c-b036-31280588e380","name":"-54.13"},{"uuid":"e18c51c6-f9f2-4b4b-bb07-26f9aded1e39","name":"-358.39"},{"uuid":"5ca5bf8e-299b-49f8-ae1c-f9131a674371","name":"-56.42"},{"uuid":"fcb4da5e-2697-440c-8b6f-a0c6ef056589","name":"-70.55"},{"uuid":"b52503ff-1a73-4933-b6cd-7c64cc48c0c4","name":"-1208.48"},{"uuid":"edce664c-e276-4d14-8eaa-0026e682f172","name":"-298.48"},{"uuid":"04d87e57-8a2a-469f-842d-b2ea34b6dd84","name":"-893.54"},{"uuid":"8b077eb6-6c2c-43c3-a227-126227d0ffa7","name":"-50.34"},{"uuid":"16146e9a-7f2f-4bde-9ce6-c6da22a69142","name":"-61.63"},{"uuid":"2d665782-c0ab-4b84-ab41-acab83775645","name":"-56.44"},{"uuid":"ea27eff6-5c70-4250-8f66-87d259754c91","name":"-912.73"},{"uuid":"78360bc0-9a09-4aa4-83da-d86f1069b208","name":"-949.87"},{"uuid":"f16623f2-80e2-48ab-89de-e6861c880179","name":"-107.42"},{"uuid":"6bfd9a10-39a5-42c4-ad60-83ae26499985","name":"-475.26"},{"uuid":"2879a853-edf5-40f7-8de9-3bcbc91a374c","name":"-126.63"},{"uuid":"8e186579-1d2c-45fd-9669-83cfbd3830a9","name":"-103.04"},{"uuid":"e556273e-751d-4950-bfef-3bc4e4eb5c46","name":"-14.92"},{"uuid":"b48dfaf8-4702-4225-bf71-48e72e62d7e3","name":"-2095.8"},{"uuid":"c298ae9b-cad4-49d2-96b3-7c1136edcb73","name":"-2066.17"},{"uuid":"c0c08a63-9f9c-4a64-ba5a-93d850fe55fb","name":"-44.75"},{"uuid":"f800c9b6-baf5-43ff-996e-309eae9c9c85","name":"-788.53"},{"uuid":"da0796c6-f9d2-4ce0-9a2a-4b31cfed40f7","name":"-95.01"},{"uuid":"96fe41d6-bd30-4180-a08e-56b1676389a8","name":"-1036.98"},{"uuid":"37690aa3-60af-4590-bcd7-7b2b256c6ccd","name":"-1166.41"},{"uuid":"bc5935f1-30ce-4e4f-b751-5d3107ee893f","name":"-224.55"},{"uuid":"646a9de1-2ab0-47a6-bdf0-8994d0e756b0","name":"Decomissioned Mast"},{"uuid":"7a7378a0-07b1-4987-85ef-9c70ed0ceda8","name":"-664.67"},{"uuid":"824845a4-7272-49d5-b14e-1a03466e9f6e","name":"-1298.02"},{"uuid":"55f0e081-f1a9-45a6-a9af-f644f5e5930c","name":"-557.13"},{"uuid":"5a1d16f5-2842-4332-97ff-9fa33e0b8dcf","name":"-3083.2"},{"uuid":"a1012e7b-bf32-4945-baff-5e2ed361e822","name":"-1097.8"},{"uuid":"d38026af-57ac-4fc3-89d3-df7a928b2273","name":"-12.91"},{"uuid":"8c1fa1a1-58ef-4d4f-998e-9fc75db0ee5c","name":"-455.96"},{"uuid":"349ee038-bec9-4b16-8051-a712051a54e8","name":"-7.61"},{"uuid":"f8c068ac-6fbb-44ab-8d04-390f8b712d16","name":"-827.09"},{"uuid":"ce8b21fd-4c73-4410-9a0c-1dbef5ea43ed","name":"-424.08"},{"uuid":"e5814af6-8b8a-41a7-b987-f2181fa1bcc2","name":"-4459.81"},{"uuid":"fe6b94d0-6733-4c2f-8923-a91509f5ea4a","name":"-166.54"},{"uuid":"728909eb-d124-457f-87cc-3d6bc90a3ddb","name":"-681.63"},{"uuid":"9d369947-d8cb-4863-94a5-c8dfd4134584","name":"-3079.77"},{"uuid":"db583bdf-1acd-4ccb-8d33-946522bd191d","name":"-763.72"},{"uuid":"2fa86bf2-6bf4-4bf3-9dfd-036698cc4810","name":"-3087.56"},{"uuid":"bbf9d9d1-3f97-41fd-85ce-e6c4d71392a3","name":"-42.66"},{"uuid":"fee94ca5-6550-46d6-a343-4bc45b2af887","name":"-855.91"},{"uuid":"03130979-c5be-427a-8ca2-47ee1d6ddb42","name":"-767.96"},{"uuid":"90bb4b7c-630c-4042-b674-8997c40585d7","name":"-1539.91"},{"uuid":"4a598e90-38f6-4fdb-93a6-1b3083333e86","name":"-89.57"},{"uuid":"47819b49-1679-4520-9e24-5c0191d54a0c","name":"-1879.05"},{"uuid":"b45b2d22-aa7c-4d9f-baaa-4678b615fb46","name":"-337.2"},{"uuid":"518a2199-51b4-41f4-a046-2392ffee313d","name":"-1089.82"},{"uuid":"683898d7-0b98-4d7b-995a-5f6750f91e35","name":"-1483.9"},{"uuid":"2efa237f-6a5f-48cf-b449-d89cccb0deda","name":"-48.9"},{"uuid":"76c60e5e-0bde-4857-b42b-7d87d6a44085","name":"-37.17"},{"uuid":"f71319a9-a0b6-4b6e-8fc1-cf8a09dd6e1a","name":"-536.8"},{"uuid":"18ad39c5-b67e-4ac8-9ae1-05e7390ade5c","name":"-65.12"},{"uuid":"392142c1-02bd-4fd8-b0a5-ddf6f085a967","name":"-91.07"},{"uuid":"54ca421f-3602-4d8c-aa72-8d46bf89a944","name":"-96.81"},{"uuid":"922720a8-7cda-45b1-929d-034af0e07616","name":"-820.61"},{"uuid":"b737a967-f3f3-47e5-9c9f-60bda241b1a0","name":"-53.52"},{"uuid":"251397e2-e5ce-4b2c-a103-7747b8aa9b77","name":"-67.74"},{"uuid":"4b632f31-e622-4801-a2f9-6ee7c848cb82","name":"-450.35"},{"uuid":"2cb85bec-eb4d-48df-b362-07c22476e36b","name":"-906.68"},{"uuid":"cae99e0a-811e-46a8-a0ea-6f0dc9738404","name":"-233.28"},{"uuid":"55ac1bc5-7c5b-4331-a8ce-a01aa0680b4e","name":"-704.21"},{"uuid":"5889f08b-b22d-43b4-8b89-1e3d2d5458b0","name":"-58.88"},{"uuid":"873e425a-cdf2-46e0-ae2b-0584788b8413","name":"-562.12"},{"uuid":"a47ac715-a1d9-42fa-b935-fe8c888ba58f","name":"-474.37"},{"uuid":"36b81b00-f665-4ea7-aa94-66700d27077f","name":"-52.39"},{"uuid":"e41e62e9-dbfb-4e8b-818c-312d325a3cc8","name":"-201.47"},{"uuid":"e1908ef5-00c3-4b63-b4eb-9fdd11e9bceb","name":"-456.71"},{"uuid":"4f846d90-3805-43cd-aebf-c07f0d958416","name":"-422.65"},{"uuid":"da61c21e-37af-40ca-a686-44a5ce6649ce","name":"-1108.78"},{"uuid":"fe7ff5a7-8214-4171-ba48-3cc45c1b8791","name":"-2259.22"},{"uuid":"47a1e171-fe06-475e-b24c-a819000e7374","name":"-289.42"},{"uuid":"cd94376b-b88f-4106-a276-f519c367a1a0","name":"-34.93"},{"uuid":"dd9d79ec-abbd-4c5e-95fa-e68b4fe872fe","name":"-1289.29"},{"uuid":"c8b69987-abcd-4e64-b824-c6296c9368b5","name":"-2106.03"},{"uuid":"590f1f73-1189-4e05-b89e-66b7383ee9ec","name":"-3040.78"},{"uuid":"bc44ac83-9904-46d5-add1-fe95dbaab0be","name":"-3500.8"},{"uuid":"7ee82f20-9ac9-4942-b605-030204a6d064","name":"-638.72"},{"uuid":"b03cc96b-6c43-4786-a56f-a115e982f1f1","name":"-635.23"},{"uuid":"613b7d8e-ebdc-4269-9c47-876fe085a55d","name":"-14.72"},{"uuid":"cc541702-d43f-4184-9251-fc977d1e4635","name":"-40.36"},{"uuid":"9d2a7f94-854b-4b03-b007-d13a8a9fb6ec","name":"-1115.26"},{"uuid":"6900d2f7-d197-408f-a211-4dc6610969f3","name":"-324.6"},{"uuid":"f6d93360-6fe9-40d3-951d-cafa4dfd1129","name":"-2409.23"},{"uuid":"99a5ce88-cb84-43b4-b9c5-732b31327e26","name":"-76.22"},{"uuid":"d5c7fbe8-7096-4fdc-b986-951400f3d5a5","name":"-382.05"},{"uuid":"0718a095-340a-4dbc-b946-70882aea9820","name":"-943.11"},{"uuid":"d26d1fbc-630b-45ea-839a-2ae06f7f3a25","name":"-22.31"},{"uuid":"460ab2f5-31cb-4545-8e2f-f3f96b265ab7","name":"-849.05"},{"uuid":"6868c417-df5a-40b1-842b-87900940e970","name":"-309.14"},{"uuid":"af911bcf-eb97-4581-a7c3-806058ad5d37","name":"-130.12"},{"uuid":"c2812ff1-e22f-4ff9-9c23-db86d73dc585","name":"-654.69"},{"uuid":"f75a780f-efed-4e21-b1c3-00cec97a4720","name":"-312.01"},{"uuid":"898eff95-f233-4f8f-bd10-c8e9bbd1bd5e","name":"-18.56"},{"uuid":"97cec3d3-6bde-479c-aa29-162a5cb1d2b7","name":"-666.52"},{"uuid":"bec9323f-4137-4e98-9c66-548bc6f1eca9","name":"-632.53"},{"uuid":"5d79c664-3cad-4008-820c-c7c06f64eac5","name":"-192.11"},{"uuid":"a605737e-9055-47aa-818a-cecb612e25a8","name":"-15.66"}]}},{"uuid":"9173ea69-8c38-4270-b474-50e5de1fd642","name":"Date Applied","title":"Date Applied","type":"date"},{"uuid":"0f1ed5c1-0a30-45c8-88b9-9516ab973f10","name":"Value","title":"Value","type":"string"},{"uuid":"bd93b6f7-22a2-4aea-a476-192441fd89b5","name":"Account Start date","title":"Account Start date","type":"string"}],"index":3459,"version":[{"name":"redgine04@gmail.com","description":"Initial save of SchemaModel.","updated":"2025-01-27T10:59:08.538495"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2025-02-08T23:54:22.008319"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2025-02-08T23:54:23.150435"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2025-02-08T23:54:27.011867"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2025-02-08T23:54:28.182746"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2025-02-08T23:54:52.780778"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2025-02-08T23:54:53.913054"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2025-02-08T23:54:56.240337"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2025-02-08T23:55:03.984965"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2025-02-08T23:55:06.352648"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2025-02-08T23:55:08.214800"}]},"schemaDestination":{"uuid":"507a0ccf-5735-4b3a-9b9e-2e7842921e4a","name":"england-and-wales-nondomestic-rates-schema","title":"England and Wales non-domestic rates schema","description":"Based on the The Open Data Collaboration group at Open Innovations (Open Innovations, Bradford Council, Calderdale Council, and Leeds Council) which started a project to visualise business rates data. Although each of the three councils publish open data about their business rates, they each have different field titles and don't always include the same fields. This project aimed to standardise the way the data was shared to make it more usable and to get value (within a council and externally) from the datasets.\nhttps://open-innovations.org/projects/business-rates/format.html","fields":[{"uuid":"e1b1d480-8481-4281-98bd-0e59f7fdb949","name":"localAuthorityCode","title":"Local authority code","description":"Unique reference identifier for each local authority. Added from the Task 'name' unique identifier field.","type":"string"},{"uuid":"f42d609b-c8a6-4017-aa9d-d55eb62bd63b","name":"localBillingReference","title":"Billing reference","description":"Local Authority hereditament billing reference used by both VOA & LA to correspond with each other in reference to a specific hereditament.","type":"string","example":"8577090904","constraints":{"required":true,"enum":[]}},{"uuid":"7774382b-7054-4100-8bf1-ff9741a47d5a","name":"occupierAccountHolderName","title":"Occupier account-holder name","description":"Account-holder's names may be split across multiple columns. They are to be joined into one for post-processing.","type":"string","example":"A.M.C (North East) Ltd"},{"uuid":"02347dbe-d491-47cb-bfd4-2a3c28c4c861","name":"occupierRecordDate","title":"Date each row entered the record","type":"date"},{"uuid":"090b1d3d-3be0-4d39-b24b-8c97969eeb8f","name":"occupierAccountStartDate","title":"Start-date for account-holder","description":"Date the Account Holder became responsible for the hereditament. Provide date only if provided in data release from LA. e.g. \"01/01/23\", \"BLANK\" or \"Not Supplied\".","type":"date"},{"uuid":"cb30a0df-ea14-4c67-b76a-1ab358f167bd","name":"occupierPropertyAddress","title":"Address of the rateable property","description":"Address may be split across multiple columns. They are to be joined into one for post-processing with the postcode at the end.","type":"string"},{"uuid":"3ed85652-0831-4cd3-a005-a353b6cc51a1","name":"occupierCorrespondenceAddress","title":"Correspondence address","description":"Account-holders correspondence address may be split across multiple columns. They are to be joined into one for post-processing with the postcode at the end. This may be different from the hereditament address.","type":"string"},{"uuid":"6305545d-7caf-40fb-b421-2e483c3fbd96","name":"occupierOccupationState","title":"Occupation state","description":"Occupational status of hereditament. Only include if provided in LA data release.\nOccupied = \"Occupied\", Void = \"Void\", No Information = \"BLANK\", no empty data provided by LA = \"Not Supplied\"","type":"string","constraints":{"enum":[{"uuid":"83cd608f-ae55-4090-836d-154c2ec256d7","name":"Occupied","description":"Occupied"},{"uuid":"de15f74d-90c3-4f38-8821-37e3178787bd","name":"Vacant","description":"Vacant"}]}},{"uuid":"17c261f9-e837-49d1-abe1-fc63f2a80f86","name":"occupierOccupationDate","title":"Occupation state date","description":"To be used in combination with 'occupierOccupationState'. If only a date is provided (e.g. for only date of void, or date of occupation), then categorise the column as a boolean and use this to specify the state and the date simultaneously.","type":"date"},{"uuid":"e325315f-5ec0-416d-a470-4facfb7bb759","name":"occupierRatesPayable","title":"Actual rates paid","description":"Rates paid, net of reliefs (if available) in pounds. Provide rates paid amount only if provided in data release from LA. e.g. \"£55,000\", \"BLANK\", \"not supplied\".","type":"number"},{"uuid":"1dba84d1-501a-427f-88ad-771e2393516a","name":"occupierReliefType","title":"Relief or Exemption Received","description":"Type of relief or exemption on rates paid received.\n\nNOTE: Type, Date and Amount are arrays and the position in the array must correspond between these three columns. This is for both 'wide' data (where there are multiple columns in the source data for each class of relief) and 'narrow' data (where there is only one set of columns). Narrow data must still be placed in an array (e.g. ['value'], with only one value).","type":"array","constraints":{"enum":[{"uuid":"60fd0916-8f55-4236-8a26-e09cbc9b3c25","name":"charity"},{"uuid":"4f6de275-e6af-4da5-adee-6e5ec60d0488","name":"covid_carf"},{"uuid":"dde9af58-e13c-4ca0-be93-4d46fadb760f","name":"discretionary"},{"uuid":"1b66abb2-3de7-4cfa-8a21-94be7abf8d0a","name":"empty"},{"uuid":"9ec6a8bd-2ee1-4061-bd1d-d454b57c167d","name":"enteprise_zone"},{"uuid":"4320d78d-4d6f-4c31-ba94-0ad2f20bc185","name":"exempt"},{"uuid":"e1895f58-8bc1-4c4c-9f4b-f9f4a507ce5d","name":"hardship"},{"uuid":"3887112a-d266-4e8d-9aa6-1771f2273c9c","name":"insolvency"},{"uuid":"82fe4da5-1e7c-4406-a076-3d8557d64b1b","name":"listed"},{"uuid":"53baf5cb-aff8-47e6-b9ab-f517365f5f7f","name":"mandatory"},{"uuid":"a8a4c3d6-ca77-4776-a754-5848a8415bf3","name":"retail","description":"Has differing definitions at different times"},{"uuid":"f50ade3e-2ea3-419e-86fe-dbb7f88d8e08","name":"rural"},{"uuid":"6ad19826-c4a0-4d55-9a47-d79012e356bd","name":"s44a_partially_empty"},{"uuid":"f87f1753-c7e7-4ab2-b57e-bd5611be4f7c","name":"small_business","description":"Includes SBRR, SBBR"},{"uuid":"2adbfa0c-0414-4827-911a-5e0203e43b34","name":"transitional"},{"uuid":"93fec68d-bcfe-42ff-a90e-0c6f663bbb33","name":"other"}]}},{"uuid":"fc7c378c-70db-4364-b8a2-5e050a5eceeb","name":"occupierReliefDate","title":"Relief or Exemption date awarded","description":"Date a relief or exemption is awarded","type":"array"},{"uuid":"af286083-171d-49da-b84b-814ee681e7b4","name":"occupierReliefAmount","title":"Relief or Exemption amount awarded","type":"array"},{"uuid":"80edf660-19c8-4a94-8da3-70b5c412dc4f","name":"occupierSurchargeType","title":"Surcharge or additional rates charge type","description":"NOTE: Type, Date and Amount are arrays and the position in the array must correspond between these three columns. This is for both 'wide' data (where there are multiple columns in the source data for each class of relief) and 'narrow' data (where there is only one set of columns). Narrow data must still be placed in an array (e.g. ['value'], with only one value).","type":"array","constraints":{"enum":[{"uuid":"317f3a54-58e6-4ad4-bc65-31881c2c5294","name":"bid_levy","description":"Business Improvement District, BID"},{"uuid":"0cc6d603-fc6f-4251-a825-ce7f44bf4cc0","name":"crossrail"},{"uuid":"74d1accc-31b1-4bb9-8c65-cff6d12f2383","name":"transitional_surcharge"},{"uuid":"0cb87dae-87f7-4a25-aa7d-0fc6c6c99ff5","name":"city_of_london","description":"If they publish"},{"uuid":"02a1859b-2b2a-4064-84a1-d808d2853926","name":"other"}]}},{"uuid":"92871604-fd9e-4f2f-b6f0-7d694bf6605e","name":"occupierSurchargeDate","title":"Surcharge award date","description":"Date surcharge and additional charges applied","type":"array"},{"uuid":"e3da11ad-d17b-4dda-9fd7-c2a829519ba9","name":"occupierSurchargeAmount","title":"Surcharge Amount Charged","type":"array"}],"version":[{"name":"gchait@whythawk.com","description":"Initial save of SchemaModel.","updated":"2024-01-18T09:45:02.887755"},{"name":"Gavin Chait ","description":"Update save of SchemaModel.","updated":"2024-01-18T14:50:55.745764"}]},"actions":[{"uuid":"aa130315-1d3d-410d-a063-2051c5c0b5ce","script":"NEW > 'localAuthorityCode' < ['E07000044']"},{"uuid":"4f84b61b-4ae5-4ff5-80ef-9c2b6a186e13","script":"RENAME > 'localBillingReference' < ['Billing Authority Reference']"},{"uuid":"e3bc571f-5d97-4b1b-bddb-e6273316c84a","script":"RENAME > 'occupierAccountHolderName' < ['Account Name']"},{"uuid":"60721c86-c151-4551-884e-ac90cb62f840","script":"RENAME > 'occupierPropertyAddress' < ['Full Property Address']"},{"uuid":"16bb7c36-418b-4357-98a5-31857c9c7164","script":"RENAME > 'occupierAccountStartDate' < ['Account Start date']"},{"uuid":"114cfe03-e94b-4729-8707-da570f24d4a8","script":"RENAME > 'occupierRatesPayable' < ['Current Charge']"},{"uuid":"83207a19-7ddf-4964-8c26-1093e92568ab","script":"CATEGORISE > 'occupierOccupationState'::'Vacant' < 'Occupied / Empty Status'::['Y']"},{"uuid":"11e819bc-d885-41af-bf0a-610b5631c4c0","script":"CATEGORISE > 'occupierOccupationState'::'Occupied' < 'Occupied / Empty Status'::['N']"},{"uuid":"835495d5-4940-4085-a6c6-91651776941e","script":"CATEGORISE > 'occupierReliefType'::'retail' < 'Relief Type 1'::['Retail Discount 2023','Retail Discount 2024','Retail Relief 50% To 31.03.2023']"},{"uuid":"5b9d43bc-36b3-4d37-998e-79b272d6dd49","script":"CATEGORISE > 'occupierReliefType'::'empty' < 'Relief Type 1'::['Empty Property Rate : Charitable','Empty Property Rate : Industrial','Empty Property Rate : Non-Industrial','Empty Property Rate: Industrial','Empty Property Rate: Non Industrial']"},{"uuid":"558ca5db-f922-4c61-9bb9-56f637f4a40a","script":"CATEGORISE > 'occupierReliefType'::'small_business' < 'Relief Type 1'::['Small Business Relief 1 Year Extension','Small Business Relief England','Supporting Small Business Relief 2023','Supporting Small Business Relief 23 1y']"},{"uuid":"88609ba8-3091-4ec1-b921-0028cc3a1a53","script":"CATEGORISE > 'occupierReliefType'::'discretionary' < 'Relief Type 1'::['Discretionary Rate Relief (Up To 20%) ','Discretionary Relief (Up To 20%)','Discretionary Relief For Cascs','Newspaper Relief Up To £1,500','Public Convenience Rate Relief','Rural Rate Relief - 100%']"},{"uuid":"82e1ea47-cdae-4f2e-a33c-d2dbc1063637","script":"CATEGORISE > 'occupierReliefType'::'mandatory' < 'Relief Type 1'::['Mandatory Charitable Relief 80%','Mandatory Rural Rate Relief','Mandatory relief for CASCS','Rural Rate Relief - 100%']"},{"uuid":"b51e6432-2912-46cf-93fb-93f9a521f4b1","script":"CATEGORISE > 'occupierReliefType'::'rural' < 'Relief Type 1'::['Rural Rate Relief - 100%']"},{"uuid":"a1ca12b7-bc5f-4496-a246-dc9f9cb8e16e","script":"CATEGORISE > 'occupierReliefType'::'mandatory' < 'Relief Type 2'::['Mandatory Charitable Relief 80%','Mandatory Rural Rate Relief','Mandatory relief for CASCS']"},{"uuid":"2141caf8-c899-40fd-85f4-9580a0491402","script":"CATEGORISE > 'occupierReliefType'::'small_business' < 'Relief Type 2'::['Small Business Relief England','Supporting Small Business Relief 2023']"},{"uuid":"41bdf2fe-6430-459f-b5ca-bca76cdc4bf8","script":"CATEGORISE > 'occupierReliefType'::'retail' < 'Relief Type 2'::['Retail Discount 2023','Retail Discount 2024']"},{"uuid":"65ea3fc8-539d-4c44-9b8f-e957903500e2","script":"CATEGORISE > 'occupierReliefType'::'discretionary' < 'Relief Type 2'::['Discretionary Rate Relief','Discretionary Rate Relief (Up To 20%) ','Discretionary Rural Relief (50% Top Up)']"},{"uuid":"7cd7ebc1-23eb-4a5d-8ca3-ca0432e5f95f","script":"CATEGORISE > 'occupierReliefType'::'small_business' < 'Relief Type 3'::['Small Business Relief England','Supporting Small Business Relief 2023']"},{"uuid":"5b4e99a7-e405-4043-b20f-6fc235d4a377","script":"CATEGORISE > 'occupierReliefType'::'mandatory' < 'Relief Type 3'::['Mandatory Charitable Relief 80%']"},{"uuid":"1faeedc0-cf7e-4eb8-987b-0e17a1d98c2c","script":"CATEGORISE > 'occupierReliefType'::'discretionary' < 'Relief Type 3'::['Discretionary Rural Relief (50% Top Up)']"},{"uuid":"efd9b02f-3a8b-4ef0-89ec-922ac857cd0a","script":"CATEGORISE > 'occupierReliefType'::'empty' < 'Occupied / Empty Status'::['Y']"},{"uuid":"9a6daa37-d777-4305-855a-0948c738b24f","script":"CATEGORISE > 'occupierReliefType'::'transitional' < 'Transitional Relief'::[True]"},{"uuid":"ca4fd4fe-a370-4430-8e62-1969f0708ab0","script":"CATEGORISE > 'occupierReliefType'::'discretionary' < 'Transitional Relief'::[True]"},{"uuid":"3b88346c-fe48-4f17-aa71-5a57844c2194","script":"CATEGORISE > 'occupierReliefType'::'exempt' < 'Exemption Type'::['Action taken by Crown','Company In Administration','Decomissioned Mast','Decommissioned Mast','Empty Land (No Buildings)','Exempt Empty Rate Land','Listed Property','Owner Subject To Winding Up','Owner subject to winding up','Prohibited By Law','Rv Less Than £2,900']"},{"uuid":"3e7d0b0a-b093-4092-a54d-cc952930f3ad","script":"CATEGORISE > 'occupierReliefType'::'listed' < 'Exemption Type'::['Listed Property']"},{"uuid":"170f6f62-a3fd-443a-90a1-1dc63326056c","script":"CATEGORISE > 'occupierReliefType'::'empty' < 'Exemption Type'::['Action taken by Crown','Company In Administration','Decomissioned Mast','Decommissioned Mast','Empty Land (No Buildings)','Exempt Empty Rate Land','Listed Property','Owner Subject To Winding Up','Owner subject to winding up','Prohibited By Law','Rv Less Than £2,900']"},{"uuid":"1da4928a-393a-45dd-8196-6f16993d8968","script":"COLLATE > 'occupierReliefDate' < ['Date Applied 1','Date Applied 1','Date Applied 1','Date Applied 1','Date Applied 1','Date Applied 1','Date Applied 2','Date Applied 2','Date Applied 2','Date Applied 2','Date Applied 3','Date Applied 3','Date Applied 3',~,~,~,'Date Applied','Date Applied','Date Applied']"},{"uuid":"0874b9c7-3031-45be-a2f2-5f22e532c189","script":"COLLATE > 'occupierReliefAmount' < ['Value 1','Value 1','Value 1','Value 1','Value 1','Value 1','Value 2','Value 2','Value 2','Value 2','Value 3','Value 3','Value 3',~,'Transitional Relief','Transitional Relief','Value','Value','Value']"}],"version":[{"name":"redgine04@gmail.com","description":"Initial save of CrosswalkModel.","updated":"2025-07-20T13:53:58.213017"}]},"version":[{"name":"redgine04@gmail.com","description":"Initial save of TransformModel.","updated":"2025-07-20T13:54:20.614736"}]} \ No newline at end of file diff --git a/tests/data/sample/stockport/2026-09-04-stockport-original.XLSX b/tests/data/sample/stockport/2026-09-04-stockport-original.XLSX new file mode 100644 index 0000000..f9f4eec Binary files /dev/null and b/tests/data/sample/stockport/2026-09-04-stockport-original.XLSX differ diff --git a/tests/data/sample/stockport/stockport.TRANSFORM b/tests/data/sample/stockport/stockport.TRANSFORM new file mode 100644 index 0000000..48e373c --- /dev/null +++ b/tests/data/sample/stockport/stockport.TRANSFORM @@ -0,0 +1 @@ +{"uuid":"1fcd7b4c-7e8a-4d25-84ef-bf65d211da9a","name":"future-sage","title":"Stockport","dataSource":{"uuid":"4751126e-a3ef-4f53-81f3-894a20370044","title":"Stockport","path":"/app/working/references/186b7c8c-bf89-4a9c-9381-c1f470174314.XLSX","mime":"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet","columns":[{"uuid":"9e2a0b52-887c-4399-b4d7-df2f128dd401","name":"PlaceRef"},{"uuid":"cefe7a2a-782c-4d6e-91fd-1f8dc31654b2","name":"PropDesc"},{"uuid":"24ee2482-ac15-4a5c-8be5-cdbbb9d3ac9d","name":"RateableVal"},{"uuid":"765162bf-aae7-4c5b-bd41-83482fb96455","name":"AccountName"},{"uuid":"1ad9b5c9-50a8-4696-90ba-e7d101b8ee9c","name":"LiabStart"},{"uuid":"e7873687-878a-4153-bad7-0d324ed7bcdf","name":"PropAddress1"},{"uuid":"61007ad6-d611-48bd-81e1-39e05c26dcb8","name":"PropAddress2"},{"uuid":"67985b2b-f240-43b5-a049-3a86d96d4727","name":"PropAddress3"},{"uuid":"09cd0048-2a1c-4a2c-becc-7daddb186db0","name":"PropAddress4"},{"uuid":"a06593d2-6bb6-4592-9122-1a0f92eb503b","name":"PropAddress5"},{"uuid":"10831e21-dc5b-4b44-a5a5-7252159f47a4","name":"PropPostCode"},{"uuid":"7746faea-d04d-4cd7-9c14-df1ba590700c","name":"Address1"},{"uuid":"96b30a9c-f470-4383-ae86-e156bbc797a7","name":"Address2"},{"uuid":"3f1c9534-0a2d-44ee-a044-dfbf8d51c7c0","name":"Address3"},{"uuid":"ffa1af19-cf06-4ff6-ba14-511d99a08c0d","name":"Address4"},{"uuid":"058b4d00-b681-40cf-b205-d277116d424c","name":"Address5"},{"uuid":"fee8dc25-3227-47c9-b2e1-6d8a20eda72c","name":"PostCode"},{"uuid":"39f71e96-9f45-47f7-9633-e5775042bf9c","name":"MandRlfCd"},{"uuid":"7a4020d8-b9c4-419a-a03d-f798bf1dcdd2","name":"DiscRlfCd"},{"uuid":"f9651468-f90f-4b08-981d-6c1bc5d91ec8","name":"AddRlfCd"},{"uuid":"879cfed2-de08-4387-b5ef-5358213a8ebb","name":"MandRlf"},{"uuid":"af91eb54-a123-4e02-a48e-4159882ecee9","name":"DiscRlf"},{"uuid":"3f677d6a-f94f-4e56-98bb-bf515d6b98eb","name":"AdditionalRlf"},{"uuid":"84730962-0171-4ce6-8f78-a87f6647c162","name":"SBRRlf"},{"uuid":"b6b3b805-d650-4e79-a54b-906e89581c42","name":"TRDisc/SSBCd"},{"uuid":"e5594e6c-5c57-4d61-9f22-31cb684c4d70","name":"TRDisc/SSBAmt"}],"preserve":["PlaceRef","PropDesc","RateableVal","AccountName","LiabStart","PropAddress1","PropAddress2","PropAddress3","PropAddress4","PropAddress5","PropPostCode","Address1","Address2","Address3","Address4","Address5","PostCode","MandRlfCd","DiscRlfCd","AddRlfCd","MandRlf","DiscRlf","AdditionalRlf","SBRRlf","TRDisc/SSBCd","TRDisc/SSBAmt"],"attributes":{},"checksum":"28ebfcdf938277052b506d44f6eea64e286177623c4a8482c471210f1ad90b0957fe9505f8c73ab812d4bf218a3ecb86c9ce2893a3e47fa88a90c9edbe6e293d","index":6614},"crosswalk":{"uuid":"b0a644d7-4e86-47f2-b921-8c3c6bc04514","name":"e08000007","title":"Stockport","schemaSource":{"uuid":"7f9980e0-354d-46e1-8ca6-6de10a7ece73","name":"e08000007","title":"Stockport","fields":[{"uuid":"b4c44b5b-631a-4874-89b1-b90c4d1512d5","name":"PlaceRef","title":"PlaceRef","type":"string"},{"uuid":"842d6369-b1ba-41b0-81bb-f69277fa47db","name":"PropDesc","title":"PropDesc","type":"string"},{"uuid":"9bfbce5a-f412-40de-b47a-7e6bd9f1628d","name":"RateableVal","title":"RateableVal","type":"string"},{"uuid":"7a4b3475-936e-433c-bbc7-a273e7a3f08e","name":"AccountName","title":"AccountName","type":"string"},{"uuid":"2834b7af-87ae-4d8f-bb07-63f91747d5f7","name":"LiabStart","title":"LiabStart","type":"string"},{"uuid":"8a580357-f1a2-4f28-ada9-563213f18761","name":"PropAddress1","title":"PropAddress1","type":"string"},{"uuid":"50eb429e-a4a3-41f5-a59d-4e885d7b8ea8","name":"PropAddress2","title":"PropAddress2","type":"string"},{"uuid":"bed7db02-ffb8-41b4-b1cd-acacd2cdfc5d","name":"PropAddress3","title":"PropAddress3","type":"string"},{"uuid":"f3175099-0f8d-4f06-b11e-b63db6b0e227","name":"PropAddress4","title":"PropAddress4","type":"string"},{"uuid":"b0def58d-b26d-47d3-98cc-9c46b6763269","name":"PropAddress5","title":"PropAddress5","type":"string"},{"uuid":"72d5ba30-8088-4c18-b73f-8880b185b61e","name":"PropPostCode","title":"PropPostCode","type":"string"},{"uuid":"4edcfc03-420b-4218-be72-4d21210bcc44","name":"Address1","title":"Address1","type":"string"},{"uuid":"1ff810e7-cc27-4381-8f43-7cacf2714f47","name":"Address2","title":"Address2","type":"string"},{"uuid":"d5401a0c-6b56-4401-9f3c-947e5c334b39","name":"Address3","title":"Address3","type":"string"},{"uuid":"2744f3cd-531b-4913-b245-f0d3eade0dad","name":"Address4","title":"Address4","type":"string"},{"uuid":"475c9b22-87a1-4300-bc1d-17da34c2dc14","name":"Address5","title":"Address5","type":"string"},{"uuid":"7d21b8f1-24f4-4db2-992a-374d0461f1f4","name":"PostCode","title":"PostCode","type":"string"},{"uuid":"713ce3f1-4121-430a-b626-1807f8c35477","name":"MandRlfCd","title":"MandRlfCd","type":"string","constraints":{"enum":[{"uuid":"650c283c-72a6-405a-a2d1-9966fc44f47a","name":"MAN"},{"uuid":"d4f4dceb-a045-4a06-b85c-a63daf843d5e","name":"CASC80"},{"uuid":"3ec6c1dc-9473-4d82-b679-f194c0c3bf6d","name":"PCON"}]}},{"uuid":"d9be0e26-622d-49e8-a0f7-8a67079154b4","name":"DiscRlfCd","title":"DiscRlfCd","type":"string","constraints":{"enum":[{"uuid":"b54a7561-1c64-4494-8adf-a20f4068e753","name":"CASC20"},{"uuid":"fc210a49-132a-4ba1-a220-ef7737aede8b","name":"DR"}]}},{"uuid":"672b72ba-4270-44de-9957-61e37328e028","name":"AddRlfCd","title":"AddRlfCd","type":"string","constraints":{"enum":[]}},{"uuid":"7a565a5b-fd0b-4392-9929-3c67bd7207f9","name":"MandRlf","title":"MandRlf","type":"string"},{"uuid":"ec9fda16-f869-42d4-adbc-fa6b517e59b6","name":"DiscRlf","title":"DiscRlf","type":"string"},{"uuid":"c5c34ab1-268f-4feb-b7df-3da2b81af1d6","name":"AdditionalRlf","title":"AdditionalRlf","type":"string"},{"uuid":"636ba15f-68dc-4700-a26e-73b70761e4a3","name":"SBRRlf","title":"SBRRlf","type":"number","constraints":{"enum":[{"uuid":"bfb67125-6a8a-4565-9ce4-1614818a8345","name":true},{"uuid":"ca6fc384-51d0-43f2-afe0-b189ff4e8188","name":false}]}},{"uuid":"fd9c4783-cfaa-4507-a1c9-b5655c15d223","name":"TRDisc/SSBCd","title":"TRDisc/SSBCd","type":"string","constraints":{"enum":[{"uuid":"1c16991b-dbe0-4929-8e08-86c71432c379","name":"SSB26"}]}},{"uuid":"f772d424-b510-4c31-a3eb-797075f70e29","name":"TRDisc/SSBAmt","title":"TRDisc/SSBAmt","type":"string"}],"index":6467,"version":[{"name":"redgine04@gmail.com","description":"Initial save of SchemaModel.","updated":"2024-03-14T04:23:50.775764"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2024-03-22T10:23:31.840599"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2024-03-22T10:23:33.619587"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2024-03-22T10:23:35.919971"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2024-03-22T10:25:14.828594"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2024-03-22T10:32:34.003875"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2025-07-30T00:24:09.445938"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2025-11-17T11:11:22.487345"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2025-11-17T11:11:25.118929"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2025-11-17T11:11:27.365864"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2025-11-17T11:11:31.529121"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2025-11-17T11:11:40.056831"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2025-11-17T11:12:09.857588"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2026-05-17T11:14:32.448363"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2026-05-17T11:14:33.767180"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2026-05-17T11:14:35.040792"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2026-05-17T11:14:37.150385"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2026-05-17T11:15:00.192836"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2026-09-04T05:02:04.602917"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2026-09-04T05:02:07.344390"},{"name":"redgine04@gmail.com","description":"Update save of SchemaModel.","updated":"2026-09-04T05:02:08.938917"}]},"schemaDestination":{"uuid":"507a0ccf-5735-4b3a-9b9e-2e7842921e4a","name":"england-and-wales-nondomestic-rates-schema","title":"England and Wales non-domestic rates schema","description":"Based on the The Open Data Collaboration group at Open Innovations (Open Innovations, Bradford Council, Calderdale Council, and Leeds Council) which started a project to visualise business rates data. Although each of the three councils publish open data about their business rates, they each have different field titles and don't always include the same fields. This project aimed to standardise the way the data was shared to make it more usable and to get value (within a council and externally) from the datasets.\nhttps://open-innovations.org/projects/business-rates/format.html","fields":[{"uuid":"e1b1d480-8481-4281-98bd-0e59f7fdb949","name":"localAuthorityCode","title":"Local authority code","description":"Unique reference identifier for each local authority. Added from the Task 'name' unique identifier field.","type":"string","constraints":{"default":{"uuid":"82e02d12-eae0-4ace-98ab-218ba3f3f4c3","name":"E08000007"}}},{"uuid":"f42d609b-c8a6-4017-aa9d-d55eb62bd63b","name":"localBillingReference","title":"Billing reference","description":"Local Authority hereditament billing reference used by both VOA & LA to correspond with each other in reference to a specific hereditament.","type":"string","example":"8577090904","constraints":{"required":true,"enum":[]}},{"uuid":"7774382b-7054-4100-8bf1-ff9741a47d5a","name":"occupierAccountHolderName","title":"Occupier account-holder name","description":"Account-holder's names may be split across multiple columns. They are to be joined into one for post-processing.","type":"string","example":"A.M.C (North East) Ltd"},{"uuid":"02347dbe-d491-47cb-bfd4-2a3c28c4c861","name":"occupierRecordDate","title":"Date each row entered the record","type":"date"},{"uuid":"090b1d3d-3be0-4d39-b24b-8c97969eeb8f","name":"occupierAccountStartDate","title":"Start-date for account-holder","description":"Date the Account Holder became responsible for the hereditament. Provide date only if provided in data release from LA. e.g. \"01/01/23\", \"BLANK\" or \"Not Supplied\".","type":"date"},{"uuid":"cb30a0df-ea14-4c67-b76a-1ab358f167bd","name":"occupierPropertyAddress","title":"Address of the rateable property","description":"Address may be split across multiple columns. They are to be joined into one for post-processing with the postcode at the end.","type":"string"},{"uuid":"3ed85652-0831-4cd3-a005-a353b6cc51a1","name":"occupierCorrespondenceAddress","title":"Correspondence address","description":"Account-holders correspondence address may be split across multiple columns. They are to be joined into one for post-processing with the postcode at the end. This may be different from the hereditament address.","type":"string"},{"uuid":"6305545d-7caf-40fb-b421-2e483c3fbd96","name":"occupierOccupationState","title":"Occupation state","description":"Occupational status of hereditament. Only include if provided in LA data release.\nOccupied = \"Occupied\", Void = \"Void\", No Information = \"BLANK\", no empty data provided by LA = \"Not Supplied\"","type":"string","constraints":{"enum":[{"uuid":"83cd608f-ae55-4090-836d-154c2ec256d7","name":"Occupied","description":"Occupied"},{"uuid":"de15f74d-90c3-4f38-8821-37e3178787bd","name":"Vacant","description":"Vacant"}]}},{"uuid":"17c261f9-e837-49d1-abe1-fc63f2a80f86","name":"occupierOccupationDate","title":"Occupation state date","description":"To be used in combination with 'occupierOccupationState'. If only a date is provided (e.g. for only date of void, or date of occupation), then categorise the column as a boolean and use this to specify the state and the date simultaneously.","type":"date"},{"uuid":"e325315f-5ec0-416d-a470-4facfb7bb759","name":"occupierRatesPayable","title":"Actual rates paid","description":"Rates paid, net of reliefs (if available) in pounds. Provide rates paid amount only if provided in data release from LA. e.g. \"£55,000\", \"BLANK\", \"not supplied\".","type":"number"},{"uuid":"1dba84d1-501a-427f-88ad-771e2393516a","name":"occupierReliefType","title":"Relief or Exemption Received","description":"Type of relief or exemption on rates paid received.\n\nNOTE: Type, Date and Amount are arrays and the position in the array must correspond between these three columns. This is for both 'wide' data (where there are multiple columns in the source data for each class of relief) and 'narrow' data (where there is only one set of columns). Narrow data must still be placed in an array (e.g. ['value'], with only one value).","type":"array","constraints":{"enum":[{"uuid":"60fd0916-8f55-4236-8a26-e09cbc9b3c25","name":"charity"},{"uuid":"4f6de275-e6af-4da5-adee-6e5ec60d0488","name":"covid_carf"},{"uuid":"dde9af58-e13c-4ca0-be93-4d46fadb760f","name":"discretionary"},{"uuid":"1b66abb2-3de7-4cfa-8a21-94be7abf8d0a","name":"empty"},{"uuid":"9ec6a8bd-2ee1-4061-bd1d-d454b57c167d","name":"enteprise_zone"},{"uuid":"4320d78d-4d6f-4c31-ba94-0ad2f20bc185","name":"exempt"},{"uuid":"e1895f58-8bc1-4c4c-9f4b-f9f4a507ce5d","name":"hardship"},{"uuid":"3887112a-d266-4e8d-9aa6-1771f2273c9c","name":"insolvency"},{"uuid":"82fe4da5-1e7c-4406-a076-3d8557d64b1b","name":"listed"},{"uuid":"53baf5cb-aff8-47e6-b9ab-f517365f5f7f","name":"mandatory"},{"uuid":"a8a4c3d6-ca77-4776-a754-5848a8415bf3","name":"retail","description":"Has differing definitions at different times"},{"uuid":"f50ade3e-2ea3-419e-86fe-dbb7f88d8e08","name":"rural"},{"uuid":"6ad19826-c4a0-4d55-9a47-d79012e356bd","name":"s44a_partially_empty"},{"uuid":"f87f1753-c7e7-4ab2-b57e-bd5611be4f7c","name":"small_business","description":"Includes SBRR, SBBR"},{"uuid":"2adbfa0c-0414-4827-911a-5e0203e43b34","name":"transitional"},{"uuid":"93fec68d-bcfe-42ff-a90e-0c6f663bbb33","name":"other"},{"uuid":"407bf8b6-dbf9-4707-80df-4713f06b46c1","name":"ssb_one_year","description":"Supporting Small Business 1 Year"},{"uuid":"57e578f6-5f02-457b-b83a-cab6f6bfce3d","name":"ssb_three_year","description":"Supporting Small Business 3 Year"},{"uuid":"d0214f96-836f-4aad-b8da-a9f860beb03e","name":"ssb_unspecified","description":"Supporting Small Business"},{"uuid":"aac2de06-bcc9-44bc-86d0-d753fefa1b76","name":"improvement","description":"Improvement"},{"uuid":"ccefe7d1-9c1e-4cd1-b262-648dc7ef7d45","name":"heat_network","description":"Heat Network"},{"uuid":"c641779a-d8c8-4fda-970e-966e931198ae","name":"public_lavatories","description":"Public Lavatories"},{"uuid":"d7b8ab22-286f-4781-8be9-d461199fab7a","name":"freeport","description":"Freeport"},{"uuid":"47c4b03e-9d6a-41e3-a7ab-5f40e17121aa","name":"film_studio","description":"Film Studio"},{"uuid":"26268f7b-8ecb-4f20-b8d6-5d0db06c932b","name":"pubs_live_music","description":"Pubs and Live Music Venues"},{"uuid":"c8094a9f-33e9-4b3f-bc9b-2087296798b2","name":"ev_charging","description":"EV Charging Point"},{"uuid":"729c0681-1746-4539-a96c-2792bdfdde04","name":"local_newspaper","description":"Local Newspaper"}]}},{"uuid":"af286083-171d-49da-b84b-814ee681e7b4","name":"occupierReliefAmount","title":"Relief or Exemption amount awarded","type":"array"},{"uuid":"fc7c378c-70db-4364-b8a2-5e050a5eceeb","name":"occupierReliefDate","title":"Relief or Exemption date awarded","description":"Date a relief or exemption is awarded","type":"array"},{"uuid":"80edf660-19c8-4a94-8da3-70b5c412dc4f","name":"occupierSurchargeType","title":"Surcharge or additional rates charge type","description":"NOTE: Type, Date and Amount are arrays and the position in the array must correspond between these three columns. This is for both 'wide' data (where there are multiple columns in the source data for each class of relief) and 'narrow' data (where there is only one set of columns). Narrow data must still be placed in an array (e.g. ['value'], with only one value).","type":"array","constraints":{"enum":[{"uuid":"317f3a54-58e6-4ad4-bc65-31881c2c5294","name":"bid_levy","description":"Business Improvement District, BID"},{"uuid":"0cc6d603-fc6f-4251-a825-ce7f44bf4cc0","name":"crossrail"},{"uuid":"74d1accc-31b1-4bb9-8c65-cff6d12f2383","name":"transitional_surcharge"},{"uuid":"0cb87dae-87f7-4a25-aa7d-0fc6c6c99ff5","name":"city_of_london","description":"If they publish"},{"uuid":"02a1859b-2b2a-4064-84a1-d808d2853926","name":"other"}]}},{"uuid":"e3da11ad-d17b-4dda-9fd7-c2a829519ba9","name":"occupierSurchargeAmount","title":"Surcharge Amount Charged","type":"array"},{"uuid":"92871604-fd9e-4f2f-b6f0-7d694bf6605e","name":"occupierSurchargeDate","title":"Surcharge award date","description":"Date surcharge and additional charges applied","type":"array"}],"version":[{"name":"gchait@whythawk.com","description":"Initial save of SchemaModel.","updated":"2024-01-18T09:45:02.887755"},{"name":"Gavin Chait ","description":"Update save of SchemaModel.","updated":"2024-01-18T14:50:55.745764"},{"name":"Gavin Chait ","description":"Update save of SchemaModel.","updated":"2026-08-05T13:35:05.494165"},{"name":"Gavin Chait ","description":"Update save of SchemaModel.","updated":"2026-08-05T13:37:45.592489"},{"name":"Gavin Chait ","description":"Update save of SchemaModel.","updated":"2026-08-11T19:40:05.912166"}]},"actions":[{"uuid":"30ae1510-2632-4e11-a6d2-5020671946a4","script":"NEW > 'localAuthorityCode' < ['E08000007']"},{"uuid":"809d6a7b-6ef3-4f61-86b1-b754f3f42a85","script":"RENAME > 'localBillingReference' < ['PlaceRef']"},{"uuid":"31060269-0d23-44dd-8e36-dcdf777cdb95","script":"RENAME > 'occupierAccountHolderName' < ['AccountName']"},{"uuid":"da2c08d1-6e8e-4aa3-8217-d28f97d55b08","script":"UNITE > 'occupierPropertyAddress' < ', '::['PropAddress1','PropAddress2','PropAddress3','PropAddress4','PropAddress5','PropPostCode']"},{"uuid":"e8e69e10-ca63-46a8-bf94-c46b8dfd0790","script":"UNITE > 'occupierCorrespondenceAddress' < ', '::['Address1','Address2','Address3','Address4','Address5','PostCode']"},{"uuid":"ddfc3dac-f450-44f1-ad1e-b5f7e49ac6aa","script":"RENAME > 'occupierAccountStartDate' < ['LiabStart']"},{"uuid":"957174ac-b08d-4f6c-b98a-2328de16cb84","script":"CATEGORISE > 'occupierReliefType'::'mandatory' < 'MandRlfCd'::['CASC80','MAN','PCON']"},{"uuid":"0d8e6380-0122-4373-b65e-a066f1b25027","script":"CATEGORISE > 'occupierReliefType'::'discretionary' < 'DiscRlfCd'::['CASC20','DR']"},{"uuid":"94c291b2-945b-4f27-974e-c41440ae52fb","script":"CATEGORISE > 'occupierReliefType'::'small_business' < 'SBRRlf'::[True]"},{"uuid":"82b078d1-16ce-428d-9c74-a45f8fd9e74b","script":"CATEGORISE > 'occupierReliefType'::'ssb_unspecified' < 'TRDisc/SSBCd'::['SSB26']"},{"uuid":"6a9bfed2-8f13-4cdb-8008-2343c5de7231","script":"COLLATE > 'occupierReliefAmount' < ['MandRlf','DiscRlf','SBRRlf','TRDisc/SSBAmt']"}]}} \ No newline at end of file diff --git a/tests/data/synthetic-excel/fixture-excel-mac.csv b/tests/data/synthetic-excel/fixture-excel-mac.csv index 18c9fef..3245bb5 100644 --- a/tests/data/synthetic-excel/fixture-excel-mac.csv +++ b/tests/data/synthetic-excel/fixture-excel-mac.csv @@ -1,6 +1,6 @@ -ref_text,ref_padded,amount,amount_mojibake,number_as_text,number,blank_like,padded_text,accented,date_text,date_mixed,date_locale,boolean_like -0000130842,0000130842,£43.40,£43.40,12.50,12.50,Ê, lead,CafŽ,13/04/2024,04/01/2024,04/01/2024,TRUE -0000000001,0000000001,£-826.80,£-826.80,0.10,0.10, ,trail ,na•ve,04/01/2024,13/04/2024,25/12/2024,false -1234567890,1234567890,"£1,234.50","£1,234.50",1e3,1000.00,, both ,®tna,2010-04-01 00:00:00,05/02/2024,01/04/2024,Y -0012,0000000012,£.00,£.00, 7,7.00,aÊb,none,MŸller £,2024-04-01,25/12/2024,,FALSE -#N/A,,,,,25,,,,,,, +ref_text,ref_padded,amount,amount_mojibake,number_as_text,number,blank_like,padded_text,accented,date_text,date_mixed,date_locale,boolean_like +0000130842,0000130842,£43.40,£43.40,12.50,12.50,Ê, lead,CafŽ,13/04/2024,04/01/2024,04/01/2024,TRUE +0000000001,0000000001,£-826.80,£-826.80,0.10,0.10, ,trail ,na•ve,04/01/2024,13/04/2024,25/12/2024,false +1234567890,1234567890,"£1,234.50","£1,234.50",1e3,1000.00,, both ,®tna,2010-04-01 00:00:00,05/02/2024,01/04/2024,Y +0012,0000000012,£.00,£.00, 7,7.00,aÊb,none,MŸller £,2024-04-01,25/12/2024,,FALSE +#N/A,,,,,25,,,,,,, diff --git a/tests/data/synthetic-excel/fixture-excel-utf8.csv b/tests/data/synthetic-excel/fixture-excel-utf8.csv index a2f2467..0840ad2 100644 --- a/tests/data/synthetic-excel/fixture-excel-utf8.csv +++ b/tests/data/synthetic-excel/fixture-excel-utf8.csv @@ -1,6 +1,6 @@ -ref_text,ref_padded,amount,amount_mojibake,number_as_text,number,blank_like,padded_text,accented,date_text,date_mixed,date_locale,boolean_like -0000130842,0000130842,£43.40,¬£43.40,12.50,12.50, , lead,Café,13/04/2024,04/01/2024,04/01/2024,TRUE -0000000001,0000000001,£-826.80,¬£-826.80,0.10,0.10, ,trail ,naïve,04/01/2024,13/04/2024,25/12/2024,false -1234567890,1234567890,"£1,234.50","¬£1,234.50",1e3,1000.00,, both ,Ætna,2010-04-01 00:00:00,05/02/2024,01/04/2024,Y -0012,0000000012,£.00,¬£.00, 7,7.00,a b,none,Müller £,2024-04-01,25/12/2024,,FALSE -#N/A,,,,,25,,,,,,, +ref_text,ref_padded,amount,amount_mojibake,number_as_text,number,blank_like,padded_text,accented,date_text,date_mixed,date_locale,boolean_like +0000130842,0000130842,£43.40,¬£43.40,12.50,12.50, , lead,Café,13/04/2024,04/01/2024,04/01/2024,TRUE +0000000001,0000000001,£-826.80,¬£-826.80,0.10,0.10, ,trail ,naïve,04/01/2024,13/04/2024,25/12/2024,false +1234567890,1234567890,"£1,234.50","¬£1,234.50",1e3,1000.00,, both ,Ætna,2010-04-01 00:00:00,05/02/2024,01/04/2024,Y +0012,0000000012,£.00,¬£.00, 7,7.00,a b,none,Müller £,2024-04-01,25/12/2024,,FALSE +#N/A,,,,,25,,,,,,, diff --git a/tests/data/synthetic-excel/fixture-excel-windows.csv b/tests/data/synthetic-excel/fixture-excel-windows.csv index 7393643..264f8ce 100644 --- a/tests/data/synthetic-excel/fixture-excel-windows.csv +++ b/tests/data/synthetic-excel/fixture-excel-windows.csv @@ -1,6 +1,6 @@ -ref_text,ref_padded,amount,amount_mojibake,number_as_text,number,blank_like,padded_text,accented,date_text,date_mixed,date_locale,boolean_like -0000130842,0000130842,£43.40,¬£43.40,12.50,12.50, , lead,Café,13/04/2024,04/01/2024,04/01/2024,TRUE -0000000001,0000000001,£-826.80,¬£-826.80,0.10,0.10, ,trail ,naïve,04/01/2024,13/04/2024,25/12/2024,false -1234567890,1234567890,"£1,234.50","¬£1,234.50",1e3,1000.00,, both ,Ætna,2010-04-01 00:00:00,05/02/2024,01/04/2024,Y -0012,0000000012,£.00,¬£.00, 7,7.00,a b,none,Müller £,2024-04-01,25/12/2024,,FALSE -#N/A,,,,,25,,,,,,, +ref_text,ref_padded,amount,amount_mojibake,number_as_text,number,blank_like,padded_text,accented,date_text,date_mixed,date_locale,boolean_like +0000130842,0000130842,£43.40,¬£43.40,12.50,12.50, , lead,Café,13/04/2024,04/01/2024,04/01/2024,TRUE +0000000001,0000000001,£-826.80,¬£-826.80,0.10,0.10, ,trail ,naïve,04/01/2024,13/04/2024,25/12/2024,false +1234567890,1234567890,"£1,234.50","¬£1,234.50",1e3,1000.00,, both ,Ætna,2010-04-01 00:00:00,05/02/2024,01/04/2024,Y +0012,0000000012,£.00,¬£.00, 7,7.00,a b,none,Müller £,2024-04-01,25/12/2024,,FALSE +#N/A,,,,,25,,,,,,, diff --git a/tests/data/synthetic-layouts/answer-key.json b/tests/data/synthetic-layouts/answer-key.json new file mode 100644 index 0000000..3ae000c --- /dev/null +++ b/tests/data/synthetic-layouts/answer-key.json @@ -0,0 +1,5158 @@ +{ + "generated": "2026-10-30T19:48:46", + "csv": { + "title-above.csv": { + "lines": [ + [ + "Business rates reliefs, April 2026" + ], + [], + [], + [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ], + "problem": "a title and two blank lines above the header", + "dialect": { + "skipRows": 3 + }, + "names": [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ], + "propose": { + "skipRows": 3, + "why": "the first line with as many fields as the data is line 3" + } + }, + "blank-line-inside.csv": { + "lines": [ + [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ], + "problem": "a blank line inside the data, kept as a row and numbered", + "dialect": {}, + "names": [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "", + "", + "", + "", + "", + "" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ], + "propose": { + "blank_rows": [ + 3 + ] + } + }, + "quoted-line-break.csv": { + "lines": [ + [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road\nNewbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ], + "problem": "a quoted cell holding a line break, one record over two lines of the file", + "dialect": {}, + "names": [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road\nNewbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ] + }, + "semicolon.csv": { + "lines": [ + [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ], + "problem": "`;` between cells, and commas inside the addresses unquoted", + "dialect": { + "delimiter": ";" + }, + "names": [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ], + "propose": { + "delimiter": ";" + } + }, + "stray-quote.csv": { + "lines": [ + [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + [ + "NN00101700004185", + "Unit 5 Orpheus House Calleva Park Aldermaston Reading RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31 Meadow Road Newbury RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "\"Old Mill Station Road Pangbourne", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1 Andromeda House Calleva Park Aldermaston Reading RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall The Street Aldermaston Reading Berkshire RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn Aldworth Reading RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ], + "problem": "a cell opening with a quote that never closes, which swallows the rest of the file if read as a quote (1.x needed `quoting=csv.QUOTE_NONE`)", + "dialect": { + "quoteChar": null + }, + "names": [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5 Orpheus House Calleva Park Aldermaston Reading RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31 Meadow Road Newbury RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "\"Old Mill Station Road Pangbourne", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1 Andromeda House Calleva Park Aldermaston Reading RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall The Street Aldermaston Reading Berkshire RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn Aldworth Reading RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ], + "propose": { + "quoteChar": null, + "why": "an opening quote with no close before the end of the file" + } + }, + "trailing-delimiter.csv": { + "lines": [ + [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact", + "" + ], + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A", + "" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " ", + "" + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk", + "" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A", + "" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord ", + "" + ] + ], + "problem": "every line ends with a delimiter, an empty last column with an empty header", + "dialect": {}, + "names": [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ], + "propose": { + "why": "the empty last column is trimmed with the grid, as fp0 trims it" + } + }, + "two-line-records.csv": { + "lines": [ + [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date" + ], + [ + "Contact", + "", + "", + "", + "" + ], + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024" + ], + [ + "N/A", + "", + "", + "", + "" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024" + ], + [ + " ", + "", + "", + "", + "" + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024" + ], + [ + "Parish Clerk", + "", + "", + "", + "" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024" + ], + [ + "", + "", + "", + "", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026" + ], + [ + "N/A", + "", + "", + "", + "" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026" + ], + [ + " Landlord ", + "", + "", + "", + "" + ] + ], + "problem": "each record, header included, over two lines: five fields, then one in the first field", + "dialect": { + "rowsPerRecord": [ + 5, + 1 + ] + }, + "names": [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ], + "propose": { + "rowsPerRecord": [ + 5, + 1 + ], + "why": "every second line has one value, in its first field" + }, + "refuse": "a record without its second line, or a second line with values past its first field" + }, + "two-line-records-unpadded.csv": { + "lines": [ + [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date" + ], + [ + "Contact" + ], + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024" + ], + [ + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024" + ], + [ + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024" + ], + [ + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024" + ], + [ + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026" + ], + [ + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026" + ], + [ + " Landlord " + ] + ], + "problem": "each record over two lines, the second holding its one value and no delimiters, so an empty value is a blank line", + "dialect": { + "rowsPerRecord": [ + 5, + 1 + ] + }, + "names": [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ], + "propose": { + "rowsPerRecord": [ + 5, + 1 + ], + "why": "every second line has at most one value" + } + }, + "backslash-escape.csv": { + "lines": [ + [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "The \"Old\" Mill, Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ], + "problem": "a quote inside a quoted cell escaped by a backslash, not doubled", + "dialect": { + "doubleQuote": false + }, + "names": [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "The \"Old\" Mill, Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ], + "propose": { + "doubleQuote": false, + "why": "a backslash before a quote inside a quoted cell" + } + }, + "cr-line-ends.csv": { + "lines": [ + [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ], + "problem": "every line ending `\\r` alone, as old Mac files do", + "dialect": {}, + "names": [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ] + }, + "header-narrower.csv": { + "lines": [ + [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date" + ], + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ], + "problem": "the header a field narrower than the data, so the last column has no name", + "dialect": {}, + "names": [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Unnamed: 5" + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ], + "renamed": "the column past the header by position, as an empty name is" + }, + "first-row-narrower.csv": { + "lines": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ], + "problem": "no header, and the first row a field narrower than the rest", + "dialect": { + "headerRowCount": 0 + }, + "names": [ + "column_0", + "column_1", + "column_2", + "column_3", + "column_4", + "column_5" + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ] + }, + "row-narrower.csv": { + "lines": [ + [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ], + "problem": "one row two fields narrower than the header, its last cells empty", + "dialect": {}, + "names": [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "", + "" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ] + }, + "hostile-headers.csv": { + "lines": [ + [ + "Owner's name", + "The \"Main\" Rate", + "Rate [2026]", + "R/V > 0", + "", + "Code, Local", + " padded ", + "line\nbreak", + "", + "Rate", + "Rate", + "Rate Relief", + "__row__" + ], + [ + "v00", + "v01", + "v02", + "v03", + "v04", + "v05", + "v06", + "v07", + "v08", + "v09", + "v010", + "v011", + "v012" + ], + [ + "v10", + "v11", + "v12", + "v13", + "v14", + "v15", + "v16", + "v17", + "v18", + "v19", + "v110", + "v111", + "v112" + ] + ], + "problem": "header names a script must still resolve: every name read exactly, none trimmed or joined", + "dialect": {}, + "names_as_written": [ + "Owner's name", + "The \"Main\" Rate", + "Rate [2026]", + "R/V > 0", + "", + "Code, Local", + " padded ", + "line\nbreak", + "", + "Rate", + "Rate", + "Rate Relief", + "__row__" + ], + "rows": [ + [ + "v00", + "v01", + "v02", + "v03", + "v04", + "v05", + "v06", + "v07", + "v08", + "v09", + "v010", + "v011", + "v012" + ], + [ + "v10", + "v11", + "v12", + "v13", + "v14", + "v15", + "v16", + "v17", + "v18", + "v19", + "v110", + "v111", + "v112" + ] + ], + "names": [ + "Owner's name", + "The \"Main\" Rate", + "Rate [2026]", + "R/V > 0", + "", + "Code, Local", + " padded ", + "line\nbreak", + "Unnamed: 8", + "Rate", + "Rate.1", + "Rate Relief", + "__row__.1" + ], + "renamed": "the empty name by position, as stored schemas name it; the repeat and the row key's own name by RowKeyParser.deconflict" + }, + "missing-like-text.csv": { + "lines": [ + [ + "Contact", + "n" + ], + [ + "N/A", + "0" + ], + [ + "NA", + "1" + ], + [ + "null", + "2" + ], + [ + "NaN", + "3" + ], + [ + "#N/A", + "4" + ], + [ + "n/a", + "5" + ], + [ + "-", + "6" + ], + [ + "None", + "7" + ] + ], + "problem": "text pandas reads as missing by default; every one is text, and only the source schema can say it means missing", + "dialect": {}, + "names": [ + "Contact", + "n" + ], + "rows": [ + [ + "N/A", + "0" + ], + [ + "NA", + "1" + ], + [ + "null", + "2" + ], + [ + "NaN", + "3" + ], + [ + "#N/A", + "4" + ], + [ + "n/a", + "5" + ], + [ + "-", + "6" + ], + [ + "None", + "7" + ] + ] + } + }, + "workbook": { + "layouts.xlsx": { + "Titled": { + "problem": "a title and subtitle, then a blank row, above the header on row 4", + "dialect": { + "skipRows": 3 + }, + "names": [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "2024-01-04", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "2024-02-05", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + null + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ], + "date_cells": [ + "E5", + "E7" + ], + "propose": { + "skipRows": 3 + } + }, + "MergedHeader": { + "problem": "a header over two rows: one name merged down both, two merged across two columns each", + "dialect": { + "headerRowCount": 2 + }, + "header_rows": [ + [ + "Propref", + "Property", + null, + "Relief", + null + ], + [ + null, + "Address", + "Postcode", + "Code", + "R/V" + ] + ], + "merged": [ + "A1:A2", + "B1:C1", + "D1:E1" + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading", + "RG7 8TA", + "SBR3", + "13250" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury", + "RG14 7AH", + "SBRE", + "10250" + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading", + "RG8 7AN", + "RH0", + "8600" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading", + "RG7 8AN", + "SBR1", + "16250" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire", + "RG7 4LW", + "MAN", + "5400" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading", + "RG8 9SE", + "PU15", + "24000" + ] + ], + "names": "supplied by the wrangler, one per column: a merged cell over two may be one title or a heading over subheadings, so whyqd never builds a name from it", + "refuse": "headerRowCount 2 without names: the header rows and merged ranges named, and names asked for", + "propose": { + "headerRowCount": 2, + "why": "merged ranges in rows 1 and 2, and data from row 3" + } + }, + "BlankColumns": { + "problem": "an empty column inside the table, and three empty but formatted columns after it", + "dialect": {}, + "names": [ + "Propref", + "Property Address", + "Unnamed: 2", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + null, + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + null, + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + null, + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + null, + "SBR1", + "16250", + "25/12/2024", + null + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + null, + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + null, + "PU15", + "24000", + "30/01/2026", + " Landlord " + ] + ], + "propose": { + "empty_columns": [ + 2 + ], + "why": "the trailing columns hold formatting only, and are trimmed" + } + }, + "SubTables": { + "problem": "a second table below a blank row, then footnotes; loaded as given, every row is in the table", + "dialect": {}, + "names": [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + null + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + " Landlord " + ], + [ + null, + null, + null, + null, + null, + null + ], + [ + "Code", + "Description", + null, + null, + null, + null + ], + [ + "SBR3", + "Small Business Rate Relief", + null, + null, + null, + null + ], + [ + "RH0", + "Retail, Hospitality and Leisure", + null, + null, + null, + null + ], + [ + "MAN", + "Mandatory Charity Relief", + null, + null, + null, + null + ], + [ + null, + null, + null, + null, + null, + null + ], + [ + "Source: business rates system, extracted 15/04/2026", + null, + null, + null, + null, + null + ], + [ + "* R/V is the rateable value in pounds", + null, + null, + null, + null, + null + ] + ], + "propose": { + "table_rows": [ + 0, + 5 + ], + "second_table": { + "header_row": 7, + "rows": 3 + }, + "footnotes": [ + 12, + 13 + ] + } + }, + "Errors": { + "problem": "error cells in a number column", + "dialect": {}, + "names": [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "#N/A", + "13/04/2024", + " " + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "#DIV/0!", + "25/12/2024", + null + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "#REF!", + "30/01/2026", + " Landlord " + ] + ], + "error_cells": { + "D3": "#N/A", + "D5": "#DIV/0!", + "D7": "#REF!" + } + }, + "Workings": { + "problem": "a hidden sheet, read like any other and flagged", + "state": "hidden", + "dialect": {}, + "names": [ + "Total R/V" + ], + "rows": [], + "formula_cells": { + "A2": "a formula written without its cached value, so the row is empty here and trimmed; the .xls, saved by LibreOffice, caches 0" + } + } + }, + "layouts.xls": "layouts.xlsx saved as .xls in LibreOffice; read to the same key", + "single-sheet.xlsx": { + "Reliefs": { + "problem": "the only sheet, with row 2 blank: 1.x's one-row probe judged it empty and dropped it", + "dialect": {}, + "names": [ + "Code", + "Description" + ], + "rows": [ + [ + null, + null + ], + [ + "SBR3", + "Small Business Rate Relief" + ], + [ + "RH0", + "Retail, Hospitality and Leisure" + ], + [ + "MAN", + "Mandatory Charity Relief" + ] + ], + "propose": { + "blank_rows": [ + 0 + ] + } + } + } + }, + "pdf": { + "pdf-source.pdf": { + "Ruled": { + "problem": "a ruled grid, long addresses wrapped over several lines in their cells", + "names": [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ] + ], + "note": "a date cell exports as its displayed text, dd/mm/yyyy, so the rows are the text", + "wrapped": "an address extracts with a line break at each wrap, and keeps it: a wrap and a line break in the cell look the same in a PDF, so joining is a declared step, never the reader's; the ruling keeps each wrapped cell in its one row" + }, + "MergedHeader": { + "problem": "a ruled header over two rows with merged cells", + "header_rows": [ + [ + "Propref", + "Property", + "", + "Relief", + "" + ], + [ + "", + "Address", + "Postcode", + "Code", + "R/V" + ] + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading", + "RG7 8TA", + "SBR3", + "13250" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury", + "RG14 7AH", + "SBRE", + "10250" + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading", + "RG8 7AN", + "RH0", + "8600" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading", + "RG7 8AN", + "SBR1", + "16250" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire", + "RG7 4LW", + "MAN", + "5400" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading", + "RG8 9SE", + "PU15", + "24000" + ] + ], + "names": "supplied by the wrangler, as layouts.xlsx § MergedHeader" + }, + "TwoTables": { + "problem": "two tables on one page, unruled, two blank rows apart", + "tables": [ + { + "names": [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ] + ] + }, + { + "names": [ + "Code", + "Description" + ], + "rows": [ + [ + "SBR3", + "Small Business Rate Relief" + ], + [ + "RH0", + "Retail, Hospitality and Leisure" + ], + [ + "MAN", + "Mandatory Charity Relief" + ] + ] + } + ] + }, + "Landscape": { + "problem": "a wide table on a landscape page", + "names": [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact", + "Liable From", + "Account Holder", + "Ward", + "Band" + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A", + "01/04/2010", + "Orpheus Holdings Ltd", + "Aldermaston", + "B" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "", + "01/04/2017", + "Meadow Road Traders", + "Newbury Central", + "C" + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk", + "12/06/2019", + "Pangbourne Station Ltd", + "Pangbourne", + "A" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "", + "01/04/2023", + "Andromeda Estates", + "Aldermaston", + "B" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A", + "01/04/2005", + "Aldermaston Parish Council", + "Aldermaston", + "A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord", + "30/09/2016", + "The Bell Inn (Aldworth) Ltd", + "Basildon", + "B" + ] + ] + }, + "RepeatedHeader": { + "problem": "150 rows over several pages, the header repeated on every page", + "names": [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + "rows": [ + [ + "NN00900000000000", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000001", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000002", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000003", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000004", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000005", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000006", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000007", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000008", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000009", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000010", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000011", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000012", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000013", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000014", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000015", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000016", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000017", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000018", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000019", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000020", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000021", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000022", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000023", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000024", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000025", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000026", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000027", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000028", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000029", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000030", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000031", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000032", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000033", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000034", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000035", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000036", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000037", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000038", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000039", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000040", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000041", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000042", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000043", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000044", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000045", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000046", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000047", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000048", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000049", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000050", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000051", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000052", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000053", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000054", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000055", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000056", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000057", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000058", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000059", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000060", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000061", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000062", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000063", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000064", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000065", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000066", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000067", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000068", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000069", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000070", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000071", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000072", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000073", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000074", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000075", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000076", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000077", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000078", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000079", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000080", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000081", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000082", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000083", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000084", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000085", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000086", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000087", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000088", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000089", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000090", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000091", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000092", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000093", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000094", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000095", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000096", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000097", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000098", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000099", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000100", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000101", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000102", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000103", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000104", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000105", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000106", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000107", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000108", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000109", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000110", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000111", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000112", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000113", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000114", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000115", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000116", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000117", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000118", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000119", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000120", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000121", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000122", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000123", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000124", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000125", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000126", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000127", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000128", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000129", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000130", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000131", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000132", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000133", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000134", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000135", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000136", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000137", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000138", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000139", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000140", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000141", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000142", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000143", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000144", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000145", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000146", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000147", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000148", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000149", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ] + ] + }, + "HeaderOnce": { + "problem": "150 rows over several pages, the header on the first page only", + "names": [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + "rows": [ + [ + "NN00900000000000", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000001", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000002", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000003", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000004", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000005", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000006", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000007", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000008", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000009", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000010", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000011", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000012", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000013", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000014", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000015", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000016", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000017", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000018", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000019", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000020", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000021", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000022", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000023", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000024", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000025", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000026", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000027", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000028", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000029", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000030", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000031", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000032", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000033", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000034", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000035", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000036", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000037", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000038", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000039", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000040", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000041", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000042", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000043", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000044", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000045", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000046", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000047", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000048", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000049", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000050", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000051", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000052", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000053", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000054", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000055", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000056", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000057", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000058", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000059", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000060", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000061", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000062", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000063", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000064", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000065", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000066", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000067", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000068", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000069", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000070", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000071", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000072", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000073", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000074", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000075", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000076", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000077", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000078", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000079", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000080", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000081", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000082", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000083", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000084", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000085", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000086", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000087", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000088", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000089", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000090", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000091", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000092", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000093", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000094", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000095", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000096", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000097", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000098", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000099", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000100", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000101", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000102", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000103", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000104", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000105", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000106", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000107", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000108", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000109", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000110", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000111", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000112", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000113", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000114", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000115", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000116", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000117", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000118", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000119", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000120", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000121", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000122", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000123", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000124", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000125", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000126", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000127", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000128", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000129", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000130", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000131", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000132", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000133", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000134", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000135", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000136", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000137", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000138", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000139", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000140", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000141", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000142", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000143", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ], + [ + "NN00900000000144", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN00900000000145", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN00900000000146", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN00900000000147", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN00900000000148", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00900000000149", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ] + ] + }, + "UnruledWrapped": { + "problem": "no ruling, and long addresses wrapped in their cells: read by text position, each wrapped line becomes a row of its own and words split across columns", + "names": [ + "Propref", + "Property Address", + "Ret Code", + "R/V", + "Start Date", + "Contact" + ], + "rows": [ + [ + "NN00101700004185", + "Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA", + "SBR3", + "13250", + "04/01/2024", + "N/A" + ], + [ + "NN03905740027005", + "27-31, Meadow Road, Newbury, RG14 7AH", + "SBRE", + "10250", + "13/04/2024", + "" + ], + [ + "NN04105100003115", + "At Station Road, Pangbourne, Reading, RG8 7AN", + "RH0", + "8600", + "05/02/2024", + "Parish Clerk" + ], + [ + "NN0010170000401A", + "1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN", + "SBR1", + "16250", + "25/12/2024", + "" + ], + [ + "NN0010420000111A", + "Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW", + "MAN", + "5400", + "01/02/2026", + "N/A" + ], + [ + "NN00201200001117", + "The Bell Inn, Aldworth, Reading, RG8 9SE", + "PU15", + "24000", + "30/01/2026", + "Landlord" + ] + ], + "report": "loaded by the default reading, the rows that look like continuations (an empty first cell, text only where a cell wraps) are shown, so the wrangler can declare how the table is read", + "declared": "with the table's reading declared (its columns' bounds, which rows continue a record), the rows above, each wrap a line break in its cell" + } + }, + "made": "pdf-source.xlsx exported in LibreOffice, every sheet, one PDF; each sheet's table read across its pages" + } +} \ No newline at end of file diff --git a/tests/data/synthetic-layouts/backslash-escape.csv b/tests/data/synthetic-layouts/backslash-escape.csv new file mode 100644 index 0000000..4282cee --- /dev/null +++ b/tests/data/synthetic-layouts/backslash-escape.csv @@ -0,0 +1,7 @@ +Propref,Property Address,Ret Code,R/V,Start Date,Contact +NN00101700004185,"Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA",SBR3,13250,04/01/2024,N/A +NN03905740027005,"27-31, Meadow Road, Newbury, RG14 7AH",SBRE,10250,13/04/2024," " +NN04105100003115,"The \"Old\" Mill, Station Road, Pangbourne, Reading, RG8 7AN",RH0,8600,05/02/2024,Parish Clerk +NN0010170000401A,"1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN",SBR1,16250,25/12/2024, +NN0010420000111A,"Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW",MAN,5400,01/02/2026,N/A +NN00201200001117,"The Bell Inn, Aldworth, Reading, RG8 9SE",PU15,24000,30/01/2026," Landlord " diff --git a/tests/data/synthetic-layouts/blank-line-inside.csv b/tests/data/synthetic-layouts/blank-line-inside.csv new file mode 100644 index 0000000..5470d39 --- /dev/null +++ b/tests/data/synthetic-layouts/blank-line-inside.csv @@ -0,0 +1,8 @@ +Propref,Property Address,Ret Code,R/V,Start Date,Contact +NN00101700004185,"Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA",SBR3,13250,04/01/2024,N/A +NN03905740027005,"27-31, Meadow Road, Newbury, RG14 7AH",SBRE,10250,13/04/2024," " +NN04105100003115,"At Station Road, Pangbourne, Reading, RG8 7AN",RH0,8600,05/02/2024,Parish Clerk + +NN0010170000401A,"1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN",SBR1,16250,25/12/2024, +NN0010420000111A,"Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW",MAN,5400,01/02/2026,N/A +NN00201200001117,"The Bell Inn, Aldworth, Reading, RG8 9SE",PU15,24000,30/01/2026," Landlord " diff --git a/tests/data/synthetic-layouts/cr-line-ends.csv b/tests/data/synthetic-layouts/cr-line-ends.csv new file mode 100644 index 0000000..2c94262 --- /dev/null +++ b/tests/data/synthetic-layouts/cr-line-ends.csv @@ -0,0 +1 @@ +Propref,Property Address,Ret Code,R/V,Start Date,Contact NN00101700004185,"Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA",SBR3,13250,04/01/2024,N/A NN03905740027005,"27-31, Meadow Road, Newbury, RG14 7AH",SBRE,10250,13/04/2024," " NN04105100003115,"At Station Road, Pangbourne, Reading, RG8 7AN",RH0,8600,05/02/2024,Parish Clerk NN0010170000401A,"1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN",SBR1,16250,25/12/2024, NN0010420000111A,"Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW",MAN,5400,01/02/2026,N/A NN00201200001117,"The Bell Inn, Aldworth, Reading, RG8 9SE",PU15,24000,30/01/2026," Landlord " \ No newline at end of file diff --git a/tests/data/synthetic-layouts/first-row-narrower.csv b/tests/data/synthetic-layouts/first-row-narrower.csv new file mode 100644 index 0000000..9604d1b --- /dev/null +++ b/tests/data/synthetic-layouts/first-row-narrower.csv @@ -0,0 +1,6 @@ +NN00101700004185,"Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA",SBR3,13250,04/01/2024 +NN03905740027005,"27-31, Meadow Road, Newbury, RG14 7AH",SBRE,10250,13/04/2024," " +NN04105100003115,"At Station Road, Pangbourne, Reading, RG8 7AN",RH0,8600,05/02/2024,Parish Clerk +NN0010170000401A,"1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN",SBR1,16250,25/12/2024, +NN0010420000111A,"Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW",MAN,5400,01/02/2026,N/A +NN00201200001117,"The Bell Inn, Aldworth, Reading, RG8 9SE",PU15,24000,30/01/2026," Landlord " diff --git a/tests/data/synthetic-layouts/header-narrower.csv b/tests/data/synthetic-layouts/header-narrower.csv new file mode 100644 index 0000000..d788d79 --- /dev/null +++ b/tests/data/synthetic-layouts/header-narrower.csv @@ -0,0 +1,7 @@ +Propref,Property Address,Ret Code,R/V,Start Date +NN00101700004185,"Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA",SBR3,13250,04/01/2024,N/A +NN03905740027005,"27-31, Meadow Road, Newbury, RG14 7AH",SBRE,10250,13/04/2024," " +NN04105100003115,"At Station Road, Pangbourne, Reading, RG8 7AN",RH0,8600,05/02/2024,Parish Clerk +NN0010170000401A,"1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN",SBR1,16250,25/12/2024, +NN0010420000111A,"Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW",MAN,5400,01/02/2026,N/A +NN00201200001117,"The Bell Inn, Aldworth, Reading, RG8 9SE",PU15,24000,30/01/2026," Landlord " diff --git a/tests/data/synthetic-layouts/hostile-headers.csv b/tests/data/synthetic-layouts/hostile-headers.csv new file mode 100644 index 0000000..bafcd67 --- /dev/null +++ b/tests/data/synthetic-layouts/hostile-headers.csv @@ -0,0 +1,4 @@ +Owner's name,"The ""Main"" Rate",Rate [2026],R/V > 0,,"Code, Local"," padded ","line +break",,Rate,Rate,Rate Relief,__row__ +v00,v01,v02,v03,v04,v05,v06,v07,v08,v09,v010,v011,v012 +v10,v11,v12,v13,v14,v15,v16,v17,v18,v19,v110,v111,v112 diff --git a/tests/data/synthetic-layouts/layouts.xls b/tests/data/synthetic-layouts/layouts.xls new file mode 100644 index 0000000..682944d Binary files /dev/null and b/tests/data/synthetic-layouts/layouts.xls differ diff --git a/tests/data/synthetic-layouts/layouts.xlsx b/tests/data/synthetic-layouts/layouts.xlsx new file mode 100644 index 0000000..e1e9cfb Binary files /dev/null and b/tests/data/synthetic-layouts/layouts.xlsx differ diff --git a/tests/data/synthetic-layouts/missing-like-text.csv b/tests/data/synthetic-layouts/missing-like-text.csv new file mode 100644 index 0000000..342fa63 --- /dev/null +++ b/tests/data/synthetic-layouts/missing-like-text.csv @@ -0,0 +1,9 @@ +Contact,n +N/A,0 +NA,1 +null,2 +NaN,3 +#N/A,4 +n/a,5 +-,6 +None,7 diff --git a/tests/data/synthetic-layouts/pdf-source.pdf b/tests/data/synthetic-layouts/pdf-source.pdf new file mode 100644 index 0000000..fb90777 Binary files /dev/null and b/tests/data/synthetic-layouts/pdf-source.pdf differ diff --git a/tests/data/synthetic-layouts/pdf-source.xlsx b/tests/data/synthetic-layouts/pdf-source.xlsx new file mode 100644 index 0000000..3291a3a Binary files /dev/null and b/tests/data/synthetic-layouts/pdf-source.xlsx differ diff --git a/tests/data/synthetic-layouts/quoted-line-break.csv b/tests/data/synthetic-layouts/quoted-line-break.csv new file mode 100644 index 0000000..c31c721 --- /dev/null +++ b/tests/data/synthetic-layouts/quoted-line-break.csv @@ -0,0 +1,8 @@ +Propref,Property Address,Ret Code,R/V,Start Date,Contact +NN00101700004185,"Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA",SBR3,13250,04/01/2024,N/A +NN03905740027005,"27-31, Meadow Road +Newbury, RG14 7AH",SBRE,10250,13/04/2024," " +NN04105100003115,"At Station Road, Pangbourne, Reading, RG8 7AN",RH0,8600,05/02/2024,Parish Clerk +NN0010170000401A,"1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN",SBR1,16250,25/12/2024, +NN0010420000111A,"Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW",MAN,5400,01/02/2026,N/A +NN00201200001117,"The Bell Inn, Aldworth, Reading, RG8 9SE",PU15,24000,30/01/2026," Landlord " diff --git a/tests/data/synthetic-layouts/row-narrower.csv b/tests/data/synthetic-layouts/row-narrower.csv new file mode 100644 index 0000000..0922be7 --- /dev/null +++ b/tests/data/synthetic-layouts/row-narrower.csv @@ -0,0 +1,7 @@ +Propref,Property Address,Ret Code,R/V,Start Date,Contact +NN00101700004185,"Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA",SBR3,13250,04/01/2024,N/A +NN03905740027005,"27-31, Meadow Road, Newbury, RG14 7AH",SBRE,10250,13/04/2024," " +NN04105100003115,"At Station Road, Pangbourne, Reading, RG8 7AN",RH0,8600 +NN0010170000401A,"1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN",SBR1,16250,25/12/2024, +NN0010420000111A,"Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW",MAN,5400,01/02/2026,N/A +NN00201200001117,"The Bell Inn, Aldworth, Reading, RG8 9SE",PU15,24000,30/01/2026," Landlord " diff --git a/tests/data/synthetic-layouts/semicolon.csv b/tests/data/synthetic-layouts/semicolon.csv new file mode 100644 index 0000000..0aab88b --- /dev/null +++ b/tests/data/synthetic-layouts/semicolon.csv @@ -0,0 +1,7 @@ +Propref;Property Address;Ret Code;R/V;Start Date;Contact +NN00101700004185;Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA;SBR3;13250;04/01/2024;N/A +NN03905740027005;27-31, Meadow Road, Newbury, RG14 7AH;SBRE;10250;13/04/2024;" " +NN04105100003115;At Station Road, Pangbourne, Reading, RG8 7AN;RH0;8600;05/02/2024;Parish Clerk +NN0010170000401A;1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN;SBR1;16250;25/12/2024; +NN0010420000111A;Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW;MAN;5400;01/02/2026;N/A +NN00201200001117;The Bell Inn, Aldworth, Reading, RG8 9SE;PU15;24000;30/01/2026;" Landlord " diff --git a/tests/data/synthetic-layouts/single-sheet.xlsx b/tests/data/synthetic-layouts/single-sheet.xlsx new file mode 100644 index 0000000..55160ca Binary files /dev/null and b/tests/data/synthetic-layouts/single-sheet.xlsx differ diff --git a/tests/data/synthetic-layouts/stray-quote.csv b/tests/data/synthetic-layouts/stray-quote.csv new file mode 100644 index 0000000..e373a33 --- /dev/null +++ b/tests/data/synthetic-layouts/stray-quote.csv @@ -0,0 +1,7 @@ +Propref,Property Address,Ret Code,R/V,Start Date,Contact +NN00101700004185,Unit 5 Orpheus House Calleva Park Aldermaston Reading RG7 8TA,SBR3,13250,04/01/2024,N/A +NN03905740027005,27-31 Meadow Road Newbury RG14 7AH,SBRE,10250,13/04/2024, +NN04105100003115,"Old Mill Station Road Pangbourne,RH0,8600,05/02/2024,Parish Clerk +NN0010170000401A,1 Andromeda House Calleva Park Aldermaston Reading RG7 8AN,SBR1,16250,25/12/2024, +NN0010420000111A,Aldermaston Parish Hall The Street Aldermaston Reading Berkshire RG7 4LW,MAN,5400,01/02/2026,N/A +NN00201200001117,The Bell Inn Aldworth Reading RG8 9SE,PU15,24000,30/01/2026, Landlord diff --git a/tests/data/synthetic-layouts/title-above.csv b/tests/data/synthetic-layouts/title-above.csv new file mode 100644 index 0000000..b6ec20a --- /dev/null +++ b/tests/data/synthetic-layouts/title-above.csv @@ -0,0 +1,10 @@ +"Business rates reliefs, April 2026" + + +Propref,Property Address,Ret Code,R/V,Start Date,Contact +NN00101700004185,"Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA",SBR3,13250,04/01/2024,N/A +NN03905740027005,"27-31, Meadow Road, Newbury, RG14 7AH",SBRE,10250,13/04/2024," " +NN04105100003115,"At Station Road, Pangbourne, Reading, RG8 7AN",RH0,8600,05/02/2024,Parish Clerk +NN0010170000401A,"1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN",SBR1,16250,25/12/2024, +NN0010420000111A,"Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW",MAN,5400,01/02/2026,N/A +NN00201200001117,"The Bell Inn, Aldworth, Reading, RG8 9SE",PU15,24000,30/01/2026," Landlord " diff --git a/tests/data/synthetic-layouts/trailing-delimiter.csv b/tests/data/synthetic-layouts/trailing-delimiter.csv new file mode 100644 index 0000000..2cdeaed --- /dev/null +++ b/tests/data/synthetic-layouts/trailing-delimiter.csv @@ -0,0 +1,7 @@ +Propref,Property Address,Ret Code,R/V,Start Date,Contact, +NN00101700004185,"Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA",SBR3,13250,04/01/2024,N/A, +NN03905740027005,"27-31, Meadow Road, Newbury, RG14 7AH",SBRE,10250,13/04/2024," ", +NN04105100003115,"At Station Road, Pangbourne, Reading, RG8 7AN",RH0,8600,05/02/2024,Parish Clerk, +NN0010170000401A,"1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN",SBR1,16250,25/12/2024,, +NN0010420000111A,"Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW",MAN,5400,01/02/2026,N/A, +NN00201200001117,"The Bell Inn, Aldworth, Reading, RG8 9SE",PU15,24000,30/01/2026," Landlord ", diff --git a/tests/data/synthetic-layouts/two-line-records-unpadded.csv b/tests/data/synthetic-layouts/two-line-records-unpadded.csv new file mode 100644 index 0000000..f7a7bc8 --- /dev/null +++ b/tests/data/synthetic-layouts/two-line-records-unpadded.csv @@ -0,0 +1,14 @@ +Propref,Property Address,Ret Code,R/V,Start Date +Contact +NN00101700004185,"Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA",SBR3,13250,04/01/2024 +N/A +NN03905740027005,"27-31, Meadow Road, Newbury, RG14 7AH",SBRE,10250,13/04/2024 +" " +NN04105100003115,"At Station Road, Pangbourne, Reading, RG8 7AN",RH0,8600,05/02/2024 +Parish Clerk +NN0010170000401A,"1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN",SBR1,16250,25/12/2024 + +NN0010420000111A,"Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW",MAN,5400,01/02/2026 +N/A +NN00201200001117,"The Bell Inn, Aldworth, Reading, RG8 9SE",PU15,24000,30/01/2026 +" Landlord " diff --git a/tests/data/synthetic-layouts/two-line-records.csv b/tests/data/synthetic-layouts/two-line-records.csv new file mode 100644 index 0000000..3c4f9c8 --- /dev/null +++ b/tests/data/synthetic-layouts/two-line-records.csv @@ -0,0 +1,14 @@ +Propref,Property Address,Ret Code,R/V,Start Date +Contact,,,, +NN00101700004185,"Unit 5, Orpheus House, Calleva Park, Aldermaston, Reading, RG7 8TA",SBR3,13250,04/01/2024 +N/A,,,, +NN03905740027005,"27-31, Meadow Road, Newbury, RG14 7AH",SBRE,10250,13/04/2024 +" ",,,, +NN04105100003115,"At Station Road, Pangbourne, Reading, RG8 7AN",RH0,8600,05/02/2024 +Parish Clerk,,,, +NN0010170000401A,"1, Andromeda House, Calleva Park, Aldermaston, Reading, RG7 8AN",SBR1,16250,25/12/2024 +,,,, +NN0010420000111A,"Aldermaston Parish Hall, The Street, Aldermaston, Reading, Berkshire, RG7 4LW",MAN,5400,01/02/2026 +N/A,,,, +NN00201200001117,"The Bell Inn, Aldworth, Reading, RG8 9SE",PU15,24000,30/01/2026 +" Landlord ",,,, diff --git a/tests/data/synthetic/fixture-utf8-crlf.csv b/tests/data/synthetic/fixture-utf8-crlf.csv index 646a9cb..1ada896 100644 --- a/tests/data/synthetic/fixture-utf8-crlf.csv +++ b/tests/data/synthetic/fixture-utf8-crlf.csv @@ -1,5 +1,5 @@ -ref_text,ref_padded,amount,amount_mojibake,number_as_text,number,blank_like,padded_text,accented,date_text,date_mixed,boolean_like -0000130842,0000130842,£43.40,¬£43.40,12.50,12.50," "," lead",Café,13/04/2024,04/01/2024,TRUE -0000000001,0000000001,£-826.80,¬£-826.80,0.10,0.10," ","trail ",naïve,04/01/2024,13/04/2024,false -1234567890,1234567890,"£1,234.50","¬£1,234.50",1e3,1000.00,," both ",Ætna,2010-04-01 00:00:00,05/02/2024,Y -0012,0000000012,£.00,¬£.00," 7",7.00,a b,none,Müller £,2024-04-01,25/12/2024,FALSE +ref_text,ref_padded,amount,amount_mojibake,number_as_text,number,blank_like,padded_text,accented,date_text,date_mixed,boolean_like +0000130842,0000130842,£43.40,¬£43.40,12.50,12.50," "," lead",Café,13/04/2024,04/01/2024,TRUE +0000000001,0000000001,£-826.80,¬£-826.80,0.10,0.10," ","trail ",naïve,04/01/2024,13/04/2024,false +1234567890,1234567890,"£1,234.50","¬£1,234.50",1e3,1000.00,," both ",Ætna,2010-04-01 00:00:00,05/02/2024,Y +0012,0000000012,£.00,¬£.00," 7",7.00,a b,none,Müller £,2024-04-01,25/12/2024,FALSE diff --git a/tests/data/synthetic/fixture-utf8-mixed-eol.csv b/tests/data/synthetic/fixture-utf8-mixed-eol.csv index 646a9cb..4e8618c 100644 --- a/tests/data/synthetic/fixture-utf8-mixed-eol.csv +++ b/tests/data/synthetic/fixture-utf8-mixed-eol.csv @@ -1,5 +1,5 @@ ref_text,ref_padded,amount,amount_mojibake,number_as_text,number,blank_like,padded_text,accented,date_text,date_mixed,boolean_like 0000130842,0000130842,£43.40,¬£43.40,12.50,12.50," "," lead",Café,13/04/2024,04/01/2024,TRUE 0000000001,0000000001,£-826.80,¬£-826.80,0.10,0.10," ","trail ",naïve,04/01/2024,13/04/2024,false -1234567890,1234567890,"£1,234.50","¬£1,234.50",1e3,1000.00,," both ",Ætna,2010-04-01 00:00:00,05/02/2024,Y -0012,0000000012,£.00,¬£.00," 7",7.00,a b,none,Müller £,2024-04-01,25/12/2024,FALSE +1234567890,1234567890,"£1,234.50","¬£1,234.50",1e3,1000.00,," both ",Ætna,2010-04-01 00:00:00,05/02/2024,Y +0012,0000000012,£.00,¬£.00," 7",7.00,a b,none,Müller £,2024-04-01,25/12/2024,FALSE diff --git a/tests/data/synthetic/fixture.xlsx b/tests/data/synthetic/fixture.xlsx index 2d9912e..5f10614 100644 Binary files a/tests/data/synthetic/fixture.xlsx and b/tests/data/synthetic/fixture.xlsx differ diff --git a/tests/data/test_world_bank_header.crosswalk b/tests/data/test_world_bank_header.crosswalk new file mode 100644 index 0000000..1db8ab3 --- /dev/null +++ b/tests/data/test_world_bank_header.crosswalk @@ -0,0 +1 @@ +{"uuid":"01a186c0-c752-7ced-8688-70a7f1be4339","name":"test_world_bank_header","schemaSource":{"uuid":"01a186c0-c74c-77b0-849f-e0e180091b88","name":"greasy-puddle","fields":[{"uuid":"01a186c0-c74c-7cf3-b047-c933585786b6","name":"Country Name","title":"Country Name","type":"string"},{"uuid":"01a186c0-c74c-77e1-8155-fc59a593facd","name":"Country Code","title":"Country Code","type":"string"},{"uuid":"01a186c0-c74c-70e9-b80a-5b2e874102a1","name":"Indicator Name","title":"Indicator Name","type":"string"},{"uuid":"01a186c0-c74c-767e-b60e-6ce0d7000dde","name":"Indicator Code","title":"Indicator Code","type":"string"},{"uuid":"01a186c0-c74c-7748-af31-9c4789f1a9b7","name":"1960","title":"1960","type":"string"},{"uuid":"01a186c0-c74c-7d41-a50c-ca1d30dfb7ca","name":"1961","title":"1961","type":"string"},{"uuid":"01a186c0-c74c-71bf-bd49-b51156714e84","name":"1962","title":"1962","type":"string"},{"uuid":"01a186c0-c74c-79fe-99dd-284c66ed68fc","name":"1963","title":"1963","type":"string"},{"uuid":"01a186c0-c74c-774a-b8f5-68fba242e150","name":"1964","title":"1964","type":"string"},{"uuid":"01a186c0-c74c-71a1-9353-bc1bfbe0945c","name":"1965","title":"1965","type":"string"},{"uuid":"01a186c0-c74c-7a1f-8d8b-1b592da368cf","name":"1966","title":"1966","type":"string"},{"uuid":"01a186c0-c74c-7dbb-9fd3-9eca279dd61c","name":"1967","title":"1967","type":"string"},{"uuid":"01a186c0-c74c-7c8e-aa7d-ecd4b65a71a3","name":"1968","title":"1968","type":"string"},{"uuid":"01a186c0-c74c-74ce-a534-2026a031224b","name":"1969","title":"1969","type":"string"},{"uuid":"01a186c0-c74c-7700-a565-1fe0a86833a3","name":"1970","title":"1970","type":"string"},{"uuid":"01a186c0-c74c-7586-a4b8-6aaf205cd396","name":"1971","title":"1971","type":"string"},{"uuid":"01a186c0-c74c-7abe-9044-2cf04271d079","name":"1972","title":"1972","type":"string"},{"uuid":"01a186c0-c74c-7bdf-a4df-60dc6df29612","name":"1973","title":"1973","type":"string"},{"uuid":"01a186c0-c74c-7ece-a439-fa2ce1efd45d","name":"1974","title":"1974","type":"string"},{"uuid":"01a186c0-c74c-7fe1-8346-84ddb8485595","name":"1975","title":"1975","type":"string"},{"uuid":"01a186c0-c74c-730b-bda5-948420652701","name":"1976","title":"1976","type":"string"},{"uuid":"01a186c0-c74c-704b-9507-608207bfdfe9","name":"1977","title":"1977","type":"string"},{"uuid":"01a186c0-c74c-7520-b2bb-e36f4f689ff5","name":"1978","title":"1978","type":"string"},{"uuid":"01a186c0-c74c-709b-a86a-cca193f7434d","name":"1979","title":"1979","type":"string"},{"uuid":"01a186c0-c74c-7949-91a5-3e40e947eece","name":"1980","title":"1980","type":"string"},{"uuid":"01a186c0-c74c-7675-b637-8fcf90266030","name":"1981","title":"1981","type":"string"},{"uuid":"01a186c0-c74c-75de-a618-6a68318347c0","name":"1982","title":"1982","type":"string"},{"uuid":"01a186c0-c74c-79c3-980a-3de31cca5e23","name":"1983","title":"1983","type":"string"},{"uuid":"01a186c0-c74c-7dbb-8966-87c94d321b4b","name":"1984","title":"1984","type":"string"},{"uuid":"01a186c0-c74d-7084-94d9-bfee1665d46f","name":"1985","title":"1985","type":"string"},{"uuid":"01a186c0-c74d-7dad-875f-36b0b9998c35","name":"1986","title":"1986","type":"string"},{"uuid":"01a186c0-c74d-78cc-bb44-18a4080cb66a","name":"1987","title":"1987","type":"string"},{"uuid":"01a186c0-c74d-7362-a126-4fab2bdea89d","name":"1988","title":"1988","type":"string"},{"uuid":"01a186c0-c74d-78a4-be01-7935a382ba63","name":"1989","title":"1989","type":"string"},{"uuid":"01a186c0-c74d-7009-929e-463df324dfbd","name":"1990","title":"1990","type":"string"},{"uuid":"01a186c0-c74d-703f-837b-c596d6c1e4b0","name":"1991","title":"1991","type":"string"},{"uuid":"01a186c0-c74d-7ff0-91a6-fd50ff225d1b","name":"1992","title":"1992","type":"string"},{"uuid":"01a186c0-c74d-7ef5-9bfc-690d3ff76e39","name":"1993","title":"1993","type":"string"},{"uuid":"01a186c0-c74d-7d36-99ee-7a30ae9271ed","name":"1994","title":"1994","type":"string"},{"uuid":"01a186c0-c74d-766e-801e-1cb71bd5109c","name":"1995","title":"1995","type":"string"},{"uuid":"01a186c0-c74d-71cf-82db-f3300ad3ae86","name":"1996","title":"1996","type":"string"},{"uuid":"01a186c0-c74d-7ebe-ba57-a5b120c61f90","name":"1997","title":"1997","type":"string"},{"uuid":"01a186c0-c74d-7a9b-beac-c42b25a33dc2","name":"1998","title":"1998","type":"string"},{"uuid":"01a186c0-c74d-77c8-8f63-6789bc6d2290","name":"1999","title":"1999","type":"string"},{"uuid":"01a186c0-c74d-718a-8179-293185e48dac","name":"2000","title":"2000","type":"string"},{"uuid":"01a186c0-c74d-7b81-8b64-1d6a6d697873","name":"2001","title":"2001","type":"string"},{"uuid":"01a186c0-c74d-77ee-b22d-4924b5ca4c54","name":"2002","title":"2002","type":"string"},{"uuid":"01a186c0-c74d-761e-a45d-102e930cccbb","name":"2003","title":"2003","type":"string"},{"uuid":"01a186c0-c74d-7fd9-8002-ef642ae95040","name":"2004","title":"2004","type":"string"},{"uuid":"01a186c0-c74d-7131-b9bf-78ffc8c7fdce","name":"2005","title":"2005","type":"string"},{"uuid":"01a186c0-c74d-7f9f-bf7f-0fcfc103f1de","name":"2006","title":"2006","type":"string"},{"uuid":"01a186c0-c74d-7b4b-8d68-2834d00b6d47","name":"2007","title":"2007","type":"string"},{"uuid":"01a186c0-c74d-7842-9f9a-4d5097f58580","name":"2008","title":"2008","type":"string"},{"uuid":"01a186c0-c74d-72d9-8fed-531bc2e165db","name":"2009","title":"2009","type":"string"},{"uuid":"01a186c0-c74d-706c-9c06-8aa452becc71","name":"2010","title":"2010","type":"string"},{"uuid":"01a186c0-c74d-7c9e-b2eb-e74b19afe3fe","name":"2011","title":"2011","type":"string"},{"uuid":"01a186c0-c74d-711b-ba92-080f50c92738","name":"2012","title":"2012","type":"string"},{"uuid":"01a186c0-c74d-77a4-b932-ee5542fec37a","name":"2013","title":"2013","type":"string"},{"uuid":"01a186c0-c74d-7db3-8759-ed25ef796ba7","name":"2014","title":"2014","type":"string"},{"uuid":"01a186c0-c74d-76e9-9dc4-9d34915ab7d5","name":"2015","title":"2015","type":"string"},{"uuid":"01a186c0-c74d-75e6-af1c-761bc20524d7","name":"2016","title":"2016","type":"string"},{"uuid":"01a186c0-c74e-7e5f-9335-612a8de9a9b6","name":"2017","title":"2017","type":"string"},{"uuid":"01a186c0-c74e-7059-add9-2e419761c16a","name":"2018","title":"2018","type":"string"},{"uuid":"01a186c0-c74e-75da-8531-387d1965480e","name":"2019","title":"2019","type":"string"}],"index":268},"schemaDestination":{"uuid":"01a186c0-c750-7ba0-904c-0e59daae8439","name":"urban_population","title":"Urban population","description":"Urban population refers to people living in urban areas as defined by national statistical offices. It is calculated using World Bank population estimates and urban ratios from the United Nations World Urbanization Prospects. Aggregation of urban and rural population may not add up to total population because of different country coverages.","fields":[{"uuid":"01a186c0-c750-72e7-9ab9-c32e84816b48","name":"indicator_code","title":"Indicator Code","description":"World Bank code reference for Indicator Name.","type":"string","constraints":{"required":true}},{"uuid":"01a186c0-c750-7e21-ae95-6b7f84f347d3","name":"country_name","title":"Country Name","description":"Official country names.","type":"string","constraints":{"required":true}},{"uuid":"01a186c0-c750-7502-be61-24e866cccc5e","name":"country_code","title":"Country Code","description":"UN ISO 3-letter country code.","type":"string","constraints":{"required":true}},{"uuid":"01a186c0-c750-7c60-88f6-5eeb3fb92327","name":"indicator_name","title":"Indicator Name","description":"Indicator described in the data series.","type":"string","constraints":{"required":true}},{"uuid":"01a186c0-c750-7a1a-b33d-5560c9213d80","name":"year","title":"Year","description":"Year of release.","type":"year","constraints":{"required":true}},{"uuid":"01a186c0-c750-78ec-beb0-3b06c14bc14d","name":"values","title":"Values","description":"Value for the Year and Indicator Name.","type":"number","constraints":{"required":true}}]},"actions":[{"uuid":"01a186c0-c754-7dac-b401-ef046d5e0a4e","script":"DEBLANK"},{"uuid":"01a186c0-c754-710e-9414-bbe81d02c601","script":"DEDUPE"},{"uuid":"01a186c0-c754-7f24-9cfd-961952d31d5e","script":"DELETE_ROWS < [0, 1, 2, 3]"},{"uuid":"01a186c0-c755-76b5-b399-c80461ae6100","script":"PIVOT_LONGER > ['year', 'values'] < ['1960', '1961', '1962', '1963', '1964', '1965', '1966', '1967', '1968', '1969', '1970', '1971', '1972', '1973', '1974', '1975', '1976', '1977', '1978', '1979', '1980', '1981', '1982', '1983', '1984', '1985', '1986', '1987', '1988', '1989', '1990', '1991', '1992', '1993', '1994', '1995', '1996', '1997', '1998', '1999', '2000', '2001', '2002', '2003', '2004', '2005', '2006', '2007', '2008', '2009', '2010', '2011', '2012', '2013', '2014', '2015', '2016', '2017', '2018', '2019']"},{"uuid":"01a186c0-c755-7665-87db-397a756b973c","script":"RENAME > 'indicator_code' < ['Indicator Code']"},{"uuid":"01a186c0-c755-7acc-8f04-542b5c8a9038","script":"RENAME > 'indicator_name' < ['Indicator Name']"},{"uuid":"01a186c0-c755-7b3f-88fd-99b741b7f714","script":"RENAME > 'country_code' < ['Country Code']"},{"uuid":"01a186c0-c755-7e41-9326-bebbb532ac0a","script":"RENAME > 'country_name' < ['Country Name']"}],"version":[{"description":"Save crosswalk.","updated":"2026-10-29T08:22:23.319789"}]} \ No newline at end of file diff --git a/tests/data/test_world_bank_header_output.parquet b/tests/data/test_world_bank_header_output.parquet new file mode 100644 index 0000000..872a0da Binary files /dev/null and b/tests/data/test_world_bank_header_output.parquet differ diff --git a/tests/data/test_world_bank_header_source.data b/tests/data/test_world_bank_header_source.data new file mode 100644 index 0000000..992d23c --- /dev/null +++ b/tests/data/test_world_bank_header_source.data @@ -0,0 +1 @@ +{"uuid":"01a186ba-c61d-79ad-95bf-11d015c0a29a","path":"tests/data/working_test_world_bank_data.xlsx","mime":"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet","header":3,"sheet_name":"Data","names":["Country Name","Country Code","Indicator Name","Indicator Code","1960","1961","1962","1963","1964","1965","1966","1967","1968","1969","1970","1971","1972","1973","1974","1975","1976","1977","1978","1979","1980","1981","1982","1983","1984","1985","1986","1987","1988","1989","1990","1991","1992","1993","1994","1995","1996","1997","1998","1999","2000","2001","2002","2003","2004","2005","2006","2007","2008","2009","2010","2011","2012","2013","2014","2015","2016","2017","2018","2019"],"columns":[{"uuid":"01a186ba-c5f9-727c-8bfe-896da7ae37f8","name":"Country Name"},{"uuid":"01a186ba-c5f9-7fed-807a-b297ee144e69","name":"Country Code"},{"uuid":"01a186ba-c5f9-7878-b229-e3cc2dcf7898","name":"Indicator Name"},{"uuid":"01a186ba-c5f9-7062-8200-4aa2cb7ad079","name":"Indicator Code"},{"uuid":"01a186ba-c5f9-7580-a42d-3a230ee21574","name":"1960"},{"uuid":"01a186ba-c5f9-7afc-955c-1fdeab1e24da","name":"1961"},{"uuid":"01a186ba-c5f9-7979-84b7-ac6a55488fb2","name":"1962"},{"uuid":"01a186ba-c5f9-7c5d-97ae-eb9f12cd08db","name":"1963"},{"uuid":"01a186ba-c5f9-7b1e-a83f-1c89ba6555c8","name":"1964"},{"uuid":"01a186ba-c5f9-78a9-ab68-336df9985f6a","name":"1965"},{"uuid":"01a186ba-c5f9-7110-bbdc-f20afd481aba","name":"1966"},{"uuid":"01a186ba-c5f9-7965-8cfb-6b3ce91d7ef0","name":"1967"},{"uuid":"01a186ba-c5f9-7c41-8793-532c36e97544","name":"1968"},{"uuid":"01a186ba-c5f9-73bf-b491-5bc7385c00f6","name":"1969"},{"uuid":"01a186ba-c5f9-73ee-9619-a9c562d5e2cb","name":"1970"},{"uuid":"01a186ba-c5f9-7039-870d-3395b5994573","name":"1971"},{"uuid":"01a186ba-c5f9-7be5-a1f8-5ecc75b26cd6","name":"1972"},{"uuid":"01a186ba-c5f9-7e29-b261-7334b229ab45","name":"1973"},{"uuid":"01a186ba-c5f9-7ec3-9db1-f01cd9c1b8fd","name":"1974"},{"uuid":"01a186ba-c5f9-7a43-af93-595b2f84005b","name":"1975"},{"uuid":"01a186ba-c5f9-7af6-96c2-62591b01eb51","name":"1976"},{"uuid":"01a186ba-c5f9-7148-afbf-4ca3a5fa7e8f","name":"1977"},{"uuid":"01a186ba-c5f9-7009-9715-e14376a6c53c","name":"1978"},{"uuid":"01a186ba-c5f9-7c33-9eaa-45151edbe480","name":"1979"},{"uuid":"01a186ba-c5f9-7f1a-b9f6-d5c9abc81145","name":"1980"},{"uuid":"01a186ba-c5f9-7957-a6ff-41e076fc091f","name":"1981"},{"uuid":"01a186ba-c5f9-72a7-888c-e22a349f6f36","name":"1982"},{"uuid":"01a186ba-c5f9-7435-9b11-052c75ed4305","name":"1983"},{"uuid":"01a186ba-c5f9-7508-a571-5cb292069ed1","name":"1984"},{"uuid":"01a186ba-c5f9-77f4-9fb0-0ffed9679410","name":"1985"},{"uuid":"01a186ba-c5f9-7fff-bd98-5665e90373bb","name":"1986"},{"uuid":"01a186ba-c5f9-7769-a6c5-1eeb1892b87b","name":"1987"},{"uuid":"01a186ba-c5f9-7f24-b087-0ad27b20e40c","name":"1988"},{"uuid":"01a186ba-c5f9-7c62-b762-c7a47e3280c3","name":"1989"},{"uuid":"01a186ba-c5f9-702a-97be-e1bc6ceb30f1","name":"1990"},{"uuid":"01a186ba-c5f9-7f62-8996-55dcf8459145","name":"1991"},{"uuid":"01a186ba-c5f9-7916-83c4-d4a405312ea9","name":"1992"},{"uuid":"01a186ba-c5f9-723c-8de3-0b465929c295","name":"1993"},{"uuid":"01a186ba-c5f9-748a-b142-83a902b8223a","name":"1994"},{"uuid":"01a186ba-c5f9-79ea-86eb-a9ee73071c0f","name":"1995"},{"uuid":"01a186ba-c5f9-7a19-bd0a-409a188cada4","name":"1996"},{"uuid":"01a186ba-c5f9-7e90-9d21-fe403d75f27a","name":"1997"},{"uuid":"01a186ba-c5f9-7942-8986-d41c23ed8230","name":"1998"},{"uuid":"01a186ba-c5f9-76b4-908a-f005ab9beb4d","name":"1999"},{"uuid":"01a186ba-c5f9-7d2b-bd07-279b985b823e","name":"2000"},{"uuid":"01a186ba-c5f9-720c-b920-d8be9cd0b730","name":"2001"},{"uuid":"01a186ba-c5f9-765c-a6c9-73f79e59f520","name":"2002"},{"uuid":"01a186ba-c5f9-77dd-a3a9-4ce4d00edcc1","name":"2003"},{"uuid":"01a186ba-c5f9-7b23-a43f-8335ceee9ae4","name":"2004"},{"uuid":"01a186ba-c5f9-749f-aec2-4e36ad0ea95a","name":"2005"},{"uuid":"01a186ba-c5f9-72e7-bf68-03d091e7167e","name":"2006"},{"uuid":"01a186ba-c5f9-71c2-bf97-0c9059e4120b","name":"2007"},{"uuid":"01a186ba-c5f9-72f3-bcbb-649040eb15e4","name":"2008"},{"uuid":"01a186ba-c5f9-75b9-9843-a4f465852f94","name":"2009"},{"uuid":"01a186ba-c5f9-7a0f-a743-4ddd035e4ec3","name":"2010"},{"uuid":"01a186ba-c5f9-7b38-82d3-dd0f6de88ef2","name":"2011"},{"uuid":"01a186ba-c5f9-7778-a054-de1d6cd45e79","name":"2012"},{"uuid":"01a186ba-c5f9-763b-ad1a-713cb099926f","name":"2013"},{"uuid":"01a186ba-c5f9-71a5-a8e6-7fc401657837","name":"2014"},{"uuid":"01a186ba-c5f9-7320-a5a8-7a102e13b74c","name":"2015"},{"uuid":"01a186ba-c5f9-7cb0-a533-dac07ff9243a","name":"2016"},{"uuid":"01a186ba-c5f9-7de2-b0d7-342776bc2294","name":"2017"},{"uuid":"01a186ba-c5f9-7e45-b2f0-dcaf417dbad0","name":"2018"},{"uuid":"01a186ba-c5f9-7cb1-8dcb-a6e19d91186d","name":"2019"}],"preserve":["Country Name","Country Code","Indicator Name","Indicator Code","1960","1961","1962","1963","1964","1965","1966","1967","1968","1969","1970","1971","1972","1973","1974","1975","1976","1977","1978","1979","1980","1981","1982","1983","1984","1985","1986","1987","1988","1989","1990","1991","1992","1993","1994","1995","1996","1997","1998","1999","2000","2001","2002","2003","2004","2005","2006","2007","2008","2009","2010","2011","2012","2013","2014","2015","2016","2017","2018","2019"],"attributes":{},"index":268,"checksum":"d715e986ff3f157e1af380d8ca28f69e5e0fb2a88b5d1c7e248ca167f38bdb121f31c9e12b58b6f5bb32974cfc322c791ddd710ed364a074c78031118f5a562f"} \ No newline at end of file diff --git a/tests/test_column_names.py b/tests/test_column_names.py new file mode 100644 index 0000000..41a6651 --- /dev/null +++ b/tests/test_column_names.py @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: BSD-3-Clause +"""Column names a script can name: every name the read gives resolves, in any script, to its own field. + +The script parser never sees the file. It resolves each quoted term against the schemas, so the read must give names +that are exact (nothing trimmed or joined), unique, and clear of the row key, and the parser must resolve any such name. +""" + +import json +from pathlib import Path + +import pytest + +import whyqd as qd +from whyqd.parsers import DataSourceParser, RowKeyParser + +LAYOUTS = Path(__file__).resolve().parent / "data" / "synthetic-layouts" +HOSTILE = json.loads((LAYOUTS / "answer-key.json").read_text(encoding="utf-8"))["csv"]["hostile-headers.csv"] +NAMES = HOSTILE["names"] +# Read as nesting, or split at the comma, by the morph parser, which does not hex a script before splitting it +MORPH_FAULTS = {"Rate [2026]", "Code, Local"} + + +def crosswalk() -> qd.CrosswalkDefinition: + source = qd.SchemaDefinition() + source.set(schema={"name": "hostile"}) + source.fields.add_multi( + terms=[{"name": n, "type": "string", "constraints": {"category": [{"name": "v00"}]}} for n in NAMES] + ) + destination = qd.SchemaDefinition() + destination.set(schema={"name": "destination"}) + destination.fields.add_multi( + terms=[ + {"name": "target", "type": "string"}, + {"name": "other", "type": "string"}, + {"name": "flag", "type": "boolean"}, + ] + ) + definition = qd.CrosswalkDefinition() + definition.set(crosswalk={"name": "hostile"}, schema_source=source, schema_destination=destination) + return definition + + +def first_field(source): + while isinstance(source, list): + source = next(s for s in source if s) + return source + + +def scripts(name: str) -> list[pytest.param]: + term = f"'{name}'" + morph = ( + pytest.mark.xfail(strict=True, raises=ValueError, reason="the morph parser splits names") + if name in MORPH_FAULTS + else () + ) + return [ + pytest.param(name, f"RENAME > 'target' < [{term}]", id=f"rename-{name!r}"), + pytest.param(name, f"UNITE > 'target' < ', '::[{term}, 'Rate Relief']", marks=morph, id=f"unite-{name!r}"), + pytest.param(name, f"SEPARATE > ['target', 'other'] < ';;'::[{term}]", marks=morph, id=f"separate-{name!r}"), + pytest.param(name, f"CATEGORISE > 'flag'::True < {term}::['v00']", id=f"categorise-{name!r}"), + pytest.param(name, f"PIVOT_CATEGORIES > 'target' < {term}::[0]", id=f"pivot-categories-{name!r}"), + ] + + +class TestColumnNames: + @pytest.mark.parametrize("name, script", [p for n in NAMES for p in scripts(n)]) + def test_every_name_resolves_to_its_field(self, name, script): + definition = crosswalk() + parsed = definition.actions.parse(script=script) + assert first_field(parsed["source"]).uuid == definition.schema_source.fields.get(name=name).uuid + + def test_names_are_unique_and_clear_of_the_row_key(self): + row_key = RowKeyParser() + assert len(set(NAMES)) == len(NAMES) + assert not any(row_key.is_part(n) for n in NAMES) + + def test_read_gives_every_name_exactly(self): + datasource = qd.DataSourceDefinition() + datasource.derive_model(source=LAYOUTS / "hostile-headers.csv", mimetype="CSV") + assert datasource.get.dialect is not None + assert [c.name for c in datasource.get.columns] == NAMES + df = DataSourceParser().get(source=datasource.get) + assert [c for c in df.columns if c != RowKeyParser.ROW] == NAMES + assert df[RowKeyParser.ROW].tolist() == list(range(len(HOSTILE["rows"]))) + assert df[NAMES].values.tolist() == HOSTILE["rows"] diff --git a/tests/test_columnar.py b/tests/test_columnar.py new file mode 100644 index 0000000..fdc7ee4 --- /dev/null +++ b/tests/test_columnar.py @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: BSD-3-Clause +"""Parquet and Feather read to the same grid as every other format: each typed value in whyqd's own text. + +A columnar file stores types, not text, so each value is rendered as `CellParser.render` gives it: a whole number +without `.0` (a column of integers with a missing value included), a date as its ISO date, a boolean as `TRUE` or +`FALSE`. The same table written as a CSV reads the same. +""" + +from datetime import date, datetime +from pathlib import Path + +import pandas as pd +import pytest + +import whyqd as qd +from whyqd.parsers import DataSourceParser, RowKeyParser + +ROW = RowKeyParser.ROW +NAMES = ["Reference", "UPRN", "Amount", "Paid", "Start Date", "Updated", "Code"] +ROWS = [ + ["769705120", "37019199", "12.5", "TRUE", "2024-04-01", "2024-04-01T09:30:00", "0012"], + ["918400640", None, "1000", "FALSE", "2024-12-25", "2024-12-25", None], +] + + +@pytest.fixture +def frame() -> pd.DataFrame: + return pd.DataFrame( + { + "Reference": pd.Series([769705120, 918400640], dtype="int64"), + "UPRN": pd.Series([37019199, None], dtype="Int64"), + "Amount": [12.5, 1000.0], + "Paid": [True, False], + "Start Date": [date(2024, 4, 1), date(2024, 12, 25)], + "Updated": [datetime(2024, 4, 1, 9, 30), datetime(2024, 12, 25)], + "Code": ["0012", None], + } + ) + + +def read(source: Path, mimetype: str) -> tuple[qd.models.DataSourceModel, pd.DataFrame]: + datasource = qd.DataSourceDefinition() + datasource.derive_model(source=source, mimetype=mimetype) + df = DataSourceParser().get(source=datasource.get) + assert df[ROW].tolist() == list(range(len(df))) + return datasource.get, df + + +def rows(df: pd.DataFrame, names: list[str]) -> list[list[str | None]]: + return [[None if pd.isna(c) else c for c in row] for row in df[names].values.tolist()] + + +class TestColumnar: + @pytest.mark.parametrize("mimetype", ["PARQUET", "FEATHER"]) + def test_each_value_reads_as_whyqds_text(self, tmp_path, frame, mimetype): + source = tmp_path / f"table.{mimetype.lower()}" + frame.to_parquet(source) if mimetype == "PARQUET" else frame.to_feather(source) + model, df = read(source, mimetype) + assert [c.name for c in model.columns] == NAMES + assert rows(df, NAMES) == ROWS + + def test_a_parquet_file_reads_as_its_csv_does(self, tmp_path, frame): + frame.to_parquet(tmp_path / "table.parquet") + (tmp_path / "table.csv").write_text( + ",".join(NAMES) + "\n" + "".join(",".join(c or "" for c in r) + "\n" for r in ROWS), encoding="utf-8" + ) + _, parquet = read(tmp_path / "table.parquet", "PARQUET") + _, text = read(tmp_path / "table.csv", "CSV") + assert rows(parquet, NAMES) == rows(text, NAMES) + + def test_a_column_named_for_the_row_key_is_renamed(self, tmp_path): + source = tmp_path / "keyed.parquet" + pd.DataFrame({ROW: ["a", "b"], "x": ["1", "2"]}).to_parquet(source) + model, df = read(source, "PARQUET") + assert [c.name for c in model.columns] == [f"{ROW}.1", "x"] + assert df[f"{ROW}.1"].tolist() == ["a", "b"] diff --git a/tests/test_datasource.py b/tests/test_datasource.py index b8b4add..2724f70 100644 --- a/tests/test_datasource.py +++ b/tests/test_datasource.py @@ -1,14 +1,13 @@ # SPDX-License-Identifier: BSD-3-Clause from pathlib import Path -import csv import pandas as pd +import pytest from pydantic import AnyUrl import whyqd as qd from whyqd.parsers import CoreParser, DataSourceParser DIRECTORY = Path(__file__).resolve().parent / "data" -BASE_DIRECTORY = f"{Path('__file__').resolve().parent}/" DATA = { "SINGLE": DIRECTORY / "raw_E06000044_014_0.XLSX", "MULTI": DIRECTORY / "raw_multi_E06000044_014.xlsx", @@ -37,16 +36,11 @@ def test_single(self, tmp_path): datasource.save(directory=TEST_DIRECTORY) datasource.validate() assert datasource.get.get_checksum("bytes") - # Consistency - d = datasource.exclude_uuid(model=datasource.get) - d["checksums"] = [c for c in d["checksums"] if c["content"] == "table-1x"] - D = datasource.exclude_uuid(model=MODEL["SINGLE"]) - # Using relative paths for test, so adjust accordingly - d["path"] = d["path"].replace(BASE_DIRECTORY, "") - D["path"] = D["path"].replace(BASE_DIRECTORY, "") - # the MODEL doesn't have AnyURL set in the citation - D["citation"]["url"] = AnyUrl(D["citation"]["url"]) - assert d == D + # The read finds the columns and rows 1.x found; its types and checksum are 2.0's, every cell text + stored = qd.DataSourceDefinition(source=MODEL["SINGLE"]).get + assert [c.name for c in datasource.get.columns] == [c.name for c in stored.columns] + assert datasource.get.index == stored.index + assert datasource.get.citation.url == AnyUrl(MODEL["SINGLE"]["citation"]["url"]) def test_load_single(self): datasource = qd.DataSourceDefinition(source=MODEL["SINGLE"]) @@ -71,13 +65,40 @@ def test_multi(self): def test_cthulhu(self): # Create datasource = qd.DataSourceDefinition() - datasource.derive_model(source=DATA["CTHULHU"], mimetype=MIMETYPE, header=None) + datasource.derive_model(source=DATA["CTHULHU"], mimetype=MIMETYPE, dialect={"headerRowCount": 0}) datasource.validate() - def test_attribute_variations(self): + def test_csv_verifies_by_its_bytes_else_its_cells(self, tmp_path): datasource = qd.DataSourceDefinition() - datasource.derive_model(source=DATA["CSV"], mimetype=CSVTYPE, quoting=csv.QUOTE_ALL) - datasource.validate() + datasource.derive_model(source=DATA["CSV"], mimetype=CSVTYPE) + assert datasource.get.get_checksum("table-1x") is None + assert datasource.validate() + # A trailing blank line changes the bytes, not a cell + resaved = tmp_path / "resaved.csv" + resaved.write_bytes(DATA["CSV"].read_bytes() + b"\n") + datasource.get.path = str(resaved) + with pytest.warns(UserWarning, match="saved again"): + assert datasource.validate() + changed = tmp_path / "changed.csv" + changed.write_bytes(DATA["CSV"].read_bytes() + b"x\n") + datasource.get.path = str(changed) + with pytest.raises(AssertionError): + datasource.validate() + + def test_get_data_is_the_table_the_derive_read(self, tmp_path): + source = tmp_path / "source.csv" + source.write_bytes(DATA["CSV"].read_bytes()) + datasource = qd.DataSourceDefinition() + datasource.derive_model(source=source, mimetype=CSVTYPE) + expected = DataSourceParser().get(source=datasource.get) + # Gone, so the data can only be the derive's + source.unlink() + pd.testing.assert_frame_equal(datasource.get_data(), expected) + # A second derive replaces it + other = tmp_path / "other.csv" + other.write_text("Code\n1\n", encoding="utf-8") + datasource.derive_model(source=other, mimetype=CSVTYPE) + assert datasource.get_data()["Code"].tolist() == ["1"] def test_coerce_keeps_rows_on_any_index(self): # Actions can leave a frame indexed by labels rather than 0..n; coercion must keep every value in its row @@ -93,3 +114,13 @@ def test_coerce_keeps_rows_on_any_index(self): df = DataSourceParser().coerce_to_schema(df=df, schema=schema) assert df["amount"].tolist() == [80.0, 88.5] assert df["reliefs"].tolist() == [["80.0", None], ["88.5", "12.0"]] + + def test_a_frames_columns_are_typed_as_1x_typed_them(self): + # A date column is recorded as `string`, as 1.x recorded it; typing records dates + df = pd.DataFrame({"a": [1.5], "b": pd.array([1], dtype="Int64"), "c": pd.to_datetime(["2024-01-04"]), "d": ["x"]}) + columns = DataSourceParser().get_header_columns(df=df) + assert [c.dtype for c in columns] == ["number", "number", "string", "string"] + + def test_a_column_takes_whyqds_types_only(self): + with pytest.raises(ValueError): + qd.models.ColumnModel(name="a", type="float64") diff --git a/tests/test_definition_slots.py b/tests/test_definition_slots.py index c9d6290..b52ff1f 100644 --- a/tests/test_definition_slots.py +++ b/tests/test_definition_slots.py @@ -6,6 +6,7 @@ import pytest import whyqd as qd +from whyqd.parsers import FormatParser DIRECTORY = Path(__file__).resolve().parent / "data" DATA_SOURCE = DIRECTORY / "test_basildon_source.data" @@ -13,7 +14,7 @@ DIALECT = { "encoding": "windows-1252", - "encodingStep": "detected", + "encodingStep": "declared", "delimiter": ";", "quoteChar": None, "doubleQuote": False, @@ -73,8 +74,10 @@ def test_profile_round_trips(self): assert round_trip(qd.DataSourceDefinition, model)["columns"][0]["profile"] == PROFILE def test_absent_by_default(self): + # A 1.x data source reads by the dialect its 1.x settings give, and gains nothing else written = round_trip(qd.DataSourceDefinition, data_source()) - assert "dialect" not in written + stored = json.loads(DATA_SOURCE.read_text(encoding="utf-8")) + assert written["dialect"] == FormatParser().dialect_1x(stored) assert all("profile" not in c for c in written["columns"]) @pytest.mark.parametrize("parts", [{"dialect": DIALECT}, {"profile": PROFILE}], ids=["dialect", "profile"]) diff --git a/tests/test_encoding.py b/tests/test_encoding.py new file mode 100644 index 0000000..4d43eb2 --- /dev/null +++ b/tests/test_encoding.py @@ -0,0 +1,125 @@ +# SPDX-License-Identifier: BSD-3-Clause +"""A CSV's encoding, as the wrangler declares it or as the ladder settles it, recorded so the read repeats. + +The ladder decodes as the WHATWG Encoding Standard does, which CSVW's dialect names its encodings by: a byte-order mark, +over any declaration; otherwise the declared encoding; otherwise strict UTF-8, then `windows-1252`, which in the standard +decodes any byte and loses none. Nothing is detected: a file the ladder reads as `windows-1252` that is in fact another +encoding loads with every byte kept, and reads to its text once the wrangler declares the encoding. +""" + +import csv +import io +import json +from pathlib import Path + +import pandas as pd +import pytest + +import whyqd as qd +from whyqd.models import DialectModel +from whyqd.parsers import DataSourceParser, RowKeyParser + +SYNTHETIC = Path(__file__).resolve().parent / "data" / "synthetic" +KEY = json.loads((SYNTHETIC / "answer-key.json").read_text(encoding="utf-8"))["csv"] +ROW = RowKeyParser.ROW +# The bytes Python's `cp1252` leaves undefined, which the standard decodes to the code point of their value +UNDEFINED = {0x81, 0x8D, 0x8F, 0x90, 0x9D} + + +def read(name: str, dialect: dict | None = None) -> tuple[qd.models.DataSourceModel, pd.DataFrame]: + datasource = qd.DataSourceDefinition() + settings = {} if dialect is None else {"dialect": dialect} + datasource.derive_model(source=SYNTHETIC / name, mimetype="CSV", **settings) + assert datasource.get.dialect is not None + return datasource.get, DataSourceParser().get(source=datasource.get) + + +def windows_1252(data: bytes) -> str: + return "".join(chr(b) if b in UNDEFINED else bytes([b]).decode("cp1252") for b in data) + + +def rows(df: pd.DataFrame, names: list[str]) -> list[list[str | None]]: + assert df[ROW].tolist() == list(range(len(df))) + return [[None if pd.isna(c) else c for c in row] for row in df[names].values.tolist()] + + +def cells(rows: list[list[str]]) -> list[list[str | None]]: + # The key writes an empty cell as the file does; the table holds it as missing, as `CellParser.render` gives it + return [[c if c != "" else None for c in row] for row in rows] + + +class TestLadder: + @pytest.mark.parametrize( + "name, step, encoding", + [ + ("fixture-utf8.csv", "utf-8", "utf-8"), + ("fixture-utf8-crlf.csv", "utf-8", "utf-8"), + ("fixture-utf8-mixed-eol.csv", "utf-8", "utf-8"), + ("fixture-utf8-bom.csv", "bom", "utf-8"), + ("fixture-utf16-bom.csv", "bom", "utf-16le"), + ], + ) + def test_a_file_reads_to_its_key_and_records_the_step(self, name, step, encoding): + model, df = read(name) + assert (model.dialect.encodingStep, model.dialect.encoding) == (step, encoding) + assert [c.name for c in model.columns] == KEY[name]["header"] + assert rows(df, KEY[name]["header"]) == cells(KEY[name]["rows"]) + + @pytest.mark.parametrize("name", ["fixture-windows-1252.csv", "fixture-latin-1.csv", "fixture-mac-roman.csv"]) + def test_bytes_not_utf8_are_read_as_windows_1252_and_every_byte_kept(self, name): + model, df = read(name) + assert (model.dialect.encodingStep, model.dialect.encoding) == ("windows-1252", "windows-1252") + header, *expected = list(csv.reader(io.StringIO(windows_1252((SYNTHETIC / name).read_bytes())))) + assert rows(df, header) == cells(expected) + + def test_text_is_kept_as_written(self): + # Zero-width characters, a byte-order mark inside text, control characters, NFC beside NFD, full-width digits + name = "fixture-unicode-utf8.csv" + model, df = read(name) + assert rows(df, KEY[name]["header"]) == cells(KEY[name]["rows"]) + + +class TestDeclared: + @pytest.mark.parametrize( + "name, encoding", + [ + ("fixture-mac-roman.csv", "macintosh"), + ("fixture-windows-1252.csv", "windows-1252"), + ("fixture-utf8.csv", "utf-8"), + ], + ) + def test_a_declared_encoding_reads_to_the_key(self, name, encoding): + model, df = read(name, {"encoding": encoding}) + assert (model.dialect.encodingStep, model.dialect.encoding) == ("declared", encoding) + assert rows(df, KEY[name]["header"]) == cells(KEY[name]["rows"]) + + def test_a_byte_order_mark_overrides_the_declaration(self): + name = "fixture-utf8-bom.csv" + model, df = read(name, {"encoding": "windows-1252"}) + assert (model.dialect.encodingStep, model.dialect.encoding) == ("bom", "utf-8") + assert rows(df, KEY[name]["header"]) == cells(KEY[name]["rows"]) + + def test_bytes_a_declared_encoding_cannot_decode_are_replaced_with_one_warning(self): + # CSVW reads with the standard's replacement error mode; the declaration still stands, and the source is unchanged + name = "fixture-windows-1252.csv" + text = (SYNTHETIC / name).read_bytes().decode("utf-8", errors="replace") + datasource = qd.DataSourceDefinition() + # Each read gives one summary, never a line per byte: the count, and the encoding to declare instead + for read_once in ( + lambda: datasource.derive_model(source=SYNTHETIC / name, mimetype="CSV", dialect={"encoding": "utf-8"}), + lambda: DataSourceParser().get(source=datasource.get), + ): + with pytest.warns(UserWarning) as record: + read_once() + messages = [str(w.message) for w in record if "utf-8" in str(w.message)] + assert len(messages) == 1 + assert str(text.count("�")) in messages[0] and "windows-1252" in messages[0] + model = datasource.get + assert (model.dialect.encodingStep, model.dialect.encoding) == ("declared", "utf-8") + header, *expected = list(csv.reader(io.StringIO(text))) + assert rows(DataSourceParser().get(source=model), header) == cells(expected) + + def test_detected_is_not_a_step(self): + # A detected encoding is a proposal, recorded as `declared` once the wrangler accepts it + with pytest.raises(ValueError): + DialectModel(encodingStep="detected") diff --git a/tests/test_excel.py b/tests/test_excel.py new file mode 100644 index 0000000..88d9b58 --- /dev/null +++ b/tests/test_excel.py @@ -0,0 +1,199 @@ +# SPDX-License-Identifier: BSD-3-Clause +"""A workbook read cell by cell: every sheet, hidden ones included, each cell its stored value in whyqd's own text. + +A cell is read as stored, never as displayed, since a displayed short date follows the machine's locale: the number 12 +stored under the format `0000000000`, which Excel shows as `0000000012`, reads as `12` (text stored as `0012` stays +`0012`); a date cell reads as its ISO date beside the text in the same column; an error cell as its code. Each sheet is +its own data source, with its own dialect and names. +""" + +import json +from datetime import date, datetime, time, timedelta +from pathlib import Path + +import openpyxl +import pandas as pd +import pytest + +import whyqd as qd +from whyqd.parsers import CellParser, DataSourceParser, RowKeyParser + +DATA = Path(__file__).resolve().parent / "data" +FIXTURE = DATA / "synthetic" / "fixture.xlsx" +CELLS = json.loads((DATA / "synthetic" / "answer-key.json").read_text(encoding="utf-8"))["workbook"] +LAYOUTS = DATA / "synthetic-layouts" +WORKBOOKS = json.loads((LAYOUTS / "answer-key.json").read_text(encoding="utf-8"))["workbook"] +LAYOUT = WORKBOOKS["layouts.xlsx"] +ROW = RowKeyParser.ROW + + +def derive( + source: Path, mimetype: str, dialect: list[dict] | None = None, names: dict[str, list[str]] | None = None +) -> list[qd.models.DataSourceModel]: + # A workbook's names are given per sheet, by its name, since each sheet is its own table + settings = {k: v for k, v in {"dialect": dialect, "names": names}.items() if v is not None} + datasource = qd.DataSourceDefinition() + datasource.derive_model(source=source, mimetype=mimetype, **settings) + models = datasource.get if isinstance(datasource.get, list) else [datasource.get] + assert all(m.dialect is not None for m in models) + return models + + +def sheet(models: list[qd.models.DataSourceModel], name: str) -> tuple[qd.models.DataSourceModel, pd.DataFrame]: + (model,) = [m for m in models if m.dialect.sheet == name] + df = DataSourceParser().get(source=model) + assert df[ROW].tolist() == list(range(len(df))) + return model, df + + +def rows(df: pd.DataFrame, names: list[str]) -> list[list[str | None]]: + return [[None if pd.isna(c) else c for c in row] for row in df[names].values.tolist()] + + +def stored(cell: dict) -> str | None: + """The key's stored value as the Python type a reader gives, in whyqd's text.""" + value = cell["stored"] + if cell["stored_type"] == "d": + value = datetime.fromisoformat(value) if " " in value else date.fromisoformat(value) + rendered = CellParser().render(value=value) + return None if rendered is None else rendered[1] + + +class TestCells: + def test_each_cell_reads_as_stored(self): + # The `Data` sheet's header is row 1 and its records rows 2 to 5; F6 is a formula saved without a value + cells = {k: v for k, v in CELLS["Data"]["cells"].items() if k[1:] in {"2", "3", "4", "5"}} + names = list(dict.fromkeys(v["column"] for v in cells.values())) + expected = [[stored(cells[f"{chr(65 + i)}{r}"]) for i in range(len(names))] for r in range(2, 6)] + model, df = sheet(derive(FIXTURE, "XLSX"), "Data") + assert [c.name for c in model.columns] == names + assert rows(df, names)[:4] == expected + + def test_error_cells_read_as_their_codes(self): + case = LAYOUT["Errors"] + model, df = sheet(derive(LAYOUTS / "layouts.xlsx", "XLSX"), "Errors") + assert rows(df, case["names"]) == case["rows"] + + +class TestSheets: + def test_every_sheet_is_read_hidden_and_empty_included(self): + # Today a sheet whose first two rows read empty is dropped, and the empty one with it + models = derive(FIXTURE, "XLSX") + assert [m.dialect.sheet for m in models] == ["Data", "Titled", "Hidden", "Empty"] + model, df = sheet(models, "Hidden") + assert [c.name for c in model.columns] == ["secret"] + assert rows(df, ["secret"]) == [["kept"]] + model, df = sheet(models, "Empty") + assert (model.columns, model.index) == ([], 0) + + @pytest.mark.parametrize("workbook", ["layouts.xlsx", "layouts.xls"]) + @pytest.mark.parametrize("name", ["Titled", "BlankColumns", "SubTables", "Errors", "Workings"]) + def test_each_sheet_reads_to_its_key_under_its_dialect(self, workbook, name): + if workbook == "layouts.xls" and name == "Workings": + pytest.skip("LibreOffice cached the formula's value in the .xls, so the cell reads 0") + dialect = [{"sheet": s, **LAYOUT[s].get("dialect", {})} for s in LAYOUT if s != "MergedHeader"] + mimetype = "XLS" if workbook.endswith(".xls") else "XLSX" + model, df = sheet(derive(LAYOUTS / workbook, mimetype, dialect), name) + assert [c.name for c in model.columns] == LAYOUT[name]["names"] + expected = [row.copy() for row in LAYOUT[name]["rows"]] + if workbook == "layouts.xls" and name == "Errors": + # LibreOffice saved the `#REF!` cell as `#NAME?` in the .xls, as xlrd also reads it + expected[5][3] = "#NAME?" + assert rows(df, LAYOUT[name]["names"]) == expected + + def test_the_only_sheet_with_a_blank_second_row_reads(self): + case = WORKBOOKS["single-sheet.xlsx"]["Reliefs"] + model, df = sheet(derive(LAYOUTS / "single-sheet.xlsx", "XLSX"), "Reliefs") + assert [c.name for c in model.columns] == case["names"] + assert rows(df, case["names"]) == case["rows"] + + def test_a_merged_header_takes_the_names_given_for_its_sheet(self): + # Another sheet keeps its own names, `Propref` among them, never deconflicted against these + case = LAYOUT["MergedHeader"] + names = ["Propref", "Address", "Postcode", "Relief Code", "R/V"] + dialect = [{"sheet": "MergedHeader", **case["dialect"]}, {"sheet": "Titled", **LAYOUT["Titled"]["dialect"]}] + models = derive(LAYOUTS / "layouts.xlsx", "XLSX", dialect, {"MergedHeader": names}) + model, df = sheet(models, "MergedHeader") + assert [c.name for c in model.columns] == names + assert rows(df, names) == case["rows"] + model, df = sheet(models, "Titled") + assert [c.name for c in model.columns] == LAYOUT["Titled"]["names"] + + def test_a_merged_header_without_names_is_refused(self): + with pytest.raises(ValueError, match=r"(?s)headerRowCount.*\bnames\b"): + derive(LAYOUTS / "layouts.xlsx", "XLSX", [{"sheet": "MergedHeader", "headerRowCount": 2}]) + + +@pytest.fixture +def references(tmp_path) -> Path: + """Reference cells as councils store them: Gosport's numbers beside text codes, Eastbourne's in scientific format + with their digits already lost, Thurrock's zeros lost by the publisher, and a UPRN column with blanks.""" + source = tmp_path / "references.xlsx" + wb = openpyxl.Workbook() + ws = wb.active + ws.append(["Reference", "UPRN"]) + for reference, uprn, fmt in [ + (769705120, 37019199, "General"), + ("N30", None, "General"), + (26620000000000, 100062000000, "0.00E+00"), + (38100171142560, None, "General"), + (20020, 10024142866, "General"), + ]: + ws.append([reference, uprn]) + for cell in ws[ws.max_row]: + cell.number_format = fmt + wb.save(source) + return source + + +def test_a_workbook_dated_from_1904_reads_its_own_dates(tmp_path): + # ECMA-376 § 18.17.4.1: the same stored number is a date four years and a day later under the 1904 system + source = tmp_path / "dated-1904.xlsx" + wb = openpyxl.Workbook() + wb.epoch = openpyxl.utils.datetime.CALENDAR_MAC_1904 + ws = wb.active + ws.append(["Start Date"]) + ws.append([date(2024, 4, 1)]) + ws["A2"].number_format = "dd/mm/yyyy" + wb.save(source) + datasource = qd.DataSourceDefinition() + datasource.derive_model(source=source, mimetype="XLSX") + model = datasource.get[0] if isinstance(datasource.get, list) else datasource.get + value = DataSourceParser().get(source=model)["Start Date"].iloc[0] + assert str(value).startswith("2024-04-01") + + +def test_a_duration_reads_as_iso_8601(tmp_path): + # A cell formatted `[h]:mm` stores a length of time; a time of day beside it reads as before + source = tmp_path / "durations.xlsx" + wb = openpyxl.Workbook() + ws = wb.active + ws.append(["Hours", "Opens"]) + ws.append([timedelta(hours=25, minutes=30), time(9, 30)]) + ws["A2"].number_format, ws["B2"].number_format = "[h]:mm", "hh:mm" + wb.save(source) + (model,) = derive(source, "XLSX") + assert rows(DataSourceParser().get(source=model), ["Hours", "Opens"]) == [["PT25H30M", "09:30:00"]] + + +class TestReferences: + """Each cell renders as stored: never `769705120.0`, never a digit restored or a zero invented.""" + + def test_a_reference_reads_as_its_stored_digits(self, references): + datasource = qd.DataSourceDefinition() + datasource.derive_model(source=references, mimetype="XLSX") + model = datasource.get[0] if isinstance(datasource.get, list) else datasource.get + df = DataSourceParser().get(source=model) + assert rows(df, ["Reference", "UPRN"]) == [ + ["769705120", "37019199"], + ["N30", None], + ["26620000000000", "100062000000"], + ["38100171142560", None], + ["20020", "10024142866"], + ] + + def test_a_mixed_column_records_its_cell_types(self, references): + (model,) = derive(references, "XLSX") + profile = {c.name: c.profile for c in model.columns} + assert profile["Reference"].cellTypes == {"n": 4, "s": 1} + assert (profile["UPRN"].cellTypes, profile["UPRN"].blank) == ({"n": 3}, 2) diff --git a/tests/test_fingerprint.py b/tests/test_fingerprint.py new file mode 100644 index 0000000..9cec10f --- /dev/null +++ b/tests/test_fingerprint.py @@ -0,0 +1,137 @@ +# SPDX-License-Identifier: BSD-3-Clause +"""Two proofs of a source: its byte hash, which any byte changes, and `fp0`, which only a cell changes. + +`fp0` hashes the file's grid as parsed, every sheet in workbook order, each cell its stored type and whyqd's text, +before any row is skipped, joined, or taken as the header, and before any cell is trimmed: it proves the source, not +the selection. A file saved again without a cell changed keeps its `fp0`, so it verifies with a warning. A PDF has the +byte hash only, since its grid depends on how it is read. +""" + +from pathlib import Path +import warnings + +import openpyxl +import pytest + +import whyqd as qd +from whyqd.parsers import CoreParser + +DATA = Path(__file__).resolve().parent / "data" +SYNTHETIC = DATA / "synthetic" +ENCODINGS = { + "fixture-utf8.csv": None, + "fixture-utf8-bom.csv": None, + "fixture-utf8-crlf.csv": None, + "fixture-utf8-mixed-eol.csv": None, + "fixture-utf16-bom.csv": None, + "fixture-latin-1.csv": None, + "fixture-windows-1252.csv": None, + "fixture-mac-roman.csv": "macintosh", +} + + +def derive(source: Path, mimetype: str, dialect: dict | None = None) -> qd.DataSourceDefinition: + datasource = qd.DataSourceDefinition() + datasource.derive_model(source=source, mimetype=mimetype, dialect=dialect) + return datasource + + +def workbook(path: Path, cells: dict[str, object], sheets: tuple[str, ...] = ("Data",)) -> Path: + wb = openpyxl.Workbook() + wb.active.title = sheets[0] + for name in sheets[1:]: + wb.create_sheet(name) + for ws in wb.worksheets: + for ref, value in cells.items(): + ws[ref] = value + wb.save(path) + return path + + +class TestFp0: + def test_one_file_in_any_encoding_has_one_fp0(self): + models = [ + derive(SYNTHETIC / name, "CSV", None if encoding is None else {"encoding": encoding}).get + for name, encoding in ENCODINGS.items() + ] + assert len({m.get_checksum("fp0") for m in models}) == 1 + # The Latin-1 and Windows-1252 fixtures hold no byte that tells the two apart, so are one file + assert len({m.get_checksum("bytes") for m in models}) == len({(SYNTHETIC / n).read_bytes() for n in ENCODINGS}) + + def test_fp0_is_blake2b_512(self): + model = derive(SYNTHETIC / "fixture-utf8.csv", "CSV").get + fp0 = next(c for c in model.checksums if c.content == "fp0") + assert fp0.algorithm == "BLAKE2b-512" + assert len(fp0.checksumValue) == 128 + + def test_fp0_is_taken_before_the_dialect_selects(self, tmp_path): + source = tmp_path / "padded.csv" + source.write_text("Title\nName,Code\n a ,1\n", encoding="utf-8") + plain = derive(source, "CSV").get + selected = derive(source, "CSV", {"skipRows": 1, "trim": True}).get + assert [c.name for c in selected.columns] == ["Name", "Code"] + assert plain.get_checksum("fp0") and selected.get_checksum("fp0") == plain.get_checksum("fp0") + + def test_a_trimmed_cell_reads_trimmed(self, tmp_path): + source = tmp_path / "padded.csv" + source.write_text("Name,Code\n a ,1\n", encoding="utf-8") + datasource = derive(source, "CSV", {"trim": True}) + df = datasource.get_data() + assert df["Name"].tolist() == ["a"] + + def test_every_sheet_of_a_workbook_shares_one_fp0(self, tmp_path): + source = workbook(tmp_path / "two.xlsx", {"A1": "Name", "A2": "a"}, sheets=("One", "Two")) + models = derive(source, "XLSX").get + assert len(models) == 2 + assert models[0].get_checksum("fp0") and models[0].get_checksum("fp0") == models[1].get_checksum("fp0") + + def test_every_sheet_is_in_fp0(self, tmp_path): + one = derive(workbook(tmp_path / "one.xlsx", {"A1": "Name", "A2": "a"}), "XLSX").get + two = derive(workbook(tmp_path / "two.xlsx", {"A1": "Name", "A2": "a"}, sheets=("Data", "More")), "XLSX").get + assert one.get_checksum("fp0") != two[0].get_checksum("fp0") + + def test_a_cells_stored_type_is_in_fp0(self, tmp_path): + number = derive(workbook(tmp_path / "number.xlsx", {"A1": "Code", "A2": 12}), "XLSX").get + text = derive(workbook(tmp_path / "text.xlsx", {"A1": "Code", "A2": "12"}), "XLSX").get + assert number.get_checksum("fp0") != text.get_checksum("fp0") + + def test_a_pdf_has_the_byte_hash_only(self): + pytest.importorskip("pdfplumber") + model = derive(DATA / "sample" / "pdf" / "NNDR EXEMPTIONS.pdf", "PDF").get + assert model.get_checksum("bytes") + assert model.get_checksum("fp0") is None + + +class TestVerify: + def test_a_file_saved_again_verifies_by_fp0_with_a_warning(self, tmp_path): + source = workbook(tmp_path / "data.xlsx", {"A1": "Name", "A2": "a"}) + datasource = derive(source, "XLSX") + wb = openpyxl.load_workbook(source) + wb.active.column_dimensions["A"].width = 40 + wb.save(source) + with pytest.warns(UserWarning, match="saved again"): + assert datasource.validate() + + def test_an_unchanged_file_verifies_without_a_warning(self, tmp_path): + source = workbook(tmp_path / "data.xlsx", {"A1": "Name", "A2": "a"}) + datasource = derive(source, "XLSX") + with warnings.catch_warnings(): + warnings.simplefilter("error") + assert datasource.validate() + + def test_a_changed_cell_fails(self, tmp_path): + source = workbook(tmp_path / "data.xlsx", {"A1": "Name", "A2": "a"}) + datasource = derive(source, "XLSX") + workbook(source, {"A1": "Name", "A2": "b"}) + with pytest.raises(AssertionError, match="cells"): + datasource.validate() + + def test_a_verified_1x_checksum_gains_fp0_and_the_byte_hash(self): + pytest.importorskip("openpyxl") + model = CoreParser().load_json(source=DATA / "raw_E06000044_014_0.data") + datasource = qd.DataSourceDefinition(source=model) + assert datasource.get.get_checksum("fp0") is None + assert datasource.validate() + derived = derive(DATA / "raw_E06000044_014_0.XLSX", "XLSX").get + assert derived.get_checksum("fp0") and datasource.get.get_checksum("fp0") == derived.get_checksum("fp0") + assert datasource.get.get_checksum("bytes") == derived.get_checksum("bytes") diff --git a/tests/test_format_version.py b/tests/test_format_version.py index 4232884..ebfad09 100644 --- a/tests/test_format_version.py +++ b/tests/test_format_version.py @@ -160,15 +160,14 @@ def test_length_bounds_ordered(self): class TestVerify1x: @pytest.mark.parametrize("path", DATA_SOURCES, ids=lambda p: p.name) - def test_verifies_and_adds_byte_hash(self, path): + def test_verifies_and_adds_byte_hash_and_fp0(self, path): original = json.loads(path.read_text(encoding="utf-8")) datasource = load(path) assert datasource.validate() file_hash = hashlib.blake2b(Path(original["path"]).read_bytes()).hexdigest() - assert as_json(datasource.get_json())["checksums"] == [ - checksum(original["checksum"], "table-1x"), - checksum(file_hash, "bytes"), - ] + table, hashed, fp0 = as_json(datasource.get_json())["checksums"] + assert [table, hashed] == [checksum(original["checksum"], "table-1x"), checksum(file_hash, "bytes")] + assert fp0["content"] == "fp0" and len(fp0["checksumValue"]) == 128 def test_1x_save_after_verifying_keeps_table_1x(self): path = DIRECTORY / "test_portsmouth_source.data" @@ -192,6 +191,41 @@ def test_without_extra_names_it(self, monkeypatch): with pytest.raises(ImportError, match=r"whyqd\[verify-1x\]"): load(DIRECTORY / "test_portsmouth_source.data").validate() + def test_verifies_through_its_own_read(self): + # In a process of its own, where no test has loaded the reader: the 1.x read alone must not load it + script = ( + "import sys, whyqd as qd\n" + "from whyqd.parsers.v1 import V1ChecksumParser\n" + f"model = qd.DataSourceDefinition(source={str(DIRECTORY / 'test_portsmouth_source.data')!r}).get\n" + "assert V1ChecksumParser().checksum(model=model) == model.get_checksum('table-1x')\n" + "print('whyqd.parsers.datasource' in sys.modules)\n" + ) + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, check=True) + assert result.stdout.strip() == "False" + + def test_without_pyarrow_only_parquet_fails(self): + # Without pyarrow pandas keeps text in Python rather than Arrow; the printed table, and so the checksum, holds + script = ( + "import sys\n" + "sys.modules['pyarrow'] = None\n" + "import whyqd as qd\n" + "from whyqd.parsers.v1 import V1ChecksumParser\n" + f"for path in {[str(p) for p in DATA_SOURCES]!r}:\n" + " try:\n" + " model = qd.DataSourceDefinition(source=path).get\n" + " assert V1ChecksumParser().checksum(model=model) == model.get_checksum('table-1x')\n" + " print(path.rsplit('/', 1)[-1], 'verifies')\n" + " except ImportError:\n" + " print(path.rsplit('/', 1)[-1], 'needs pyarrow')\n" + ) + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, check=True) + outcomes = dict(line.split(" ", 1) for line in result.stdout.strip().splitlines()) + parquet = { + p.name for p in DATA_SOURCES if json.loads(p.read_text(encoding="utf-8"))["mime"].endswith("parquet") + } + assert parquet + assert outcomes == {p.name: "needs pyarrow" if p.name in parquet else "verifies" for p in DATA_SOURCES} + class TestDefinitionsOnly: def test_loading_definitions_imports_no_pandas(self, tmp_path): diff --git a/tests/test_header.py b/tests/test_header.py new file mode 100644 index 0000000..1706dfd --- /dev/null +++ b/tests/test_header.py @@ -0,0 +1,137 @@ +# SPDX-License-Identifier: BSD-3-Clause +"""The header row, as the wrangler declares it, and the rows the table keeps. + +Under the dialect, the header row and every row above it are not in the table: row 0 is the first row after the +header, or the file's first where there is none. A 1.x data source upgrades to the dialect that reads its rows as 1.x +did, so a stored crosswalk addresses the same rows: `header` N names the columns from row N and keeps every row as data, +which is a file with no header row and those names. +""" + +import json +from pathlib import Path + +import pandas as pd +import pytest + +import whyqd as qd +from whyqd.parsers import DataSourceParser, RowKeyParser +from whyqd.parsers.v1 import V1ChecksumParser + +DATA = Path(__file__).resolve().parent / "data" +LAYOUTS = DATA / "synthetic-layouts" +KEY = json.loads((LAYOUTS / "answer-key.json").read_text(encoding="utf-8"))["csv"] +WORLD_BANK = DATA / "working_test_world_bank_data.xlsx" +ROW = RowKeyParser.ROW + + +def table(datasource: qd.DataSourceDefinition, index: int | None = None): + model = datasource.get if index is None else datasource.get[index] + return DataSourceParser().get(source=model) + + +def cells(rows: list[list[str]]) -> list[list[str | None]]: + # The key writes an empty cell as the file does; the table holds it as missing, as `CellParser.render` gives it + return [[c if c != "" else None for c in row] for row in rows] + + +def values(df: pd.DataFrame) -> list[list[str | None]]: + return [[None if pd.isna(c) else c for c in row] for row in df.values.tolist()] + + +def write(tmp_path: Path, text: str) -> Path: + source = tmp_path / "source.csv" + source.write_text(text, encoding="utf-8") + return source + + +class TestHeader: + def test_rows_above_the_header_leave_the_table(self): + case = KEY["title-above.csv"] + datasource = qd.DataSourceDefinition() + datasource.derive_model(source=LAYOUTS / "title-above.csv", mimetype="CSV", dialect=case["dialect"]) + df = table(datasource) + assert [c.name for c in datasource.get.columns] == case["names"] + assert df[ROW].tolist() == list(range(len(case["rows"]))) + assert values(df[case["names"]]) == cells(case["rows"]) + + def test_each_sheet_takes_its_own_dialect(self): + datasource = qd.DataSourceDefinition() + datasource.derive_model(source=WORLD_BANK, mimetype="XLSX", dialect=[{"skipRows": 3}, {}, {}]) + df = table(datasource, 0) + assert datasource.get[0].index == 264 + assert df[ROW].tolist() == list(range(264)) + assert df["Country Name"].iloc[0] == "Aruba" + assert "Country Name" not in df["Country Name"].tolist() + + def test_no_header_row_is_named_by_position(self, tmp_path): + datasource = qd.DataSourceDefinition() + datasource.derive_model(source=write(tmp_path, "a,b\n1,2\n"), mimetype="CSV", dialect={"headerRowCount": 0}) + df = table(datasource) + assert [c.name for c in datasource.get.columns] == ["column_0", "column_1"] + assert df[ROW].tolist() == [0, 1] + assert df[["column_0", "column_1"]].values.tolist() == [["a", "b"], ["1", "2"]] + + def test_no_header_row_takes_the_names_given(self, tmp_path): + datasource = qd.DataSourceDefinition() + source = write(tmp_path, "a,b\n1,2\n") + datasource.derive_model(source=source, mimetype="CSV", dialect={"headerRowCount": 0}, names=["x", "y"]) + df = table(datasource) + assert datasource.get.dialect.headerRowCount == 0 + assert df[["x", "y"]].values.tolist() == [["a", "b"], ["1", "2"]] + + def test_a_header_over_two_rows_takes_the_names_given(self, tmp_path): + datasource = qd.DataSourceDefinition() + source = write(tmp_path, "Rate,Rate\nfrom,to\n1,2\n") + datasource.derive_model(source=source, mimetype="CSV", dialect={"headerRowCount": 2}, names=["start", "end"]) + df = table(datasource) + assert df[ROW].tolist() == [0] + assert df[["start", "end"]].values.tolist() == [["1", "2"]] + + def test_a_header_over_two_rows_without_names_is_refused(self, tmp_path): + datasource = qd.DataSourceDefinition() + source = write(tmp_path, "Rate,Rate\nfrom,to\n1,2\n") + with pytest.raises(ValueError, match=r"(?s)headerRowCount.*\bnames\b"): + datasource.derive_model(source=source, mimetype="CSV", dialect={"headerRowCount": 2}) + + def test_an_unknown_setting_is_refused_naming_those_known(self, tmp_path): + datasource = qd.DataSourceDefinition() + with pytest.raises(ValueError, match="skipRows"): + datasource.derive_model(source=write(tmp_path, "a\n1\n"), mimetype="CSV", dialect={"skiprows": 1}) + + @pytest.mark.parametrize("setting", [{"header": 3}, {"quoting": 1}]) + def test_a_reader_keyword_is_refused_naming_the_dialect(self, tmp_path, setting): + datasource = qd.DataSourceDefinition() + with pytest.raises(TypeError, match="dialect"): + datasource.derive_model(source=write(tmp_path, "a\n1\n2\n3\n4\n"), mimetype="CSV", **setting) + + +class TestHeader1x: + """A 1.x data source reads the rows 1.x read, numbered as 1.x numbered them.""" + + def test_a_header_below_row_0_keeps_every_row(self): + model = qd.DataSourceDefinition(source=DATA / "test_world_bank_header_source.data").get + model.path = str(WORLD_BANK) + assert model.dialect is not None and model.dialect.headerRowCount == 0 + expected = V1ChecksumParser().read(source=model) + df = DataSourceParser().get(source=model) + assert df[ROW].tolist() == list(range(len(expected))) == list(range(268)) + assert values(df[["Country Name"]])[:5] == [["Data Source"], ["Last Updated Date"], [None], ["Country Name"], ["Aruba"]] + assert df["Country Name"].fillna("").tolist() == expected["Country Name"].fillna("").tolist() + + def test_a_stored_crosswalk_gives_the_same_output(self): + # Its `DELETE_ROWS < [0, 1, 2, 3]` removes the title rows and the header row, which 1.x kept as data + source = qd.DataSourceDefinition(source=DATA / "test_world_bank_header_source.data") + source.get.path = str(WORLD_BANK) + crosswalk = qd.CrosswalkDefinition(crosswalk=DATA / "test_world_bank_header.crosswalk") + transform = qd.TransformDefinition(crosswalk=crosswalk, data_source=source.get) + transform.process() + expected = pd.read_parquet(DATA / "test_world_bank_header_output.parquet") + pd.testing.assert_frame_equal(transform.data.reset_index(drop=True), expected) + + def test_no_header_row_with_names_keeps_them(self): + model = qd.DataSourceDefinition(source=DATA / "test_cthulu_source.data").get + model.path = str(DATA / Path(model.path).name) + assert model.dialect is not None and model.dialect.headerRowCount == 0 + df = DataSourceParser().get(source=model) + assert [c for c in df.columns if c != ROW] == model.names + assert df[ROW].tolist() == list(range(model.index)) diff --git a/tests/test_known_faults.py b/tests/test_known_faults.py index d2bc1d3..7cde804 100644 --- a/tests/test_known_faults.py +++ b/tests/test_known_faults.py @@ -10,11 +10,9 @@ import math from pathlib import Path -import openpyxl import pandas as pd import pytest -import whyqd as qd from whyqd.crosswalk.actions.categorise import Action as Categorise from whyqd.crosswalk.actions.pivot_categories import Action as PivotCategories from whyqd.crosswalk.actions.pivot_longer import Action as PivotLonger @@ -88,41 +86,3 @@ def test_pivot_categories_refuses_a_null_header(self): PivotCategories().transform( df=df, destination=FieldModel(name="cat", type="string"), source=FieldModel(name="c", type="string"), assigned=[0, 2] ) - - @pytest.mark.parametrize("name", ["fixture-windows-1252.csv", "fixture-latin-1.csv", "fixture-mac-roman.csv"]) - @pytest.mark.xfail(strict=True, raises=AssertionError, reason="bytes that are not UTF-8 are deleted") - def test_csv_keeps_every_byte(self, name): - df = DataSourceParser().get(source=DATA / "synthetic" / name, mimetype="CSV", preserve=True) - assert df["amount"].tolist()[0] == "£43.40" - - @pytest.mark.xfail(strict=True, raises=pd.errors.ParserError, reason="UTF-16 is read as UTF-8") - def test_csv_reads_utf16(self): - df = DataSourceParser().get(source=DATA / "synthetic" / "fixture-utf16-bom.csv", mimetype="CSV", preserve=True) - assert df["amount"].tolist()[0] == "£43.40" - - @pytest.mark.xfail(strict=True, raises=AssertionError, reason="missing-value text is read as missing") - def test_csv_keeps_missing_value_text(self, tmp_path): - source = tmp_path / "na.csv" - source.write_text("contact\nN/A\nx\n", encoding="utf-8") - df = DataSourceParser().get(source=source, mimetype="CSV", preserve=True) - assert [None if pd.isna(v) else v for v in df["contact"]] == ["N/A", "x"] - - @pytest.mark.xfail(strict=True, raises=IndexError, reason="a blank second row hides the only sheet") - def test_single_sheet_with_a_blank_second_row_reads(self, tmp_path): - source = tmp_path / "titled.xlsx" - wb = openpyxl.Workbook() - ws = wb.active - ws["A1"] = "Reliefs key" - ws["A3"], ws["B3"] = "Code", "Description" - ws["A4"], ws["B4"] = "MAN", "Mandatory" - wb.save(source) - datasource = qd.DataSourceDefinition() - datasource.derive_model(source=source, mimetype="XLSX", header=None) - assert datasource.get.index == 4 - - @pytest.mark.xfail(strict=True, raises=AssertionError, reason="blank CSV lines are skipped and not numbered") - def test_csv_blank_line_is_a_row(self, tmp_path): - source = tmp_path / "blank.csv" - source.write_text("a\n1\n\n2\n", encoding="utf-8") - df = DataSourceParser().get(source=source, mimetype="CSV") - assert df.index.tolist() == [0, 1, 2] diff --git a/tests/test_layouts.py b/tests/test_layouts.py new file mode 100644 index 0000000..0513c19 --- /dev/null +++ b/tests/test_layouts.py @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: BSD-3-Clause +r"""Each CSV layout in the synthetic fixture read to its answer key, under the dialect the key declares. + +Read with fixed defaults, a file loads as written: a blank line is a numbered row, a quoted line break stays in its +cell, `N/A` and its kind are text, and a trailing delimiter's empty last column is trimmed with the grid. Every line +end (`\r\n`, `\n`, `\r`) ends a row. The grid is as wide as its widest row: a column past the header is named by +position, and a short row's missing cells are empty. Where the default cannot read the file as meant (`;` between +cells, a quote that never closes, a quote escaped by a backslash), the key's dialect declares it. +The layouts read elsewhere: `title-above.csv` in `test_header.py`, `hostile-headers.csv` in `test_column_names.py`, and +`two-line-records.csv` in `test_records.py`. +""" + +import json +from pathlib import Path + +import pandas as pd +import pytest + +import whyqd as qd +from whyqd.parsers import DataSourceParser, RowKeyParser + +LAYOUTS = Path(__file__).resolve().parent / "data" / "synthetic-layouts" +KEY = json.loads((LAYOUTS / "answer-key.json").read_text(encoding="utf-8"))["csv"] +ROW = RowKeyParser.ROW +CASES = [ + "blank-line-inside.csv", + "quoted-line-break.csv", + "semicolon.csv", + "stray-quote.csv", + "trailing-delimiter.csv", + "missing-like-text.csv", + "backslash-escape.csv", + "cr-line-ends.csv", + "header-narrower.csv", + "first-row-narrower.csv", + "row-narrower.csv", + "two-line-records-unpadded.csv", +] + + +@pytest.mark.parametrize("name", CASES) +def test_each_layout_reads_to_its_key(name): + case = KEY[name] + datasource = qd.DataSourceDefinition() + datasource.derive_model( + source=LAYOUTS / name, mimetype="CSV", **({"dialect": case["dialect"]} if case["dialect"] else {}) + ) + assert datasource.get.dialect is not None + df = DataSourceParser().get(source=datasource.get) + assert [c.name for c in datasource.get.columns] == case["names"] + assert df[ROW].tolist() == list(range(len(case["rows"]))) + # The key writes an empty cell as the file does; the table holds it as missing, as `CellParser.render` gives it + expected = [[c if c != "" else None for c in row] for row in case["rows"]] + assert [[None if pd.isna(c) else c for c in row] for row in df[case["names"]].values.tolist()] == expected + + +def test_only_a_line_end_ends_a_row(tmp_path): + # U+0085, U+2028, and a form feed end a line for `str.splitlines`, never a CSV row; the last line has no end + source = tmp_path / "separators.csv" + source.write_bytes("Code,Name\n1,a\u0085b\n2,c
d\f\n3,e".encode("utf-8")) + datasource = qd.DataSourceDefinition() + datasource.derive_model(source=source, mimetype="CSV") + df = DataSourceParser().get(source=datasource.get) + assert df[["Code", "Name"]].values.tolist() == [["1", "a\u0085b"], ["2", "c
d\f"], ["3", "e"]] + + +def test_a_1x_csv_with_a_blank_line_warns_that_its_rows_are_renumbered(): + # 1.x skipped the blank line, so it read one row fewer and numbered every row after it one lower + model = qd.models.DataSourceModel( + path=str(LAYOUTS / "blank-line-inside.csv"), + mime="text/csv", + columns=[{"name": n} for n in KEY["blank-line-inside.csv"]["names"]], + # Its 1.x checksum is what marks it as read by 1.x; the value is never checked here + checksums=[{"algorithm": "BLAKE2b-512", "checksumValue": "0", "content": "table-1x"}], + index=len(KEY["blank-line-inside.csv"]["rows"]) - 1, + ) + with pytest.warns(UserWarning, match="1.x read 6"): + df = DataSourceParser().get(source=model) + assert len(df) == 7 diff --git a/tests/test_pdf.py b/tests/test_pdf.py new file mode 100644 index 0000000..84fe9e8 --- /dev/null +++ b/tests/test_pdf.py @@ -0,0 +1,121 @@ +# SPDX-License-Identifier: BSD-3-Clause +"""A PDF's tables, read by what the file shows and by what the wrangler declares. + +What a PDF shows is read, never asked: whether it has text at all, its orientation, and whether its table is ruled, +which sets how cells are found (`lines`, by the ruling, else `text`, by where the text sits), recorded in `cells`. A row +equal to the header row is the header repeated atop a page and is dropped. What it cannot show is declared: the pages +(`pages`), the columns of an unruled table (`columns`, their left edges in points), and the names. A cell's line +breaks are kept as extracted, since a wrap and a break look the same in a PDF, so the tests compare a cell's words. +""" + +import csv +import json +from collections import Counter +from pathlib import Path + +import pandas as pd +import pytest + +import whyqd as qd +from whyqd.parsers import DataSourceParser, RowKeyParser + +# Reading a PDF needs the `pdf` extra +pytest.importorskip("pdfplumber") + +DATA = Path(__file__).resolve().parent / "data" +SOURCE = DATA / "synthetic-layouts" / "pdf-source.pdf" +KEY = json.loads((DATA / "synthetic-layouts" / "answer-key.json").read_text(encoding="utf-8"))["pdf"]["pdf-source.pdf"] +SAMPLE = DATA / "sample" / "pdf" +ROW = RowKeyParser.ROW +# The pages each sheet of `pdf-source.xlsx` printed to +PAGES = {"Ruled": "1", "MergedHeader": "2", "Landscape": "4", "RepeatedHeader": "5-7", "HeaderOnce": "8-10"} + + +def read(source: Path, dialect: dict, names: list[str] | None = None) -> tuple[qd.models.DataSourceModel, pd.DataFrame]: + datasource = qd.DataSourceDefinition() + datasource.derive_model( + source=source, mimetype="PDF", dialect=dialect, **({} if names is None else {"names": names}) + ) + model = datasource.get[0] if isinstance(datasource.get, list) else datasource.get + df = DataSourceParser().get(source=model) + assert df[ROW].tolist() == list(range(len(df))) + return model, df + + +def words(rows: list[list[str | None]]) -> list[list[str | None]]: + # A cell's words, so a wrap kept as a line break compares equal to the space it replaced; an empty cell is missing + return [ + [None if c is None or pd.isna(c) or not str(c).split() else " ".join(str(c).split()) for c in r] for r in rows + ] + + +def table(df: pd.DataFrame, names: list[str]) -> list[list[str | None]]: + return words(df[names].values.tolist()) + + +def expected(path: Path) -> tuple[list[str], list[list[str | None]]]: + with path.open(encoding="utf-8-sig", newline="") as f: + header, *rows = list(csv.reader(f)) + return header, words(rows) + + +class TestRead: + @pytest.mark.parametrize("name", ["Ruled", "Landscape", "RepeatedHeader", "HeaderOnce"]) + def test_a_ruled_table_reads_to_its_key(self, name): + # Landscape needs no setting; a header repeated atop each page is dropped + model, df = read(SOURCE, {"pages": PAGES[name]}) + assert model.dialect.cells == "lines" + assert [c.name for c in model.columns] == KEY[name]["names"] + assert table(df, KEY[name]["names"]) == words(KEY[name]["rows"]) + + def test_a_line_break_in_a_cell_is_kept(self): + model, df = read(SOURCE, {"pages": PAGES["Ruled"]}) + assert any("\n" in str(v) for v in df["Property Address"]) + + def test_a_header_over_two_rows_takes_the_names_given(self): + names = ["Propref", "Address", "Postcode", "Relief Code", "R/V"] + model, df = read(SOURCE, {"pages": PAGES["MergedHeader"], "headerRowCount": 2}, names) + assert table(df, names) == words(KEY["MergedHeader"]["rows"]) + + def test_an_unruled_table_reads_by_text_position(self): + model, df = read(SOURCE, {"pages": "11"}) + assert model.dialect.cells == "text" + # Undeclared, each wrapped line is a row of its own: loaded, not refused, for the wrangler to see and declare + assert len(df) > len(KEY["UnruledWrapped"]["rows"]) + + def test_a_scanned_page_is_refused_saying_why(self): + with pytest.raises(ValueError, match=r"(?is)(image|scan).*text|text.*(image|scan)"): + read(SAMPLE / "maldon-page-1-scanned.pdf", {}) + + +class TestSample: + """The sample's PDFs against the extractions saved beside them.""" + + def test_west_berkshire_reliefs_reads_by_its_ruling(self): + header, rows = expected(SAMPLE / "west-berkshire-reliefs.csv") + model, df = read(SAMPLE / "NNDR RELIEFS 15 04 2026 pages 1-3.pdf", {}) + assert [c.name for c in model.columns] == header + assert table(df, header) == rows + + def test_west_berkshire_exemptions_reads_with_its_repeated_headers_dropped(self): + header, rows = expected(SAMPLE / "west-berkshire-exemptions.csv") + model, df = read(SAMPLE / "NNDR EXEMPTIONS.pdf", {}) + assert model.dialect.cells == "lines" + assert table(df, header) == rows + + def test_maldon_reads_with_its_columns_declared(self): + # Unruled: read by text position its words split mid-word, so the columns' left edges, from its header line + header, rows = expected(SAMPLE / "2026-03-27-maldon-empty.CSV") + dialect = {"skipRows": 1, "columns": [19, 151, 262, 364, 463, 496, 533, 679]} + model, df = read(SAMPLE / "NDR_Empty_Property_List___March_2026.pdf", dialect) + assert model.dialect.cells == "text" + assert [c.name for c in model.columns] == header + # Compared by a record's words, since a word printed past its column reads in the next + found = [" ".join(c for c in r if c) for r in table(df, header)] + wanted = [" ".join(c for c in r if c) for r in rows] + position = 0 + for record in wanted: + position = found.index(record, position) + 1 + # The extraction was also cleaned: the rest are the title and number printed on each page, and repeated records + extra = Counter(found) - Counter(wanted) + assert all(r == "NDR Empty Property List - March 2026" or r.isdigit() or r in wanted for r in extra) diff --git a/tests/test_polars.py b/tests/test_polars.py new file mode 100644 index 0000000..47eb0d1 --- /dev/null +++ b/tests/test_polars.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: BSD-3-Clause +r"""What the read relies on Polars for, and what it does not. + +The grid is held in Polars and handed on as pandas, so every name and cell must cross between the two unchanged, in +either direction, including a table Polars' own reader could not have opened. + +Polars' CSV reader is not used, since it does not read every file as written: a quote escaped by a backslash, `\r` +line ends, rows wider than the first, and a row's own width, which `rowsPerRecord` reads. Each is a strict `xfail` below, so a Polars release that reads it fails the +test, and the choice can be weighed again. +""" + +import csv +import json +from pathlib import Path + +import pandas as pd +import polars as pl +import pytest + +LAYOUTS = Path(__file__).resolve().parent / "data" / "synthetic-layouts" +KEY = json.loads((LAYOUTS / "answer-key.json").read_text(encoding="utf-8"))["csv"] + +HOSTILE = [ + ["a", 'Some column \\"with\\" escapes', "one\rtwo", "two\nlines", "", None], + [" hard space", "​zero-width", "inner mark", "ctrl\x01\x1f", "é beside é", "12"], + [" spaced ", "\t", "N/A", "null", "x" * 100_000, "\x00nul"], +] +NAMES = ["Unnamed: 0", "__row__.1", "Rate [2026]", "Code, Local", " lead", "trail "] + + +def cells(df: pd.DataFrame) -> list[list[str | None]]: + return [[None if pd.isna(c) else c for c in row] for row in df.values.tolist()] + + +def lines(name: str) -> list[list[str]]: + # A lone empty value is written as a blank line, which reads as no fields + return [[] if line == [""] else line for line in KEY[name]["lines"]] + + +def polars_lines(name: str, **settings) -> list[list[str]]: + # Each row as read: a field the row lacks is null, an empty field empty text + df = pl.read_csv( + LAYOUTS / name, has_header=False, infer_schema=False, empty_string_is_null=False, **settings + ) + read = [] + for row in df.rows(): + row = list(row) + while row and row[-1] is None: + row.pop() + read.append(row) + return read + + +class TestHandover: + @pytest.mark.parametrize("dtype", ["object", "string"]) + def test_pandas_to_polars_and_back(self, dtype): + df = pd.DataFrame(HOSTILE, columns=NAMES).astype(dtype) + back = pl.from_pandas(df).to_pandas() + assert list(back.columns) == NAMES + assert cells(back) == HOSTILE + + def test_polars_to_pandas(self): + grid = pl.DataFrame([dict(zip(NAMES, row, strict=True)) for row in HOSTILE], schema={n: pl.String for n in NAMES}) + df = grid.to_pandas() + assert list(df.columns) == NAMES + assert cells(df) == HOSTILE + assert df.iloc[0, 4] == "" and pd.isna(df.iloc[0, 5]) + + @pytest.mark.parametrize("name", ["backslash-escape.csv", "cr-line-ends.csv", "header-narrower.csv"]) + def test_a_table_polars_cannot_open_crosses_unchanged(self, name): + settings = {"doublequote": False, "escapechar": "\\"} if name == "backslash-escape.csv" else {} + with (LAYOUTS / name).open(encoding="utf-8", newline="") as f: + rows = list(csv.reader(f, **settings)) + width = max(len(r) for r in rows) + df = pd.DataFrame([r + [None] * (width - len(r)) for r in rows], columns=[f"c{i}" for i in range(width)]) + assert cells(pl.from_pandas(df).to_pandas()) == cells(df) + + +class TestPolarsReader: + def test_a_quoted_line_break(self): + assert polars_lines("quoted-line-break.csv") == lines("quoted-line-break.csv") + + @pytest.mark.parametrize("name", ["blank-line-inside.csv", "row-narrower.csv", "two-line-records-unpadded.csv"]) + @pytest.mark.xfail(strict=True, raises=AssertionError, reason="a short row is padded, so its own width is lost") + def test_rows_narrower_than_the_first(self, name): + assert polars_lines(name) == lines(name) + + @pytest.mark.xfail(strict=True, reason="a backslash ends the quoted cell, and no setting escapes it") + def test_a_quote_escaped_by_a_backslash(self): + assert polars_lines("backslash-escape.csv") == lines("backslash-escape.csv") + + @pytest.mark.xfail(strict=True, reason="`\\r` alone does not end a row") + def test_cr_line_ends(self): + assert polars_lines("cr-line-ends.csv") == lines("cr-line-ends.csv") + + @pytest.mark.parametrize("name", ["header-narrower.csv", "first-row-narrower.csv"]) + @pytest.mark.xfail(strict=True, raises=pl.exceptions.ComputeError, reason="the first row fixes the width") + def test_rows_wider_than_the_first(self, name): + assert polars_lines(name) == lines(name) diff --git a/tests/test_profile.py b/tests/test_profile.py new file mode 100644 index 0000000..f8f7736 --- /dev/null +++ b/tests/test_profile.py @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: BSD-3-Clause +"""What each column holds, as read, in counts: its cells by stored type, blank and whitespace-only cells, the +anomalies, and, for an `.xlsx`, its number and date cells by their format code. Profiling never changes a cell. + +A column's profile counts the table's rows, below its header, each cell as stored, before any trim. Excel's date cells +beside text in one column are the case that needs it: Stockport's `LiabStart` holds 4,232 date cells, swapped by Excel +under a month-first locale, beside 2,382 text cells it could not read so. +""" + +from pathlib import Path + +import openpyxl +import pytest + +import whyqd as qd + +DATA = Path(__file__).resolve().parent / "data" +STOCKPORT = DATA / "sample" / "stockport" / "2026-09-04-stockport-original.XLSX" + + +def profiles(source: Path, mimetype: str, dialect: dict | None = None) -> dict[str, qd.models.ProfileModel]: + datasource = qd.DataSourceDefinition() + datasource.derive_model(source=source, mimetype=mimetype, dialect=dialect) + model = datasource.get[0] if isinstance(datasource.get, list) else datasource.get + return {c.name: c.profile for c in model.columns} + + +@pytest.fixture +def anomalies(tmp_path) -> Path: + cells = [ + "", + " ", + "a b", + " ", + "x​", + "y", + "c\x07", + "tab\there", + "0012", + "0", + "12/03/2024", + "2024-03-12", + "1 Apr 2024", + "Apr-24", + "April", + "1.5", + ] + source = tmp_path / "anomalies.csv" + source.write_text("Title\nValue\n" + "\n".join(f'"{c}"' for c in cells) + "\n", encoding="utf-8", newline="") + return source + + +class TestText: + def test_each_anomaly_is_counted(self, anomalies): + profile = profiles(anomalies, "CSV", {"skipRows": 1})["Value"].model_dump() + assert profile == { + "cellTypes": {"s": 15}, + "blank": 1, + "whitespace": 2, + "hardSpace": 2, + "zeroWidth": 2, + "control": 1, + "dateLike": 4, + "zeroPadded": 1, + "numberFormats": {}, + } + + def test_the_header_and_rows_above_it_are_not_counted(self, anomalies): + profile = profiles(anomalies, "CSV")["Title"] + assert (profile.cellTypes, profile.blank) == ({"s": 16}, 1) + + def test_cells_are_counted_as_stored_before_any_trim(self, anomalies): + profile = profiles(anomalies, "CSV", {"skipRows": 1, "trim": True})["Value"] + assert (profile.whitespace, profile.blank) == (2, 1) + + def test_a_derived_data_source_saves_its_profile(self, anomalies): + datasource = qd.DataSourceDefinition() + datasource.derive_model(source=anomalies, mimetype="CSV", dialect={"skipRows": 1}) + written = qd.DataSourceDefinition(source=datasource.get.model_dump(by_alias=True, exclude_none=True)).get + assert written.columns[0].profile and written.columns[0].profile == datasource.get.columns[0].profile + + +class TestWorkbook: + def test_excel_date_cells_beside_text_are_recorded(self): + profile = profiles(STOCKPORT, "XLSX") + assert profile["LiabStart"].cellTypes == {"s": 2382, "d": 4232} + assert profile["LiabStart"].numberFormats == {"mm-dd-yy": 4232} + + def test_number_cells_are_counted_by_format_code(self): + profile = profiles(STOCKPORT, "XLSX") + assert profile["PropAddress1"].numberFormats == {"General": 12, "d-mmm": 5} + assert profile["Address1"].numberFormats == {"General": 201, "0.00E+00": 79} + + def test_a_custom_format_is_read_from_the_workbook(self, tmp_path): + source = tmp_path / "formats.xlsx" + wb = openpyxl.Workbook() + ws = wb.active + ws.append(["Reference"]) + for value, fmt in [(12, "0000000000"), (13, "0000000000"), ("0014", "@"), (15, "General")]: + ws.append([value]) + ws.cell(row=ws.max_row, column=1).number_format = fmt + wb.save(source) + profile = profiles(source, "XLSX")["Reference"] + assert profile.cellTypes == {"s": 1, "n": 3} + assert profile.numberFormats == {"0000000000": 2, "General": 1} + assert profile.zeroPadded == 1 + + def test_an_xls_has_no_number_formats(self): + profile = profiles(DATA / "synthetic-layouts" / "layouts.xls", "XLS", {"sheet": "Titled", "skipRows": 3}) + assert any({"n", "d"} & set(p.cellTypes) for p in profile.values()) + assert all(p.numberFormats == {} for p in profile.values()) diff --git a/tests/test_records.py b/tests/test_records.py new file mode 100644 index 0000000..a5bfdd2 --- /dev/null +++ b/tests/test_records.py @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: BSD-3-Clause +"""Records that span several rows, joined as the wrangler declares them in `rowsPerRecord`. + +`[5, 1]` is five fields on a record's first row and one on its next, the header included. The join acts on rows as +read, so a quoted line break stays inside its cell, and the row key counts records. A row that does not fit the +declaration is refused, naming it, since every record after it would be misread. +""" + +import csv +import json +from pathlib import Path + +import pandas as pd +import pytest + +import whyqd as qd +from whyqd.parsers import DataSourceParser, RowKeyParser + +DATA = Path(__file__).resolve().parent / "data" +LAYOUTS = DATA / "synthetic-layouts" +KEY = json.loads((LAYOUTS / "answer-key.json").read_text(encoding="utf-8"))["csv"]["two-line-records.csv"] +FENLAND = DATA / "sample" / "fenland" / "2026-10-25-Fenland District Council.csv" +ROW = RowKeyParser.ROW + + +def read(source: Path, dialect: dict) -> tuple[qd.models.DataSourceModel, pd.DataFrame]: + datasource = qd.DataSourceDefinition() + datasource.derive_model(source=source, mimetype="CSV", dialect=dialect) + return datasource.get, DataSourceParser().get(source=datasource.get) + + +def rows(df: pd.DataFrame, names: list[str]) -> list[list[str | None]]: + return [[None if pd.isna(c) else c for c in row] for row in df[names].values.tolist()] + + +def cells(rows: list[list[str]]) -> list[list[str | None]]: + # The key writes an empty cell as the file does; the table holds it as missing, as `CellParser.render` gives it + return [[c if c != "" else None for c in row] for row in rows] + + +def write(tmp_path: Path, lines: list[list[str]]) -> Path: + source = tmp_path / "records.csv" + with source.open("w", encoding="utf-8", newline="") as f: + csv.writer(f).writerows(lines) + return source + + +PAIRS = [["a", "b", "c"], ["d", "", ""], ["p1", "q1", "r1"], ["s1", "", ""], ["p2", "q2", "r2"], ["s2", "", ""]] + + +class TestRowsPerRecord: + def test_records_over_two_rows_join_to_the_key(self): + model, df = read(LAYOUTS / "two-line-records.csv", KEY["dialect"]) + assert model.dialect.rowsPerRecord == KEY["dialect"]["rowsPerRecord"] + assert [c.name for c in model.columns] == KEY["names"] + assert df[ROW].tolist() == list(range(len(KEY["rows"]))) + assert rows(df, KEY["names"]) == cells(KEY["rows"]) + + def test_fenland_joins_each_pair(self): + # 47 fields, then one on the next row, header included + with FENLAND.open(encoding="utf-8", newline="") as f: + lines = list(csv.reader(f)) + joined = [first + second[:1] for first, second in zip(lines[0::2], lines[1::2], strict=True)] + # Its header repeats `Disc %` and `MAND %`, each repeat renamed as any is + names = RowKeyParser().deconflict(columns=joined[0]) + model, df = read(FENLAND, {"rowsPerRecord": [47, 1]}) + assert [c.name for c in model.columns] == names + assert model.index == len(joined) - 1 == 50 + assert df[ROW].tolist() == list(range(50)) + assert rows(df, names) == cells(joined[1:]) + + def test_a_quoted_line_break_stays_in_its_cell(self, tmp_path): + lines = [*PAIRS[:2], ["p1", "two\nlines", "r1"], ["s1", "", ""]] + model, df = read(write(tmp_path, lines), {"rowsPerRecord": [3, 1]}) + assert rows(df, ["a", "b", "c", "d"]) == [["p1", "two\nlines", "r1", "s1"]] + + +class TestBrokenRecords: + """The message names the declaration and the line, as a text editor numbers it, where the file stops fitting it.""" + + def test_a_continuation_with_values_past_its_fields_is_refused(self, tmp_path): + # `p1` has lost its continuation, so `p2, q2, r2` on line 4 would be read as it + lines = [*PAIRS[:3], *PAIRS[4:]] + with pytest.raises(ValueError, match=r"(?s)rowsPerRecord.*\bline 4\b|\bline 4\b.*rowsPerRecord"): + read(write(tmp_path, lines), {"rowsPerRecord": [3, 1]}) + + def test_a_record_without_its_continuation_is_refused(self, tmp_path): + # The file ends after `p2`'s first row, on line 5 + with pytest.raises(ValueError, match=r"(?s)rowsPerRecord.*\bline 5\b|\bline 5\b.*rowsPerRecord"): + read(write(tmp_path, PAIRS[:-1]), {"rowsPerRecord": [3, 1]}) diff --git a/tests/test_row_counts.py b/tests/test_row_counts.py new file mode 100644 index 0000000..10c63eb --- /dev/null +++ b/tests/test_row_counts.py @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: BSD-3-Clause +"""A row count belongs to one file, so a reused schema carries none. + +Under format 2.0 a schema has no row count, and a row-level action's rows are checked against the data source's `index` +where the crosswalk meets its file: a schema derived from one file runs on a longer or a shorter one, and only a row +past the end of the file being read is refused. +""" + +from pathlib import Path + +import pytest + +import whyqd as qd + + +def write(tmp_path: Path, name: str, count: int) -> qd.DataSourceDefinition: + source = tmp_path / name + source.write_text("a\n" + "".join(f"r{i}\n" for i in range(count)), encoding="utf-8") + datasource = qd.DataSourceDefinition() + datasource.derive_model(source=source, mimetype="CSV") + return datasource + + +def crosswalk(datasource: qd.DataSourceDefinition, rows: list[int]) -> qd.CrosswalkDefinition: + source = qd.SchemaDefinition() + source.derive_model(data=datasource.get) + destination = qd.SchemaDefinition() + destination.set(schema={"name": "destination"}) + destination.fields.add(term={"name": "x", "type": "string"}) + definition = qd.CrosswalkDefinition() + definition.set(crosswalk={"name": "reused"}, schema_source=source, schema_destination=destination) + definition.actions.add_multi(terms=[f"DELETE_ROWS < {rows}", "RENAME > 'x' < ['a']"]) + return definition + + +def run(definition: qd.CrosswalkDefinition, datasource: qd.DataSourceDefinition) -> list[str]: + transform = qd.TransformDefinition(crosswalk=definition, data_source=datasource.get) + transform.process() + return transform.data["x"].tolist() + + +class TestRowCounts: + def test_a_derived_schema_has_no_row_count(self, tmp_path): + schema = qd.SchemaDefinition() + schema.derive_model(data=write(tmp_path, "short.csv", 3).get) + assert schema.get.index is None + + def test_a_row_past_the_schemas_file_runs_on_a_longer_one(self, tmp_path): + # The schema came from 3 rows; the file read has 5, so row 4 is in it + definition = crosswalk(write(tmp_path, "short.csv", 3), [4]) + assert run(definition, write(tmp_path, "long.csv", 5)) == ["r0", "r1", "r2", "r3"] + + def test_a_row_past_the_file_read_is_refused_naming_it(self, tmp_path): + # The schema came from 5 rows, so row 3 passes its check; the file read has 3 + definition = crosswalk(write(tmp_path, "long.csv", 5), [3]) + with pytest.raises(ValueError, match=r"\b3\b"): + run(definition, write(tmp_path, "short.csv", 3)) + + def test_a_row_in_both_files_runs_on_either(self, tmp_path): + definition = crosswalk(write(tmp_path, "long.csv", 5), [1]) + assert run(definition, write(tmp_path, "short.csv", 3)) == ["r0", "r2"] diff --git a/tests/test_row_key.py b/tests/test_row_key.py index 94e7174..d961867 100644 --- a/tests/test_row_key.py +++ b/tests/test_row_key.py @@ -32,15 +32,15 @@ def test_sort_order(self): columns = ["name", "__row__2", "__row__", "__row__.1", "__row__1"] assert RowKeyParser().sort_order(columns=columns) == ["__row__", "__row__1", "__row__2"] - @pytest.mark.parametrize("header", [0, None, 1]) - def test_numbering_matches_the_read(self, tmp_path, header): - # Row numbers that stored DELETE_ROWS and PIVOT_CATEGORIES scripts hold come from this read + @pytest.mark.parametrize("dialect", [{}, {"headerRowCount": 0}, {"skipRows": 1}]) + def test_numbering_matches_the_read(self, tmp_path, dialect): + # Row numbers that DELETE_ROWS and PIVOT_CATEGORIES scripts hold come from this read source = tmp_path / "rows.csv" lines = ["title,", "name,value", "a,1", "b,2"] source.write_text("\n".join(lines) + "\n", encoding="utf-8") datasource = qd.DataSourceDefinition() - datasource.derive_model(source=source, mimetype="CSV", header=header) + datasource.derive_model(source=source, mimetype="CSV", dialect=dialect) df = DataSourceParser().get(source=datasource.get) - start = RowKeyParser().first_position(header=header) - assert df.index.tolist() == list(range(len(lines) - start)) - assert df.iloc[0].tolist()[0] == lines[start].split(",")[0] + start = RowKeyParser().first_position(dialect=datasource.get.dialect) + assert df[RowKeyParser.ROW].tolist() == df.index.tolist() == list(range(len(lines) - start)) + assert df.iloc[0].tolist()[1] == lines[start].split(",")[0] diff --git a/tests/test_sample.py b/tests/test_sample.py new file mode 100644 index 0000000..cda8c22 --- /dev/null +++ b/tests/test_sample.py @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: BSD-3-Clause +import json +from pathlib import Path + +import pytest + +import whyqd as qd + +from tests.test_json_schema import errors + +SAMPLE = Path(__file__).resolve().parent / "data" / "sample" +TRANSFORMS = sorted(SAMPLE.glob("*/*.TRANSFORM")) + + +def without_nulls(value): + if isinstance(value, dict): + return {k: without_nulls(v) for k, v in value.items() if v is not None} + if isinstance(value, list): + return [without_nulls(v) for v in value] + return value + + +def source_file(transform: Path) -> Path: + """The spreadsheet beside a stored transform, which was saved with the path it had where it was made.""" + (found,) = [p for p in transform.parent.iterdir() if p.suffix.lower() in {".xlsx", ".xls", ".csv"}] + return found + + +class TestStoredTransforms: + @pytest.mark.parametrize("path", TRANSFORMS, ids=lambda p: p.parent.name) + def test_reads_and_writes_both_formats(self, path): + original = json.loads(path.read_text(encoding="utf-8")) + transform = qd.TransformDefinition(transform=path) + assert json.loads(transform.get_json(format="1.x")) == without_nulls(original) + written = json.loads(transform.get_json()) + assert written["formatVersion"] == "2.0" + assert errors("transform", original, format="1.x") == [] + assert errors("transform", written, format="2.0") == [] + + @pytest.mark.parametrize("path", TRANSFORMS, ids=lambda p: p.parent.name) + def test_source_checksum_verifies(self, path): + model = qd.TransformDefinition(transform=path).get.dataSource + model = model.model_copy(update={"path": str(source_file(path))}) + assert qd.DataSourceDefinition(source=model).validate() diff --git a/tests/test_schema.py b/tests/test_schema.py index 6c609db..7ad03d2 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -83,4 +83,6 @@ def test_derive(self): D = s.exclude_uuid(model=SCHEMA_DATA["DERIVED"]) D["name"] = s.get.name D.pop("version", None) + # The stored schema is 1.x's, with its file's row count; a derived one has none, its data source holding it + D.pop("index") assert json.loads(d) == {"formatVersion": "2.0"} | D diff --git a/tests/test_tutorial_1.py b/tests/test_tutorial_1.py index 0c3b3f8..db0d663 100644 --- a/tests/test_tutorial_1.py +++ b/tests/test_tutorial_1.py @@ -79,7 +79,7 @@ def test_tutorial_portsmouth_rates_data(self, tmp_path): # 3. Define a Crosswalk crosswalk = qd.CrosswalkDefinition() crosswalk.set(schema_source=schema_source, schema_destination=schema_destination) - crosswalk.actions.add_multi(terms=SCHEMA_SCRIPTS[ds.sheet_name]) + crosswalk.actions.add_multi(terms=SCHEMA_SCRIPTS[ds.dialect.sheet]) # 4. Transform a data source transform = qd.TransformDefinition(crosswalk=crosswalk, data_source=ds) transform.process() diff --git a/tests/test_tutorial_2.py b/tests/test_tutorial_2.py index b54aa08..70713b0 100644 --- a/tests/test_tutorial_2.py +++ b/tests/test_tutorial_2.py @@ -19,11 +19,11 @@ def test_tutorial_urban_population(self, tmp_path): # 1. Import a data source and derive a source schema datasource = qd.DataSourceDefinition() # There are three sheets: - # - Data, header row + 3 + # - Data, its header below three rows of title, which the read leaves out of the table # - Metadata - Countries # - Metadata - Indicators # We're only going to use the first - datasource.derive_model(source=SOURCE_DATA, mimetype=MIMETYPE, header=[3, 0, 0]) + datasource.derive_model(source=SOURCE_DATA, mimetype=MIMETYPE, dialect=[{"skipRows": 3}, {}, {}]) schema_source = qd.SchemaDefinition() schema_source.derive_model(data=datasource.get[0]) # 2. Create a destination schema @@ -86,8 +86,7 @@ def test_tutorial_urban_population(self, tmp_path): schema_scripts = [ "DEBLANK", "DEDUPE", - "DELETE_ROWS < [0, 1, 2, 3]", - f"PIVOT_LONGER > ['year', 'values'] < {datasource.model[0].names[4:]}", + f"PIVOT_LONGER > ['year', 'values'] < {[c.name for c in datasource.model[0].columns][4:]}", "RENAME > 'indicator_code' < ['Indicator Code']", "RENAME > 'indicator_name' < ['Indicator Name']", "RENAME > 'country_code' < ['Country Code']", diff --git a/tests/test_tutorial_3.py b/tests/test_tutorial_3.py index 6792bb0..0cb0b19 100644 --- a/tests/test_tutorial_3.py +++ b/tests/test_tutorial_3.py @@ -19,7 +19,7 @@ def test_tutorial_cthulhu(self, tmp_path): ##################################### FIRST CROSSWALK ##################################### # 1. Import a data source and derive a source schema datasource = qd.DataSourceDefinition() - datasource.derive_model(source=SOURCE_DATA, mimetype=MIMETYPE, header=None) + datasource.derive_model(source=SOURCE_DATA, mimetype=MIMETYPE, dialect={"headerRowCount": 0}) schema_source = qd.SchemaDefinition() schema_source.derive_model(data=datasource.get) # 2. Create an interim schema @@ -64,7 +64,7 @@ def test_tutorial_cthulhu(self, tmp_path): crosswalk.set(schema_source=schema_source, schema_destination=schema_interim) # Create the crosswalk schema_scripts = [ - f"DELETE_ROWS < {list(range(15)) + list(range(144, schema_source.get.index))}", + f"DELETE_ROWS < {list(range(15)) + list(range(144, datasource.get.index))}", "PIVOT_CATEGORIES > 'HDI Category' < 'column_0'::[15, 45, 121]", ] indices = [0, 1, 3, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21] diff --git a/uv.lock b/uv.lock index a599d3b..54ae0cd 100644 --- a/uv.lock +++ b/uv.lock @@ -441,6 +441,69 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl", hash = "sha256:c615d91d75f7f04f095b30d1c1711babd43bdc6419c1be9886a85f2f4e489417", size = 7294, upload-time = "2025-07-25T14:02:02.896Z" }, ] +[[package]] +name = "cryptography" +version = "50.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/af/182eb91b0df3fe75c4d9f26fe70684569566745f6ba7e5c9c73a862c5252/cryptography-50.0.2.tar.gz", hash = "sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5", size = 880623, upload-time = "2026-09-30T15:30:04.884Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/56/d194340cc4a57535e82e1bee9e89667ac4b7c13b5d3f59686deae3094dd5/cryptography-50.0.2-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb", size = 3914904, upload-time = "2026-09-30T14:43:44.339Z" }, + { url = "https://files.pythonhosted.org/packages/d9/69/c9bd862c3bf43d6399c433caf002df16e2dffd4be49bdf515cda38038711/cryptography-50.0.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0", size = 4731146, upload-time = "2026-09-30T14:43:47.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/69/64cef1f702bf6657e0cc186ed1a2891d50d29fb41586b254e1c07adea261/cryptography-50.0.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2", size = 4719841, upload-time = "2026-09-30T14:43:49.01Z" }, + { url = "https://files.pythonhosted.org/packages/38/6b/61a3f8d8c5e1e49a6cddccafc4015cc1c0021360ab0acb4080e7a423644a/cryptography-50.0.2-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480", size = 4738340, upload-time = "2026-09-30T14:43:50.932Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2e/7212ca32fd43dc91f2f41db20160b268098874b4c9a0e7be94d6835f5b2e/cryptography-50.0.2-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134", size = 5367029, upload-time = "2026-09-30T14:43:52.911Z" }, + { url = "https://files.pythonhosted.org/packages/1a/f1/b474e930c4d910328780e3940da76f5aa5cbc48ce1fc14e44d239d9ea9db/cryptography-50.0.2-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856", size = 4753050, upload-time = "2026-09-30T14:43:55.272Z" }, + { url = "https://files.pythonhosted.org/packages/7c/52/9af10e80ac16b0fcc2123f9cbd5e7afbd0fd5075bb7a607c592258a39cda/cryptography-50.0.2-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e", size = 4376724, upload-time = "2026-09-30T14:43:57.24Z" }, + { url = "https://files.pythonhosted.org/packages/71/37/6202e488cc1eb625ea110c292c6bda92823176e023f427d8d5660ce8d632/cryptography-50.0.2-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04", size = 4737859, upload-time = "2026-09-30T14:43:59.541Z" }, + { url = "https://files.pythonhosted.org/packages/8f/30/e86d7d518489b0ae2497091a35287abcb1a2ce4037837a34afbe9b1d6964/cryptography-50.0.2-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc", size = 5324103, upload-time = "2026-09-30T14:44:01.901Z" }, + { url = "https://files.pythonhosted.org/packages/d3/69/2c833a049475e0a3444e94c7d0aca0aa51d166374a449b09e92ac98138de/cryptography-50.0.2-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079", size = 4752576, upload-time = "2026-09-30T14:44:04.545Z" }, + { url = "https://files.pythonhosted.org/packages/6c/5d/906970b83bbfc1f5bbfb677a143c181f2801f23b6a7204a3b47c42c97e65/cryptography-50.0.2-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51", size = 4870819, upload-time = "2026-09-30T14:44:06.884Z" }, + { url = "https://files.pythonhosted.org/packages/68/e3/f2298d3bb55e0c4a91841ec4d01b3f020ba8c5fbf15ccdcc6dcf03f97025/cryptography-50.0.2-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93", size = 5030152, upload-time = "2026-09-30T14:44:09.443Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4f/adfc442765721292fff86d314ce385d3249d22db42295c0dd057727b60f3/cryptography-50.0.2-cp311-abi3-win_amd64.whl", hash = "sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c", size = 3824692, upload-time = "2026-09-30T14:44:11.671Z" }, + { url = "https://files.pythonhosted.org/packages/ce/cb/52eb3770c0d0be2702a98c6e96065ddc0a2877cf0845aa9c23397c142cd4/cryptography-50.0.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f785f6161f202ab04d8ca194158968798e480ca058943907972da5f12e2881e8", size = 3892731, upload-time = "2026-09-30T14:44:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/19/8e/aa1fc533d4546b127b45de8aa024eb5933d23eff9debfe25931e56861095/cryptography-50.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0ecbc5652bdb6fc9eaf89a7d196e20941adfe812f43bc4ca05d9150496821047", size = 4710431, upload-time = "2026-09-30T14:44:15.427Z" }, + { url = "https://files.pythonhosted.org/packages/6a/64/72bc3f75176e7e406b748a3e3830432b8c51297b38368713df04dc04898a/cryptography-50.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ab50ee449bf968271e820086f10a33d101dd060370abc10bcd22279be2656539", size = 4694824, upload-time = "2026-09-30T14:44:17.69Z" }, + { url = "https://files.pythonhosted.org/packages/4e/c6/62c77550edfa5ca3f14bf44a1e6739b9fa09d6e998a11d97ed8213bccc98/cryptography-50.0.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a9f7355e6fab51f6c369b86fb7571cffa05edee2c2121e0380a37fb9ac1cd5c1", size = 4716967, upload-time = "2026-09-30T14:44:19.661Z" }, + { url = "https://files.pythonhosted.org/packages/f4/37/cce70f150c432914460157a6ecc161752e053aa5ec0ef3b3f7dc6e31039a/cryptography-50.0.2-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:94e5e9f108ee10471288214d3d233fbfbb492840a8457eb85178d643ddeb32c7", size = 5328676, upload-time = "2026-09-30T14:44:21.744Z" }, + { url = "https://files.pythonhosted.org/packages/aa/9a/6f2f0304d634ceafdeaf23e84537336664ac419b5d07611675c2ad3f6b7a/cryptography-50.0.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:241449bf940a5d27309bd317e6f9a2af6932113818bb2b8f5c59ddc7ef16da18", size = 4727698, upload-time = "2026-09-30T14:44:24.178Z" }, + { url = "https://files.pythonhosted.org/packages/1d/de/66bcf9244d118663b2e1aaded8990f4640e3d7b7411870a5765f252074d2/cryptography-50.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:d8947001be83df1394050758ce0e745dd74fb134eef0a4b5124208dfc3a68c37", size = 4354821, upload-time = "2026-09-30T14:44:26.263Z" }, + { url = "https://files.pythonhosted.org/packages/bd/e6/db28a28c7b6c676addce89136de3d8db49ea825a8c863472e36e42ead4ad/cryptography-50.0.2-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:4a20ce1e5cb4284a86692fdcba7cb8754185c6b2e5c56fcef3751cf451d3cdc2", size = 4716748, upload-time = "2026-09-30T14:44:28.447Z" }, + { url = "https://files.pythonhosted.org/packages/30/96/01546c7f69ea0e2ab790a2e4f0934a4052fb9b388147fbf83c2fd72f1e57/cryptography-50.0.2-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:84f964e537f916e2cc85199e5a88742e964939b575ac8598b3f9d6cc416cdaf1", size = 5285085, upload-time = "2026-09-30T14:44:30.704Z" }, + { url = "https://files.pythonhosted.org/packages/6c/01/03263395f74d50b071e9e66daace3f8bef80493e5d410726f2ba8554736b/cryptography-50.0.2-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:828d49b0ff5a0e3975865571c5d91dbbdd0d38d8289b249a163e9425413a5e05", size = 4727268, upload-time = "2026-09-30T14:44:32.92Z" }, + { url = "https://files.pythonhosted.org/packages/eb/94/2bfe8f29ec0cc9c0d99359c4161adf32858e4934b72c6d100d2ac0bbe962/cryptography-50.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:deb9fde5c60e437ee4821bc9bc39ff31b42135c27e1dc61ef0a629389c1de62e", size = 4849503, upload-time = "2026-09-30T14:44:34.969Z" }, + { url = "https://files.pythonhosted.org/packages/54/44/e80651ecbf0e42b62e2bb5f5768916e07eea72e1297338956a61df361f88/cryptography-50.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8c71ba2cd31fc93748c38e1b613200ff1c2665cbfd5341fe3a61cfde35a1430e", size = 5004057, upload-time = "2026-09-30T14:44:37.064Z" }, + { url = "https://files.pythonhosted.org/packages/f8/cc/1d33befb3cd7ea7e77d2d73f43f2066471da1b21f24a6156efcaabf6d2e8/cryptography-50.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:78198641e5be9521beea5aa782bb551a58068d10e6eb04c9c680c1b69f2e7d45", size = 3795868, upload-time = "2026-09-30T14:44:39.71Z" }, + { url = "https://files.pythonhosted.org/packages/2d/49/93f6a6e7a87c9aa68d44d3e1cdb5fe8f60c90d5d2f46acae9a56892816b8/cryptography-50.0.2-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:edc3342adf8f697fc5f59c887a304356f147b397809440ed64e2fa6af2f50f37", size = 4133708, upload-time = "2026-09-30T14:44:41.807Z" }, + { url = "https://files.pythonhosted.org/packages/8c/75/32ac2a56243d778805c16ca6a32b8f74fb757df7e28d7ecb560afafb59cf/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a", size = 4956267, upload-time = "2026-09-30T14:44:43.693Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a4/2c8d734e43d97f0842ee9f1b7b4bfb3d0cf5e19edebf43c2afe6675c2320/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67", size = 4966465, upload-time = "2026-09-30T14:44:45.769Z" }, + { url = "https://files.pythonhosted.org/packages/c2/58/ee288c829a6f41f6235ae9dd33d82fd19b45442b65b4c8a3da36963d9f7a/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_aarch64.whl", hash = "sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc", size = 4959356, upload-time = "2026-09-30T14:44:48.211Z" }, + { url = "https://files.pythonhosted.org/packages/92/20/9ded6d51ddd9897f6b6e81fb9ebea7951d7cc5d6c890b0ed8abf77a51a80/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_ppc64le.whl", hash = "sha256:b13478603dcd0a2479ff8e87e2c19a7d525734686fe3c49542472293a204212d", size = 5548822, upload-time = "2026-09-30T14:44:50.86Z" }, + { url = "https://files.pythonhosted.org/packages/02/a8/8df951850d6b31d2a00218f19e2b3f999523437ed7a819df7fa427942fca/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl", hash = "sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7", size = 5001199, upload-time = "2026-09-30T14:44:53.379Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f9/36b3022218ce75b7cdf068fb95f809f9bd0d820e4955ef43b90c255cc7ac/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_31_armv7l.whl", hash = "sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408", size = 4629333, upload-time = "2026-09-30T14:44:55.635Z" }, + { url = "https://files.pythonhosted.org/packages/8c/72/20f99a219f6af47cdd1cbd978c243b92d71496e168a746138af44ded4f29/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_aarch64.whl", hash = "sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b", size = 4958822, upload-time = "2026-09-30T14:44:59.639Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/196f112617fb08eb4d608a2a6c422373d46f9cc2857f38fc0667033c0899/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_ppc64le.whl", hash = "sha256:cbc8738fd8526d80f35cb3a40d41f41a2e7030bb3b18b09a6778ef63d291c2fd", size = 5506351, upload-time = "2026-09-30T14:45:02.267Z" }, + { url = "https://files.pythonhosted.org/packages/24/95/83378121ef3eaaaf71d4b781577ff794acb39b9e1b87a3f156898c8497ed/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_x86_64.whl", hash = "sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c", size = 5000859, upload-time = "2026-09-30T14:45:05.009Z" }, + { url = "https://files.pythonhosted.org/packages/22/f7/70fd7ae4d1dbfa7ba29b02e1b9068771519a86027756510b700ce81086a8/cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be", size = 5092151, upload-time = "2026-09-30T15:29:15.932Z" }, + { url = "https://files.pythonhosted.org/packages/d4/be/688367b74de86984bd58d8efacfc7c9e68b89a6a22ced0fb4f38db50254a/cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020", size = 5286120, upload-time = "2026-09-30T15:29:18.309Z" }, + { url = "https://files.pythonhosted.org/packages/39/d1/55f8a3f2ef5d1529e16835ef10cf0fe3d559ce237b46dddc440c0bba3649/cryptography-50.0.2-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:c423ab384a46c4dff7217b2ea5ba2e11cffdeab6441acd04cf65a369caf0366c", size = 4111557, upload-time = "2026-09-30T15:29:20.155Z" }, + { url = "https://files.pythonhosted.org/packages/23/ad/ac987755d00e1e64273760228d2635ae38dae2be83e3c6e0d3289d91dec3/cryptography-50.0.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2", size = 3943588, upload-time = "2026-09-30T15:29:22.265Z" }, + { url = "https://files.pythonhosted.org/packages/d5/8d/6d585339bedf85d45044c85d8412dac53f2bb6f918e8b7777efba1787844/cryptography-50.0.2-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd", size = 4756166, upload-time = "2026-09-30T15:29:24.58Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f1/1c1f6874e8550cfddd4b688ceb38cefb6ed15ceed224d56f133f3d88c214/cryptography-50.0.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767", size = 4749145, upload-time = "2026-09-30T15:29:26.807Z" }, + { url = "https://files.pythonhosted.org/packages/c1/63/61b15dc1a8de03fe0adbe3fd7608b3ad5c73bf50993bbcb1faaa930afe33/cryptography-50.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454", size = 4763638, upload-time = "2026-09-30T15:29:28.588Z" }, + { url = "https://files.pythonhosted.org/packages/fc/35/b345bdfa40c9126df1a9d33236aa98418367931b8725f84fc3ae2b98dc59/cryptography-50.0.2-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd", size = 5382217, upload-time = "2026-09-30T15:29:30.589Z" }, + { url = "https://files.pythonhosted.org/packages/4f/87/ef344a9e616871f2519c22d6afcda79ddd5d35e9592d95eb6e677608d055/cryptography-50.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5", size = 4781387, upload-time = "2026-09-30T15:29:32.605Z" }, + { url = "https://files.pythonhosted.org/packages/90/5b/f2fdb13cd0b96f6f932c8627bb292a45f11c64d21620a8e120aee9a3b848/cryptography-50.0.2-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107", size = 4403790, upload-time = "2026-09-30T15:29:34.374Z" }, + { url = "https://files.pythonhosted.org/packages/bc/ce/7e4f662b1e3c393513569e402cfc85ac7da0bd3d5435e122a3140219eb2d/cryptography-50.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602", size = 4764319, upload-time = "2026-09-30T15:29:36.149Z" }, + { url = "https://files.pythonhosted.org/packages/3c/3f/86ff33ce34cc0de6847fb96e035a1a760d81652e38643f617c02ad32ef7a/cryptography-50.0.2-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227", size = 5338560, upload-time = "2026-09-30T15:29:39.053Z" }, + { url = "https://files.pythonhosted.org/packages/40/cf/6b5c8e2fd9202d98988ab7cb5cc5c991704c4ad55f492ff408e4969f83f1/cryptography-50.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c", size = 4780973, upload-time = "2026-09-30T15:29:41.251Z" }, + { url = "https://files.pythonhosted.org/packages/10/bf/8d6ebc7dded797bd0f0160d52188021211f011a2b164ef0ae1dac4587465/cryptography-50.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e", size = 4897738, upload-time = "2026-09-30T15:29:43.106Z" }, + { url = "https://files.pythonhosted.org/packages/d4/aa/f3f6e0de7e6253b8baa8b2d8fb9d50924fa75cee3d4624bd4bc1208ee923/cryptography-50.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94", size = 5058280, upload-time = "2026-09-30T15:29:44.827Z" }, + { url = "https://files.pythonhosted.org/packages/f6/b6/a1faf3a27ae9405fb34b1713cc73b2d8a26b04d5c561578fa2e6ef3e5bb9/cryptography-50.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de", size = 3854095, upload-time = "2026-09-30T15:29:46.782Z" }, +] + [[package]] name = "debugpy" version = "1.8.22" @@ -493,6 +556,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4d/dd/8bc67e7d5ffc1d88aa5af511189dfb76dc4e1e5b808fab186146ef2663e5/executing-2.3.0-py3-none-any.whl", hash = "sha256:736e859c9f8701f11fcf516856f26f562e04776387824b43a35a1dfe21c84122", size = 29393, upload-time = "2026-10-10T14:06:02.777Z" }, ] +[[package]] +name = "fastexcel" +version = "0.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ab/16/d3b4465e1c32736ada7e1bc5a11334f3b38d747074aa01c60877d01dff81/fastexcel-0.21.0.tar.gz", hash = "sha256:07313c1267ab47ba639abf1122efd5985a1fb08efc996194f422ab17f06149c5", size = 61036, upload-time = "2026-08-19T13:00:20.184Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/98/461c22faa286d7635343fcfbacbed4edf77d98f06fb4426e646ae5438d66/fastexcel-0.21.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c3e7ab5d8c8b6c5a787aaf2b64604bd8b93b94694920a2ed731ea556a81d9a35", size = 3421831, upload-time = "2026-08-19T13:00:07.163Z" }, + { url = "https://files.pythonhosted.org/packages/69/ff/a6b1b97a94bbcc0d64b946e831ff937c2c803b019a7600fc69f953c38370/fastexcel-0.21.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:768b663728cb5f29e159428fdf3a3f74e379534c2f0304b300bd95039d482abe", size = 3264928, upload-time = "2026-08-19T13:00:09.133Z" }, + { url = "https://files.pythonhosted.org/packages/a8/a1/27454838aca7921826dd02be3828a20fcaaa36e641762bf070642c8ad65e/fastexcel-0.21.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c6e66906fe3b9f68f94c4c94e2ac21b6eebd862b703983c8e0c009f91c71754", size = 3719994, upload-time = "2026-08-19T12:59:50.076Z" }, + { url = "https://files.pythonhosted.org/packages/30/b8/2f5de2ec4026aa2e121a5da3d25b1d20f653bffdd569dfb74df6732ab99d/fastexcel-0.21.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ddb458fecbbf1804c0952155fb99d18025d86e345b57a5435e0553944f25578", size = 3789119, upload-time = "2026-08-19T12:59:52.278Z" }, + { url = "https://files.pythonhosted.org/packages/5d/b2/1e08ffca9481fa2103409a9bef52a91f0963867b4ea649a3d9e8f5c45554/fastexcel-0.21.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0376944edf90c98008b49b200f7354122ba9abac6c21bab76487655738b041b7", size = 3895258, upload-time = "2026-08-19T12:59:54.374Z" }, + { url = "https://files.pythonhosted.org/packages/6d/68/4f0d0b5d41c9fe22d45ec2b8412566cb79fbd4f412b6f33a7f60a302c1e8/fastexcel-0.21.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e919a4eaa15330341744cfee33d1f87d041d08228ce68809790e3738e80811e8", size = 4047752, upload-time = "2026-08-19T12:59:56.424Z" }, + { url = "https://files.pythonhosted.org/packages/8a/88/6879abe39db93b2c1939fe146d1335d95c30e961c2807f5bc516d4e305e1/fastexcel-0.21.0-cp310-abi3-win_amd64.whl", hash = "sha256:e1db4666a0790b48c76bb5a43cda06ffecebb22706f9ac6b3f07bcb0e7336134", size = 3318648, upload-time = "2026-08-19T13:00:14.784Z" }, + { url = "https://files.pythonhosted.org/packages/f3/03/5c8c97b47289bead5a3ba0b6cba01d27377b857446c65918c43e1b008d94/fastexcel-0.21.0-cp310-abi3-win_arm64.whl", hash = "sha256:86af0a1e3c3d8657916ea434f11636df4e4b49e0cf665b4ea39349a83d4ca3c8", size = 3035704, upload-time = "2026-08-19T13:00:16.64Z" }, + { url = "https://files.pythonhosted.org/packages/74/9d/ef3dd2022d943620653f65fd160f81be27c576a54b9ecd26cd1731da365b/fastexcel-0.21.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f6cf28f5f3fed1f34aa15bf021d2c04bf947720df70f54b131258c913bc3b4cf", size = 3419154, upload-time = "2026-08-19T13:00:11.145Z" }, + { url = "https://files.pythonhosted.org/packages/e4/82/763ecd88db11d6f98b78aa1b951c2a259d84d6d285af2f6dd525948062f4/fastexcel-0.21.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ef2a6953e8350966d32632e3bc064edaab64ea2899f2027e564269fa7d75fb58", size = 3251184, upload-time = "2026-08-19T13:00:12.965Z" }, + { url = "https://files.pythonhosted.org/packages/7c/0d/fce85550c9138e5e2517b33d9ec000222710b3bdc6563a6c91fddff3eb52/fastexcel-0.21.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f8fdbfd80647714a2b3d49de2517d0466f6c046aa215c16fb569c48aef8d0ee", size = 3711549, upload-time = "2026-08-19T12:59:58.613Z" }, + { url = "https://files.pythonhosted.org/packages/ac/47/b768f8165e16f15345b5eec06507b33e88cc8934d5e9d0e602d26bfdba8a/fastexcel-0.21.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47c6f42b3b82a158e4e6c4e1ed53ba0b96cec132d1fed828c8411e6f6ba5caab", size = 3778980, upload-time = "2026-08-19T13:00:00.807Z" }, + { url = "https://files.pythonhosted.org/packages/d1/e8/3d9626a0b1e50704bfc19df2f69e2b3e7870f43e6cd8509565b5aa32e5b6/fastexcel-0.21.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bce27f751cf1661f823088e89c11375448d19e425e3c3aa993c356720305c873", size = 3888071, upload-time = "2026-08-19T13:00:03.134Z" }, + { url = "https://files.pythonhosted.org/packages/a7/ff/23f43ec08ac44a02798508593f2af5c84bbad58db17da3237428577f5b1b/fastexcel-0.21.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1a5742e598516734740ef4142cf3328d6ef6c8e43947d9a66d6a91a5d9bfa3ec", size = 4041849, upload-time = "2026-08-19T13:00:05.103Z" }, + { url = "https://files.pythonhosted.org/packages/13/90/4b2614123e185f20e386695771898c97a469f39129472db731a2c3d248ad/fastexcel-0.21.0-cp314-cp314t-win_amd64.whl", hash = "sha256:fe52f6053aac6ff3b8cc879052b671af9cb3ada16853b1c8b4bcac44574e4c10", size = 3311557, upload-time = "2026-08-19T13:00:18.614Z" }, +] + [[package]] name = "fastjsonschema" version = "2.22.2" @@ -1481,6 +1567,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, ] +[[package]] +name = "pdfminer-six" +version = "20260107" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "charset-normalizer" }, + { name = "cryptography" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/a4/5cec1112009f0439a5ca6afa8ace321f0ab2f48da3255b7a1c8953014670/pdfminer_six-20260107.tar.gz", hash = "sha256:96bfd431e3577a55a0efd25676968ca4ce8fd5b53f14565f85716ff363889602", size = 8512094, upload-time = "2026-01-07T13:29:12.937Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/8b/28c4eaec9d6b036a52cb44720408f26b1a143ca9bce76cc19e8f5de00ab4/pdfminer_six-20260107-py3-none-any.whl", hash = "sha256:366585ba97e80dffa8f00cebe303d2f381884d8637af4ce422f1df3ef38111a9", size = 6592252, upload-time = "2026-01-07T13:29:10.742Z" }, +] + +[[package]] +name = "pdfplumber" +version = "0.11.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pdfminer-six" }, + { name = "pillow" }, + { name = "pypdfium2" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/05/56/6f450312ba05a27d7713b73857c1a25100dbda04fbc1331b13fb227a607d/pdfplumber-0.11.10.tar.gz", hash = "sha256:b95b2d28c66efb0a794a83b88c6c6aea5987532a445d20a1cbcfa657022e6e57", size = 102892, upload-time = "2026-06-15T03:31:31.035Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/9a/07d658e1e7fad860f1c541ab941348125dbdab773be3a0afaf32361866c7/pdfplumber-0.11.10-py3-none-any.whl", hash = "sha256:7741ea81bf165b474b153e6789d10d18e06b6ddcf3ec84289c3ef2fed6802580", size = 60047, upload-time = "2026-06-15T03:31:29.702Z" }, +] + [[package]] name = "pexpect" version = "4.9.0" @@ -1493,6 +1606,82 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", size = 63772, upload-time = "2023-11-25T06:56:14.81Z" }, ] +[[package]] +name = "pillow" +version = "13.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/a4/9cc30690a394a8305e8be4c9036aa16d0dfddadd09079860b582f562496d/pillow-13.0.0.tar.gz", hash = "sha256:cdcd1ba258178f7356d74170c9a81ba4eab6a5d55748bc402a5a2a6fe6be3510", size = 47058631, upload-time = "2026-10-16T22:18:09.718Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/b8/58d79332229482eb0d27c82973850808c19d37e5bdc14443d3417bd373f5/pillow-13.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:67c8d44ad89fa7eb88de5d6ef6199b1e03dd24974f85a67639fc8badb801fe79", size = 5568643, upload-time = "2026-10-16T22:15:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/39/4b/f2ed53d83aa44ac19599ac0a54897f867fafb95f406980d1e24083f8cbe3/pillow-13.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc481f742ccda20e9ed760b8f978dff50f2a4e9bc253ac167fb0a05b0783ef52", size = 4852329, upload-time = "2026-10-16T22:15:46.284Z" }, + { url = "https://files.pythonhosted.org/packages/3b/bb/9e30e8ed18c20800ee327a12107d5249ad8dc444867b3adfcc21654e3c71/pillow-13.0.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8bf295fe459343bc6093c8e5e0df60bfa189e0fcb775a22e2c2488fdeccc6c0d", size = 6519309, upload-time = "2026-10-16T22:15:48.14Z" }, + { url = "https://files.pythonhosted.org/packages/e6/99/fdf65cc539e266ab93546c261bdf98411fe165c6e4e13f9dbf597097192b/pillow-13.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42d63be3af4f9b951b199b0a9d23365d37ab14687e6d77b74940432c8515c4ca", size = 7144542, upload-time = "2026-10-16T22:15:50.093Z" }, + { url = "https://files.pythonhosted.org/packages/26/9b/f38b7c4d8f599537e2d1a6d546cec9beef7536ff8f5c49a3f9c78179d8ac/pillow-13.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4bd024d567eca931b668978d672dba914d4b8e2628e2c3073a2ad178de8a2620", size = 6618042, upload-time = "2026-10-16T22:15:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a8/7c/45ee28531f89ee4582967d8a17f7a26d71ca64cd902b93c269590cd60a00/pillow-13.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23edf4ec4b6f4f240e8df6e4985e9ee2ced86474632433e55fb1e2548744bf40", size = 7228137, upload-time = "2026-10-16T22:15:54.009Z" }, + { url = "https://files.pythonhosted.org/packages/01/bc/9c2a969cb1b70201d04fa17fd5aedf02a732870b8da86bceaaa30b0ca385/pillow-13.0.0-cp312-cp312-win32.whl", hash = "sha256:02464da86f561f410e13b061ec37c3068a9cb557a095494157742d07b3c24a27", size = 6515423, upload-time = "2026-10-16T22:15:56.25Z" }, + { url = "https://files.pythonhosted.org/packages/ff/cd/73a79ecdc822076002eb24b5e1e4a530b0bb5310fcb6d239abbec877110b/pillow-13.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:ff3fe9eb621fbb9db4c4be33ab946f6bac5f98ab50ca8d3260503653656c961b", size = 7272166, upload-time = "2026-10-16T22:15:58.597Z" }, + { url = "https://files.pythonhosted.org/packages/4a/a3/be5a1236efd085b3984e4305da8f179ae9dab80c991d2ad78f2bd5c4c843/pillow-13.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:c3bc8ee0271387e0b1e3ad0ff893645990a656e79bf9c42f4f734a3642b11d51", size = 2698943, upload-time = "2026-10-16T22:16:00.637Z" }, + { url = "https://files.pythonhosted.org/packages/82/e2/9132d38df320e8099e2642d9fa538f18a7dfebf68a875a31725dc2a115aa/pillow-13.0.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:e98b4f91f99b24b1da72e10621ba181f7281922384a7403094debb4199de6c00", size = 4190620, upload-time = "2026-10-16T22:16:02.527Z" }, + { url = "https://files.pythonhosted.org/packages/cb/be/436dac0996876f65ef1916ffb711c5ef4af2ebc57414b6f16642e07f6b47/pillow-13.0.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:34fb3db1149c879343861831728c243c432d4a5afc8f1878ca7612ef8de2a033", size = 4290286, upload-time = "2026-10-16T22:16:04.861Z" }, + { url = "https://files.pythonhosted.org/packages/60/16/7c1007162e4169ea0d5bec47e2e0b2174f56c50cf5b8c02a1dc23e41037e/pillow-13.0.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:2d50500a4a3bc6469adcdb319cf05d688b08ca3bf0f22a05339f3b25b433a599", size = 3723169, upload-time = "2026-10-16T22:16:06.956Z" }, + { url = "https://files.pythonhosted.org/packages/08/bc/578ab2eaa04705a8978bd9be0fe15c3d43a8fc95097c4f7e39540e79e4fa/pillow-13.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ef5fe6b7db3f501c56000ba8fe7c7e1a35ca1d45824fbeb95a67daa85cff4dbe", size = 5568469, upload-time = "2026-10-16T22:16:08.664Z" }, + { url = "https://files.pythonhosted.org/packages/98/c8/ccede3dcc4f274b80fb49199eed1ef34ba05566c8b71976b9d1f111da2e3/pillow-13.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f11a3c1358bb3c68012a7b1f41b223c3db9f7bfde90b481df028cfe1c3363313", size = 4852187, upload-time = "2026-10-16T22:16:10.693Z" }, + { url = "https://files.pythonhosted.org/packages/fb/3c/cd0770e96d20bfd990db024bcbd43897ff3e88da1f4782cf90ea231b1d4f/pillow-13.0.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19b66ca0a4d25cc0200b381ce824c7ea090db61427315e73cfb05dbf1b35c369", size = 6515908, upload-time = "2026-10-16T22:16:12.896Z" }, + { url = "https://files.pythonhosted.org/packages/ef/fe/f5cdaa2d1c4086e757430f368c271a58ee29cfe7378e7ae7b9d964015c25/pillow-13.0.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5f56499c1a74d2b862beb0c33b8016f99d095858455df9d21583bbbc1c15b582", size = 7142544, upload-time = "2026-10-16T22:16:14.844Z" }, + { url = "https://files.pythonhosted.org/packages/fb/b5/a54f6d44adbcd23737c8c12953f42be107e4cca64e247c0ca3a5b563061c/pillow-13.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2a14cd73d5e6c927f644b0110ee81f59c4ef506259ec9183904747f2caacc81", size = 6613097, upload-time = "2026-10-16T22:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b4/9e6c1207c21b2e91db0d15c16d30f3701ce5add1bd4da0e787761e66cb6e/pillow-13.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:08b826a9bf1265c76de04a42061d61ea6b109418b7826f28ff1ecdfaa26c5502", size = 7224846, upload-time = "2026-10-16T22:16:19.195Z" }, + { url = "https://files.pythonhosted.org/packages/78/9f/d3c3cb0518f3dd1eee0ef501535f53d15597a53c5c4118d178086143397a/pillow-13.0.0-cp313-cp313-win32.whl", hash = "sha256:12b0fa15879fe5a9d9eebe66e4e2d74128a773b1a4448010c9c714f45e04b9d8", size = 6516013, upload-time = "2026-10-16T22:16:21.389Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ed/1b16fb66a37e1b9e41370b93b2c614c7da761a5b2a41dc333f7af4639962/pillow-13.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:9dda0788a9932776a75d8bcc020076edc308ce2508b51f6739a45e3158b3fc42", size = 7284777, upload-time = "2026-10-16T22:16:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/7d/8d02eba9f7ecc87f8e90f731e032cc4773802f2a843763afd0c66a535ff4/pillow-13.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:40a266371ece581e5de13b6635ef78d7487824af80561b6ca27694a88c19bc0f", size = 2698826, upload-time = "2026-10-16T22:16:25.382Z" }, + { url = "https://files.pythonhosted.org/packages/8e/45/fbad9c7f5e2f1745c373a434d46072bc84eddfefaa28b852425ac957fda6/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:44629d4498c9cb6b2051806155f22141df0d8a929c56134b166655232dfb2f8c", size = 4190721, upload-time = "2026-10-16T22:16:27.524Z" }, + { url = "https://files.pythonhosted.org/packages/a2/bf/2074902a259a90692c5dbf132885d21671644cbbd3f690047516243a55db/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:bf97ba1b5542050e9b0c74e632323b6d8136cc99d64e193abebee22fe6013c76", size = 4290236, upload-time = "2026-10-16T22:16:29.562Z" }, + { url = "https://files.pythonhosted.org/packages/ff/a5/3d1bb93b8ea38d15d21356c1b8e9f50971ac592aaa248f7fb71739094c0e/pillow-13.0.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:aa694c49787040943570d61c9b8fc01271bd2a2cfdd55f0d896f8aecdce1a981", size = 3723093, upload-time = "2026-10-16T22:16:31.249Z" }, + { url = "https://files.pythonhosted.org/packages/84/a1/be060135d85187b25fa8b2573786c0e45c62ae27527fff5257675eb97f4a/pillow-13.0.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:49c12af515073e9bf511e4ef736743bf70d531a643668722279314b937742ba4", size = 5572253, upload-time = "2026-10-16T22:16:33.103Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f4/0bf6f1ac9ba7ee2ffe9914aeb410a1035a0c1a3c42f122388d7fc950bc4b/pillow-13.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d9b0a2bee3f3e03e9f29057aedb807dbf1e1d96ee1b9d0f422d9db6c8583271d", size = 4852269, upload-time = "2026-10-16T22:16:35.102Z" }, + { url = "https://files.pythonhosted.org/packages/83/16/b603274617b3e6728da3a0133d4fba1ea8d7fbe0c4ea5fef9cbffa398b8b/pillow-13.0.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5933fcea5a5ba64c686f55c886a115cbfd92f2900460784a1e79909fdbafd7b6", size = 6516058, upload-time = "2026-10-16T22:16:37.228Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f2/1371707ce438ba9a43f9d24089828494ec6f9396e28cd7903aa704dd6306/pillow-13.0.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e91c2b1cbeae768c98752e67e717e283d68dae45717b41f2d91eaee648e37b7d", size = 7142574, upload-time = "2026-10-16T22:16:39.172Z" }, + { url = "https://files.pythonhosted.org/packages/b9/a9/62cf68587f910bc87c5951bcb6e9430c7643a65076adf5f18b4bf350e2bd/pillow-13.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f6299e494e46253d40c34653cac369412f44ca53b5f85d848d3f95b1ed0546c", size = 6614295, upload-time = "2026-10-16T22:16:41.117Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c9/9e9b48178649730299be0bafbf5f810a5b6977538f5a92d47e4ba5f5d608/pillow-13.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f9bd7d260d986f3553518c5d723544346fa48031f8aef3265d786aeb59845bc0", size = 7224841, upload-time = "2026-10-16T22:16:43.118Z" }, + { url = "https://files.pythonhosted.org/packages/8f/48/c44e91384b1b6c072a714a63d00608acc16a6a968735e478e4be4384e5ac/pillow-13.0.0-cp314-cp314-win32.whl", hash = "sha256:2e31c4e68cffd7b1f116127626813b82fcba19edf6b940a7b0c266d6484d0055", size = 6508694, upload-time = "2026-10-16T22:16:45.883Z" }, + { url = "https://files.pythonhosted.org/packages/50/e2/7ad17747517e4c813dca78257d6bb593466f8231baf1384bc98d92745edb/pillow-13.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:f7cbb7a64e7ec664775195e4f81923bc8a93d9451b963f844e59118f63147a7a", size = 7281830, upload-time = "2026-10-16T22:16:47.725Z" }, + { url = "https://files.pythonhosted.org/packages/41/9f/e5b51f2ce8ef4563215334035403ce61a244f93a2e65c24a940131618f87/pillow-13.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:a6aaa7871aebca081eb5c45f83e845bbd9712ae99fa9286dbfdcee835e63f4b4", size = 2695468, upload-time = "2026-10-16T22:16:49.515Z" }, + { url = "https://files.pythonhosted.org/packages/97/73/a53502f731f6c84805a4ce87a8881e86d07a65aee64728b900d112dc703e/pillow-13.0.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7b999816b5850a35ada08443e6e03945f1f6488144a83afe2d37b240a552aafa", size = 5578196, upload-time = "2026-10-16T22:16:51.291Z" }, + { url = "https://files.pythonhosted.org/packages/21/50/7e949959a56b5559c78ef172b70fc6025706acb289f3f648a15329e4fa6f/pillow-13.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:67871a13cf47275201553b536c025da777f997fa562a19eb798af4ad4b97c18f", size = 4857276, upload-time = "2026-10-16T22:16:53.399Z" }, + { url = "https://files.pythonhosted.org/packages/c9/52/43ded0ac08e77e39d17bd6d894162572201b5e8fdc4a022b14c55a1960a4/pillow-13.0.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8ca8f69b7c77fa96dad99f8638c1fc4f1c0bed4acaa1baf7a3c31bdb80a0ab1", size = 6607008, upload-time = "2026-10-16T22:16:55.525Z" }, + { url = "https://files.pythonhosted.org/packages/0d/2e/4a57ed216d5bf877387cba4d44c02aa87abe51ad2e06404aa6b597c4da39/pillow-13.0.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bce6a29c3898191f4c9998e4d4e8f69790caeabde46e54da4bc235ce5bde683", size = 7224281, upload-time = "2026-10-16T22:16:57.738Z" }, + { url = "https://files.pythonhosted.org/packages/5c/3d/2929b248e135752d63aaeabf53f6de797e905b7b6263f9c1e2adade20aba/pillow-13.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:816752a5ef76a80484c8c441c85593b7ea9304f6231fc5776ad1fbff7e773058", size = 6695600, upload-time = "2026-10-16T22:16:59.842Z" }, + { url = "https://files.pythonhosted.org/packages/bc/33/fb0da992cf084ed1ee6ee05e96b6191e35c351eb707ca2df874feb24ce47/pillow-13.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5bdb38ba220b61b4ec31dd1046dea6b016d3c6850a23f43ed8b58332b2fc668e", size = 7300589, upload-time = "2026-10-16T22:17:01.686Z" }, + { url = "https://files.pythonhosted.org/packages/bb/af/c30dd9ef6a3dce84ecb1b2393cac629047a32e66279056c6cb46c3ffd65c/pillow-13.0.0-cp314-cp314t-win32.whl", hash = "sha256:06903aac074d00c988e77a1fc435286dc498506d4ca8d6f69df9babddf4ecc15", size = 6517204, upload-time = "2026-10-16T22:17:03.722Z" }, + { url = "https://files.pythonhosted.org/packages/85/d5/ef3a8ae7609c8ab26570abf18194882534b480d245a9148b895a2262c004/pillow-13.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:9a635068d9793c7f0d929f39f18b58f1bd86c236e8e7b64167b86af9232ebca0", size = 7286695, upload-time = "2026-10-16T22:17:05.543Z" }, + { url = "https://files.pythonhosted.org/packages/84/3c/0861db0c69a03689ebe3e7599d35668a414deb5aaafd8255215e4e62e3d3/pillow-13.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0d1a9cf7bc6917538b20f8d06b9ab2147e542f68849e96fc58eb66bdf6acc692", size = 2698098, upload-time = "2026-10-16T22:17:07.674Z" }, + { url = "https://files.pythonhosted.org/packages/6e/eb/e86a41962a3a31ad495a3d6a912d9002b01bba4a6fa71c28703abdca7f69/pillow-13.0.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7d5e097eabd5d4915c27ac9f6e9325bb1e01a7c94d7348b5c4451d3fc30e8521", size = 4191213, upload-time = "2026-10-16T22:17:09.449Z" }, + { url = "https://files.pythonhosted.org/packages/dd/04/7e7445e706afec84ef06dc25b2c77f9cecff242c23730c62928ad1089d4a/pillow-13.0.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:51ebb529fc5e8abdd55302f6c3f8943037348ec2384329a3e55677f69ae2fdfc", size = 4290650, upload-time = "2026-10-16T22:17:11.196Z" }, + { url = "https://files.pythonhosted.org/packages/46/85/d534089068ac40c0f61f650389a296c812eab745a8a733f81feec39ae955/pillow-13.0.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:c87a04e7f152ee4db5dbe15f0bfa3fe628a1aeedbe66b81c43b87f52c813f9ac", size = 3723078, upload-time = "2026-10-16T22:17:12.772Z" }, + { url = "https://files.pythonhosted.org/packages/ce/2b/00b5c3b401df538b5dbe311d3899730fae5ba1e21cb8b80a128887ac6747/pillow-13.0.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:20722cd5375533ad754053363a80b8fb30c79d37930a6f7228030e237ce3bd5d", size = 5572292, upload-time = "2026-10-16T22:17:14.539Z" }, + { url = "https://files.pythonhosted.org/packages/72/f5/a6b0610f946e9b54ae888ba7ad29d6e2f8d9b11f69e212d70416b7186c09/pillow-13.0.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:04a63720f79e22fd54d897c233a68314c50ca0603360f54ce2bd5db8f0f43782", size = 4852222, upload-time = "2026-10-16T22:17:16.521Z" }, + { url = "https://files.pythonhosted.org/packages/1e/d0/c0ff25bb0427ec70f32aac995453321fc3c4494a46c7292ff02d9d260b04/pillow-13.0.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f8db54177035a2b5f717f94475b74907e12a96dc74f6d487f3966b0a4694071", size = 6527349, upload-time = "2026-10-16T22:17:18.534Z" }, + { url = "https://files.pythonhosted.org/packages/85/11/e46fa0bcc0fc117f1eacc206b3d51f04d583fab328b4b740672d0438d83f/pillow-13.0.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f36aa77f74b14a2103622b196402c71330e4e7cf4122ee45c854e9a01e0eecf", size = 7152873, upload-time = "2026-10-16T22:17:20.415Z" }, + { url = "https://files.pythonhosted.org/packages/7c/53/2bdeca705f3a58679fa9201233193c86a30238340ca29b3a152c3fcc719b/pillow-13.0.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:149a1c323ba366f90b48e6f1a6db6582eb50512df374588de56fc909abc0f0f3", size = 6626871, upload-time = "2026-10-16T22:17:22.361Z" }, + { url = "https://files.pythonhosted.org/packages/14/de/3829030f209722ade3753bb456b805a0e7f885cf885b483407365c82e9ca/pillow-13.0.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e892354a424b1e82a78ba16785822bdd40de06f19446a3d4a28047df1a4f8932", size = 7236439, upload-time = "2026-10-16T22:17:24.352Z" }, + { url = "https://files.pythonhosted.org/packages/44/b1/67fc37388382bc036a74f04da5bb6fcbf90ec3ac1f45893e06176c373fd3/pillow-13.0.0-cp315-cp315-win32.whl", hash = "sha256:8e459b613cf31f8980f584387b4fe47bf08e076e288d61aa8eba1f7fd4ddbb81", size = 6508688, upload-time = "2026-10-16T22:17:26.314Z" }, + { url = "https://files.pythonhosted.org/packages/44/05/fdb6a79aee10861165526b41b7b440e36ee1ff58613b2fbefe3acbf06d4f/pillow-13.0.0-cp315-cp315-win_amd64.whl", hash = "sha256:d4737f50d23ddd11d6685faacbb9252887dde0bc7997127757247983d74399d6", size = 7281107, upload-time = "2026-10-16T22:17:28.198Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9c/6c6f7dc2a566467baf121394e83de25ac287cffe34cc030726beb4be0224/pillow-13.0.0-cp315-cp315-win_arm64.whl", hash = "sha256:2310e2adfc13c6d457d7d0f2ffdd5c56a74a317650cf765887df8d515a647ac0", size = 2695513, upload-time = "2026-10-16T22:17:30.013Z" }, + { url = "https://files.pythonhosted.org/packages/23/16/e9120defdb8bff693b582d035312acc6f530cdc2bc5238a87f410efccd82/pillow-13.0.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:21c32061a5512dc13f87cc7303e120e4d77c2f70df89a35af0e3ce246e5ac57f", size = 5578129, upload-time = "2026-10-16T22:17:31.642Z" }, + { url = "https://files.pythonhosted.org/packages/47/22/76faf2bb2448f48a84cf894c1e2bb1090d7d2b1aa521ece131cb2f05d631/pillow-13.0.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:1c774b9c0ba0d0b356f71530677b39028878a2cae5a93e0a13779cfd17120f88", size = 4857428, upload-time = "2026-10-16T22:17:33.43Z" }, + { url = "https://files.pythonhosted.org/packages/b3/64/0861ea4e03c50b14af9ed9dd8b9f388ab42f445c623a9fcbab19aef7aa55/pillow-13.0.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20822140c9a2ba9c5e42df342b0a892ab2e102fc0e3db967a133bb9ce3f898bb", size = 6612582, upload-time = "2026-10-16T22:17:35.415Z" }, + { url = "https://files.pythonhosted.org/packages/5c/20/cd1e9b3f60a9f3d5ebe46da36955c26004092a501f7f2c42f9fcc26611bc/pillow-13.0.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ffea6b2986a797c84b6d1d27d4f43e0b83c73fc16281a930891cd237cf0d08fe", size = 7230176, upload-time = "2026-10-16T22:17:37.362Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ef/ffb0a2fc85690714fb7ac662956949645d57f15bf4426f520da361e7b6e3/pillow-13.0.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:05cd2cd39055d0b9650060e4513ef4a9a0aea870083950bd046a0d6490f93a22", size = 6703640, upload-time = "2026-10-16T22:17:39.313Z" }, + { url = "https://files.pythonhosted.org/packages/e0/c2/6402ee3ccb3363aeed8fdb3f9fe56703ac5377a539f0b81b53999d7e121f/pillow-13.0.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:c6fd031333dedcad7ce3e03d244093f43504f209e86925ebdb2b868fa4037630", size = 7309231, upload-time = "2026-10-16T22:17:41.021Z" }, + { url = "https://files.pythonhosted.org/packages/fc/96/d6b82e8b8c06ad143b66b12e0b31051e4bf489b4fabca1f390a14fe1ebde/pillow-13.0.0-cp315-cp315t-win32.whl", hash = "sha256:c50c80e3591e5cf9a8dfa6007d4f10732f02e32263850999286e56ccd650cfb2", size = 6517200, upload-time = "2026-10-16T22:17:42.886Z" }, + { url = "https://files.pythonhosted.org/packages/60/d4/0062e6005e9b270b165015aafd343364ac378ec5a685431ea42baad33357/pillow-13.0.0-cp315-cp315t-win_amd64.whl", hash = "sha256:adb3c2124c68e480f4632d06cb7caa3b48a51678475f483b4edb1c6dee29ea43", size = 7287488, upload-time = "2026-10-16T22:17:45.56Z" }, + { url = "https://files.pythonhosted.org/packages/e4/81/b8eb349746be7bce47dd95337c6634e84910af91b59aaa9159c034c614fa/pillow-13.0.0-cp315-cp315t-win_arm64.whl", hash = "sha256:4a25756244bdcce3b50580baadf320fa48297f0eeb8e114ba64139a40d4d0fc8", size = 2698073, upload-time = "2026-10-16T22:17:47.556Z" }, + { url = "https://files.pythonhosted.org/packages/60/da/f6d644e631c5b370c5d0b899f3ccadd2780aab070b29a110527cf4e1fd2c/pillow-13.0.0-pp312-pypy312_pp80-macosx_10_15_x86_64.whl", hash = "sha256:8619685cad8efc4c38c66d3b36aad854523297c43d748b3b6f0cc03c90ba3463", size = 5523188, upload-time = "2026-10-16T22:17:59.334Z" }, + { url = "https://files.pythonhosted.org/packages/e4/b0/2594f202d743a35d4fe80682bfd96f277e5ca7aecb27cf64ef5bfc1d000b/pillow-13.0.0-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:7e91e97a929232cdb828889ee6ad0e48d7f40f4ec38818f4e4a8968b2b1af22e", size = 4806808, upload-time = "2026-10-16T22:18:01.188Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ed/d504165ef79af6fafe25c76726cf30f544cc236002f7ed054f1d51adc4ff/pillow-13.0.0-pp312-pypy312_pp80-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93ff9c9c3663ce23aaa5d0b6b6abac55359e2a0c06c7e30f3771829af816be63", size = 5464358, upload-time = "2026-10-16T22:18:02.779Z" }, + { url = "https://files.pythonhosted.org/packages/7b/fe/a10aba5503ca13e9e009b8bd0a1a4ce5584bdce246245fd6f3a83701369c/pillow-13.0.0-pp312-pypy312_pp80-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e65dd6e75de67b2362a4fa3e16aa627595f65054dc4ec73d64680c84d592fa61", size = 6043514, upload-time = "2026-10-16T22:18:04.538Z" }, + { url = "https://files.pythonhosted.org/packages/c0/37/66b12dc476041fe523bca29c9d58fbf1ea92a4eaf73acfb9e993948f1191/pillow-13.0.0-pp312-pypy312_pp80-win_amd64.whl", hash = "sha256:567f20f1c75897d7663830da6ea8af719e03d712b1d9ee7f813319f9ece42e5a", size = 7366030, upload-time = "2026-10-16T22:18:06.638Z" }, +] + [[package]] name = "platformdirs" version = "4.13.5" @@ -1511,6 +1700,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, ] +[[package]] +name = "polars" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "polars-runtime-32" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/da/c27daf205a824f89f71172a75ad0098c5837a5a31ddeaac7f8737ea8574b/polars-2.1.0.tar.gz", hash = "sha256:9eee488d6e10773c028753dfc8dac647fd60a708108dcb818a733e85d33df253", size = 811508, upload-time = "2026-10-25T18:00:33.43Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/d4/e696a46c74ec8cac6434764deaf72f7edcc2e91c2145a9b3f61900a7aa7a/polars-2.1.0-py3-none-any.whl", hash = "sha256:63c50ce3533f5a03d17c3a2a24f1d0e0c7eda98fbde4268a3cba900d34060e67", size = 911474, upload-time = "2026-10-25T17:58:26.863Z" }, +] + +[[package]] +name = "polars-runtime-32" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/a6/46e09af693326735b1fba3a8bf5a725b57136fb1342e707e75d3113c826b/polars_runtime_32-2.1.0.tar.gz", hash = "sha256:c575f5f3cdb8a5f9be65477b9b4202b062593687fe62a0082add87fbd4571067", size = 3878004, upload-time = "2026-10-25T18:00:34.932Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/40/b424f3946c702ff48eef14e44d35df81a25a784f20f77355f39c746528fc/polars_runtime_32-2.1.0-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:de243b6e5a0b9f3abeaefe542136fcdc6dd8df5be3c988a40838d075314e246c", size = 52169672, upload-time = "2026-10-25T17:58:32.437Z" }, + { url = "https://files.pythonhosted.org/packages/fe/38/07066430ac655a8cab2f9e96ea1fec6ca1f20ec47844afe117662d43ace0/polars_runtime_32-2.1.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:317314d65949f1d58c4939adf83aa0d2cfaf8da50fd4a725a2de27bde2c10018", size = 47626883, upload-time = "2026-10-25T17:58:38.688Z" }, + { url = "https://files.pythonhosted.org/packages/49/a8/a773e20b451bbaab4d8267867b7c43a0c3aa67ebb6f781423a42a5a1af37/polars_runtime_32-2.1.0-cp311-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:777228cc8bf9cda36dee85ba74d13a25974ad4caffc5da93e10b1da9d810ce2d", size = 50033514, upload-time = "2026-10-25T17:58:44.261Z" }, + { url = "https://files.pythonhosted.org/packages/d2/e6/649c524252d32b24d0ba92278033823fa8e0040ee3748ce5afbeda107403/polars_runtime_32-2.1.0-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b838056cf365d7497971d9d9825c547766923338771c48c545b86ead2ebc572", size = 53863954, upload-time = "2026-10-25T17:58:48.737Z" }, + { url = "https://files.pythonhosted.org/packages/90/f7/2eb405bf5ead6373a33fa5dd4637fe996565a1f07ccce8911ed167673740/polars_runtime_32-2.1.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:04cfc3db1571b74ea4145944de7feceed4bfc9b26e5b2c265e27ad3b2d3866a9", size = 50204290, upload-time = "2026-10-25T17:58:53.33Z" }, + { url = "https://files.pythonhosted.org/packages/33/4d/d76cd42042605e079d6df91cc4fea09ed5f4cb836c72c952a17f585860a9/polars_runtime_32-2.1.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:af58b5f6e5cc44b578f2a93d752f87c135c3a53205b4360301bc3a9b89dfbfd1", size = 53794253, upload-time = "2026-10-25T17:58:57.99Z" }, + { url = "https://files.pythonhosted.org/packages/4d/4e/2a419d0def2908255dd1b7318c48430319859dc92ca156e2f81ff7bcd78b/polars_runtime_32-2.1.0-cp311-abi3-win_amd64.whl", hash = "sha256:f2fe32f48af2395cd6ec52a433a03970a69aab1dbb66a613729fd549ce92c904", size = 53926265, upload-time = "2026-10-25T17:59:03.13Z" }, + { url = "https://files.pythonhosted.org/packages/c8/1b/15ffba3264baf21d2f04c51f54b01159cb6f4fad4e84de1404be552d601f/polars_runtime_32-2.1.0-cp311-abi3-win_arm64.whl", hash = "sha256:0af30dd53a7725f703d0d21dd421a39e0165378bfaa41431008f286608376d71", size = 48636436, upload-time = "2026-10-25T17:59:08.476Z" }, +] + [[package]] name = "prometheus-client" version = "0.26.0" @@ -1818,6 +2035,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8a/d0/758afe732c2da502c051b4d03ff88daa3c0fc1a1e671e6c5a6318177bab4/pymdown_extensions-12.3-py3-none-any.whl", hash = "sha256:345493a8e3497e24382902903a1c43a28e34a52f2b7571a1c9641429b820ca74", size = 277597, upload-time = "2026-10-14T22:40:41.543Z" }, ] +[[package]] +name = "pypdfium2" +version = "5.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/d0/c81d3a7c2a9af37b817ace1de0acd40cf44d15f12407c5e86b3668364a5c/pypdfium2-5.14.0.tar.gz", hash = "sha256:c5f009b3157f10e97dceb55963f5910eff92feb00587ba10a76f12b87ce1a4b6", size = 376498, upload-time = "2026-10-04T15:19:19.835Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/03/79e89eac9d811e83d606342e129f5f39e168442ddf23b024fea4a7ee4762/pypdfium2-5.14.0-py3-none-android_23_arm64_v8a.whl", hash = "sha256:bed597b2cea3990164e43f9003f71db18959d0abd5d73adc9c176e7be2d84b98", size = 3453370, upload-time = "2026-10-04T15:18:40.79Z" }, + { url = "https://files.pythonhosted.org/packages/cc/68/369b80e408017b18eaecaa3c730bded07d90bfb65562215df200b56fb8e2/pypdfium2-5.14.0-py3-none-android_23_armeabi_v7a.whl", hash = "sha256:1951f0aed469150b13c62eabd501a9839e608ab9983ca8579be9eb73213b72b6", size = 2889924, upload-time = "2026-10-04T15:18:42.825Z" }, + { url = "https://files.pythonhosted.org/packages/d1/ea/14673bc9d8b7beeaa1eb46e9951b22543edaf2a4676c586e3b1e032ff6ee/pypdfium2-5.14.0-py3-none-macosx_13_0_arm64.whl", hash = "sha256:2de384df66ba55fcaab0775f30f28ec1090af3dfa60276a07821efc96d993118", size = 3542294, upload-time = "2026-10-04T15:18:44.345Z" }, + { url = "https://files.pythonhosted.org/packages/a6/11/b720097b01fa0874854f2f6669cbea4e4ea4e075769687714fac64d68964/pypdfium2-5.14.0-py3-none-macosx_13_0_x86_64.whl", hash = "sha256:e4e203ea9710fd00e5448edb6f1615dc8587035357f75f40b432dde0c33e8da1", size = 3735845, upload-time = "2026-10-04T15:18:45.975Z" }, + { url = "https://files.pythonhosted.org/packages/92/b4/0c31aa51887cd6cd032191dfe010a6d01ed43cf03204cfbd2184ebe4b715/pypdfium2-5.14.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1b696e6901e16f114a2ec6332e5e3f8f5033a901614ead28499ab18ca6024f5", size = 3719672, upload-time = "2026-10-04T15:18:47.455Z" }, + { url = "https://files.pythonhosted.org/packages/93/a8/ae6ef96bf66559328d07b9e402ea704352ea00c49b6a73573da57e1fb378/pypdfium2-5.14.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:593f2c952ae3ffdca0efcbb3d9464fbccb876254386114ff900cabef21157c3f", size = 3435593, upload-time = "2026-10-04T15:18:49.131Z" }, + { url = "https://files.pythonhosted.org/packages/59/ff/a78405fab4c8bad0ec25b49c5efba2c85ed14609ec73645f95220560bd81/pypdfium2-5.14.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d436ee9e024f981e68f5775f5a9d115f93ea14ee6c2c6efd35dd17d83edf4942", size = 3868604, upload-time = "2026-10-04T15:18:51.304Z" }, + { url = "https://files.pythonhosted.org/packages/5d/6e/09e9b62ab66c9acef5ad14f8a8c0d7b4d8d6ea6492e4e65b612ef146d373/pypdfium2-5.14.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f6f13bbcc5f4adabc2676e52f662c6cb375de86b314790b0ae08f3ab62eb116a", size = 4279333, upload-time = "2026-10-04T15:18:52.948Z" }, + { url = "https://files.pythonhosted.org/packages/4f/a3/c9cc797fc8bdfb8f37b9b0f8b9d02a5fc196b2015f408d53624cab5b0519/pypdfium2-5.14.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11f281613fa22313d9c7ab89947665e84eccf8ebe40e1198a84a88352305648d", size = 3799581, upload-time = "2026-10-04T15:18:54.913Z" }, + { url = "https://files.pythonhosted.org/packages/b9/76/54355a4bbd88bdd5ed3f4405bdc345eb593df9995daf90d285cbdf5c1410/pypdfium2-5.14.0-py3-none-manylinux_2_27_s390x.manylinux_2_28_s390x.whl", hash = "sha256:51d9e9b64ebc34effaf57f9b6d4511b3f66ad3744bd1690d2cc6700853173dcf", size = 4113022, upload-time = "2026-10-04T15:18:56.774Z" }, + { url = "https://files.pythonhosted.org/packages/7d/bc/ea461961ed0e0c4866df7a5610e76f769ef468bff28cd007e2aeecc8b882/pypdfium2-5.14.0-py3-none-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:605ab9d0d4c5e223599c9065b88d16b2c1f131c807c80dea8adbb16f1433e95b", size = 4062832, upload-time = "2026-10-04T15:18:58.471Z" }, + { url = "https://files.pythonhosted.org/packages/32/30/dde99bc8cb3f8ace1d856095c2b4a29c80eecf9089b186a3b0845d0abc69/pypdfium2-5.14.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:382de7fe20d32c42993a274d7b6c555a5623a97570dfc1d2f5e0a16fe0d5d482", size = 5058436, upload-time = "2026-10-04T15:18:59.993Z" }, + { url = "https://files.pythonhosted.org/packages/ec/16/5314182dda2695fdf5bd414a450ee866087068cca4725703932770d4be04/pypdfium2-5.14.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:dbfd6deff68cc46b134acd6be380d98d694a9f018fbb622c07229225c85db389", size = 4595505, upload-time = "2026-10-04T15:19:01.835Z" }, + { url = "https://files.pythonhosted.org/packages/63/3f/474c42e726f0020095c7d5f3fb88cfd4e5d39c1361105a72899ada0ecd1b/pypdfium2-5.14.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9f4d77db5232826dd03a63481f32164331b96c21fd68f0667b2e43dbae141a93", size = 5309775, upload-time = "2026-10-04T15:19:03.564Z" }, + { url = "https://files.pythonhosted.org/packages/6b/0c/723a6cf11cff00f125310d8c2c08362dc6c100d05fff8f92285a4df1bd41/pypdfium2-5.14.0-py3-none-musllinux_1_2_ppc64le.whl", hash = "sha256:b40a0913196a1483f0fdc22a53f8719c3aef87f1c4d8d9c38d2ad4e207500fdf", size = 5224565, upload-time = "2026-10-04T15:19:05.264Z" }, + { url = "https://files.pythonhosted.org/packages/5c/c5/86ab02a41e77a7aa962af6545a406815aeb9abaecd9f25dec34dbc336b72/pypdfium2-5.14.0-py3-none-musllinux_1_2_riscv64.whl", hash = "sha256:790e2cac1641a65912b73bd7243f45195d36f1663c85a3e1a126a8f5867c82a3", size = 4704416, upload-time = "2026-10-04T15:19:07.05Z" }, + { url = "https://files.pythonhosted.org/packages/ac/de/fb75013f924c5a4dde4a4a41ec13e7495f9b80022bf35dd51baa54e05910/pypdfium2-5.14.0-py3-none-musllinux_1_2_s390x.whl", hash = "sha256:09b99c8f0cb427eb17fec13c0862ed598bba34b4843df153f70fff806a2820bc", size = 5163621, upload-time = "2026-10-04T15:19:09.021Z" }, + { url = "https://files.pythonhosted.org/packages/cd/77/e59c814f10b533bc4565abe90ccef888ba29be45ada4627ebbf710961f0d/pypdfium2-5.14.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e70d87cb0577eab38f2106f9c9606b458930beef612a1b5f298772ed259f5ec0", size = 5121606, upload-time = "2026-10-04T15:19:10.609Z" }, + { url = "https://files.pythonhosted.org/packages/21/25/e067396b4bdd26c19f0997bfa3422d3975a49ceec2c59668e7599f2adcba/pypdfium2-5.14.0-py3-none-pyemscripten_2026_0_wasm32.whl", hash = "sha256:c73be14076bedebd9bcaf9b062579c95c668580043bccd29eb0db502101d5716", size = 2675501, upload-time = "2026-10-04T15:19:12.588Z" }, + { url = "https://files.pythonhosted.org/packages/7f/0c/6c21f68a57d0c4c506b9e5f72506ba91d8dde47eef699f3fd9561f7bff0e/pypdfium2-5.14.0-py3-none-win32.whl", hash = "sha256:9fd5cc94a389d50298e4d8cb79af6b9b8e0d785606e2a937725dc6e271c9c6e6", size = 3805374, upload-time = "2026-10-04T15:19:14.357Z" }, + { url = "https://files.pythonhosted.org/packages/00/dc/ca7874924c9cfd701ad53f89529968523790e70473e0b71e834668316148/pypdfium2-5.14.0-py3-none-win_amd64.whl", hash = "sha256:149fd5c6397b8df8bf7911a93506eff0be874f877afe7ac936cf5d37d21a6a06", size = 3947280, upload-time = "2026-10-04T15:19:16.302Z" }, + { url = "https://files.pythonhosted.org/packages/46/ab/35f2276deeeebb781925e2647dd88a39f8ea1a910104a0dbb28218473502/pypdfium2-5.14.0-py3-none-win_arm64.whl", hash = "sha256:eb8aeca157808f323e39ea298cc6d6c8e080c192ea2efb1ca81daa0f0ff4d095", size = 3745021, upload-time = "2026-10-04T15:19:18.276Z" }, +] + [[package]] name = "pytest" version = "9.1.1" @@ -1834,6 +2081,96 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] +[[package]] +name = "python-calamine" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/0c/ad1568b2a544d6a72d683c10f4910b46f2e405cccc45530a1ca30383cf74/python_calamine-0.9.0.tar.gz", hash = "sha256:cdae9f10335361eaee7d87d9d94593ff25c076561f95105cc880c57477d8f4af", size = 201917, upload-time = "2026-10-16T13:15:02.753Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/27/546010a3aa3fa08ab654afb71dfdf7f0be060c092d5f2ce5d6270a1f89d8/python_calamine-0.9.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3d76dbb8c6ad90ec087d356a7157118ba1af69beffb0a52f3613efeceadb1721", size = 875286, upload-time = "2026-10-16T13:12:39.42Z" }, + { url = "https://files.pythonhosted.org/packages/e3/c3/d033f3a5be96b326d7415c5bef8116c31f6b53e9036dbc1b90fdcc855f6f/python_calamine-0.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fea885bd944dbf013654ef97bea5d3a88f444c0ef5ce1b21246a093622d3c63e", size = 855629, upload-time = "2026-10-16T13:12:40.976Z" }, + { url = "https://files.pythonhosted.org/packages/3a/ae/57aee6361c6a599610bbb5a7d200db011a27341e6e2bf2bf0c90cb044bee/python_calamine-0.9.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:11d9889e79d1dbf9cdb453447126b026847697b9d840835cf3a40ed642e7c652", size = 929235, upload-time = "2026-10-16T13:12:42.631Z" }, + { url = "https://files.pythonhosted.org/packages/87/9d/41d2b8dd044bb77d27f1f9ec08001d4fade5aab2f3b89140ae77194f5ece/python_calamine-0.9.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1c883e4bf4c13ed0cc05c7bf9850ca8078538342988b3fab67282518448027a0", size = 922337, upload-time = "2026-10-16T13:12:44.34Z" }, + { url = "https://files.pythonhosted.org/packages/23/94/912273c9220bb401e801a5142392fc7629aa814aec4bdfb7c62c6fbf1ab6/python_calamine-0.9.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:13a4c206905f76f61f135217c4a3b8e7ceb62251612e057c264135292e8e9a5b", size = 1085877, upload-time = "2026-10-16T13:12:46.09Z" }, + { url = "https://files.pythonhosted.org/packages/20/98/f16dee8e5d34c8b6419331fe800a27b9ae1c5cfc108c05d72385a8c722ac/python_calamine-0.9.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdd2b256851c9002a9bb830bdd8973ff26d733233bef5697d4462c1e18ca35b8", size = 996001, upload-time = "2026-10-16T13:12:47.58Z" }, + { url = "https://files.pythonhosted.org/packages/08/31/34246eaf9c0ed4ed0255eaf4cd4447561fa9783a8b4554319979e71a6c92/python_calamine-0.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0985f4df983781df9bb1364a79bdac17b13ea43f9fc9d90c111500e695e4c142", size = 936159, upload-time = "2026-10-16T13:12:48.981Z" }, + { url = "https://files.pythonhosted.org/packages/8d/09/18b6d14921e7921458a479f49ad53725f7fdf4c7b4baf71a2dcc12bc88fc/python_calamine-0.9.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ac1759135d40ffb08b51d58fe801a67ada1bb682941adb71cf078bca6ea224c4", size = 995272, upload-time = "2026-10-16T13:12:50.403Z" }, + { url = "https://files.pythonhosted.org/packages/42/d6/807adf7ea44e54c27e6f94c12796696da8a353daaf68ecc382942cf87935/python_calamine-0.9.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:85b67145f51299188c4d39262df994e5da4c90130a2602a81657df89208f05c0", size = 1107413, upload-time = "2026-10-16T13:12:51.767Z" }, + { url = "https://files.pythonhosted.org/packages/d2/19/47535fa25557c32e237b9f074968c13a127641a45959d78e152da39c90cf/python_calamine-0.9.0-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:859608e5c5bd932974d18ef6fed7ff169d83692f1d245a3f0cc571ff7abdebb9", size = 1195663, upload-time = "2026-10-16T13:12:53.803Z" }, + { url = "https://files.pythonhosted.org/packages/f8/c0/be99fe39cf61f8a9035a3c0f69363b01e3bfd1aa8266ba1b96e4fc269fb0/python_calamine-0.9.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9e157da62281830ef88c9e80d2a0b16e73377b6576bd11d9e99c7988f78db4ee", size = 1151630, upload-time = "2026-10-16T13:12:55.368Z" }, + { url = "https://files.pythonhosted.org/packages/70/76/22bd1181cb43ccc3aeed2525beb5e69e4f253e72aeb071babab9dc6e92b0/python_calamine-0.9.0-cp312-cp312-win32.whl", hash = "sha256:9748c6faed06986f760ecc594e2621b0672c51c05c8dfab5845c69fef24f328f", size = 733442, upload-time = "2026-10-16T13:12:56.786Z" }, + { url = "https://files.pythonhosted.org/packages/ee/3e/c3e88f540b481da42dad055ac5de99a7f68839cb522202b887391ae5f2ac/python_calamine-0.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:a96342e0c7349f593113512894afa95232aaa419e2b6e0777426b83a0291f0e5", size = 782764, upload-time = "2026-10-16T13:12:58.434Z" }, + { url = "https://files.pythonhosted.org/packages/6c/74/74e5efd267f3762ce8c967e697f1269534a7342997b4f7257369f57e9ae9/python_calamine-0.9.0-cp312-cp312-win_arm64.whl", hash = "sha256:fef9843da160efed7c974cef8ad3e29107635e3c6809c8e6d1369f251f6a8cc5", size = 752387, upload-time = "2026-10-16T13:12:59.799Z" }, + { url = "https://files.pythonhosted.org/packages/06/20/d65727fb5b474b4100bf6186a8781cfce257d618d1669d27b1a735a21b63/python_calamine-0.9.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:b2dd4d6f4500198582d2811c35d58b8d32b212f38e837b3900ccc5a755f143f2", size = 874442, upload-time = "2026-10-16T13:13:01.279Z" }, + { url = "https://files.pythonhosted.org/packages/83/ef/371c68b05ed777464b17d7835df8ed253e24527d41d027584767d79e1aa3/python_calamine-0.9.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:93c28081959e2183cf1b3812a78e77e811a5d1b25ac5ac7dc532163aecb42e3a", size = 854509, upload-time = "2026-10-16T13:13:02.695Z" }, + { url = "https://files.pythonhosted.org/packages/3c/ac/a95e02beff0e47ce44520ea57c758d2d28346302214a3c5614e39779f3e3/python_calamine-0.9.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6d29b283c3659626f471e1352e7dc45dad1bcb6996aa3995d17d8b974c8604e", size = 929285, upload-time = "2026-10-16T13:13:04.483Z" }, + { url = "https://files.pythonhosted.org/packages/fa/5a/d8909cbd437166c3b5f60a1a1cda4830b8d0843fe3ab85251ea52bbde672/python_calamine-0.9.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c3c6527fb1a5b57fcf15f2b4a183ddd493b66ab5813dfd03171b90436d9ec5b2", size = 922415, upload-time = "2026-10-16T13:13:06.126Z" }, + { url = "https://files.pythonhosted.org/packages/fd/62/368f68e16efd60c1f6e211eb3ae0221f933b4ad2397087950b677c21b7a5/python_calamine-0.9.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f12b0fd3078f0ed61bf8b495d43beb6a53ff1b92a5d20dee1bfd7999e25ebe5b", size = 1086578, upload-time = "2026-10-16T13:13:07.651Z" }, + { url = "https://files.pythonhosted.org/packages/52/2a/e0f26d5f5c96202739861b58a55ae6009d361401350b278399396b7a3ee5/python_calamine-0.9.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:883c1d934cc4a28750a505cbb93490c774e1351badf010c92a724c62dc17d6dd", size = 995476, upload-time = "2026-10-16T13:13:09.304Z" }, + { url = "https://files.pythonhosted.org/packages/26/6e/b4e490e3295d6208b9b034ae29565c563b60c07c6267f0a01616b27c537c/python_calamine-0.9.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f11cb0e34a9e640ef56f65171f39135105201fbb6c6164d465a1dea0d9708cb2", size = 934976, upload-time = "2026-10-16T13:13:10.786Z" }, + { url = "https://files.pythonhosted.org/packages/ea/bf/9aebff24b8797cd614150f57a9e10bb68373616f4d88d13c2db840e57d87/python_calamine-0.9.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:107f10d1f42ee4de99caea99b825c8f30d85f96f77e5d2473f1a1680d93aa6d9", size = 995405, upload-time = "2026-10-16T13:13:12.288Z" }, + { url = "https://files.pythonhosted.org/packages/73/fc/8591efe1c7df444623d33702b5292391d292aeb7b98c7f4d650c5028e7eb/python_calamine-0.9.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:6de15b8a1327d5c493c8f41e7b43755622b836f68c16ed9c3adb1ce741167dd3", size = 1106360, upload-time = "2026-10-16T13:13:14.188Z" }, + { url = "https://files.pythonhosted.org/packages/52/5e/0f68dc8ab33da39273e741170c41eb59f010ff279c313db5a79c0c987455/python_calamine-0.9.0-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:e520560bbd5b6da12f3c4ce0276f725e50067130d1682792e22f6272c9f222ca", size = 1195327, upload-time = "2026-10-16T13:13:15.867Z" }, + { url = "https://files.pythonhosted.org/packages/13/f7/0f83fdd511fe05f744ca19f76f82a0874879c588c3f8fd95289d3e5c444f/python_calamine-0.9.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:b68dbfe23b79bd1d5300b988b470ba4e1c67189456de6026291192b7436783f7", size = 1149839, upload-time = "2026-10-16T13:13:17.554Z" }, + { url = "https://files.pythonhosted.org/packages/b2/2f/06eea3a7ba2cff44b8cb71a556268255cb15e4afdfcfde965cdc7ad5fe08/python_calamine-0.9.0-cp313-cp313-win32.whl", hash = "sha256:f36b18f59f50f38f91282c70cc0384780a0b2e02489fa5180b0afa6483e520ee", size = 731500, upload-time = "2026-10-16T13:13:18.959Z" }, + { url = "https://files.pythonhosted.org/packages/55/74/6c4223f291eba31a960a997b9d7d5335632b86055cc7448ca8628406581a/python_calamine-0.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:2eece55c58bb3e528274d9b405e4e572d1a2a77ba5772adbcd3548cc06737057", size = 782374, upload-time = "2026-10-16T13:13:20.372Z" }, + { url = "https://files.pythonhosted.org/packages/e6/bb/5175c6b6f516870398552ac15a1f4b0f29c480186844446b4c25c25a7393/python_calamine-0.9.0-cp313-cp313-win_arm64.whl", hash = "sha256:bba975a3a359555fd69a1413f4fe4bd95b91525ce5f588d1bdc39773a813a267", size = 751939, upload-time = "2026-10-16T13:13:21.851Z" }, + { url = "https://files.pythonhosted.org/packages/d3/7d/d49aacc73cb9622ac4f7a0bef7826a7657e9118e8ef25afc0e463dd69f35/python_calamine-0.9.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:e7939788945171046f77c8069ac23a4c9ecd2dbb8a41998882d3dc472558b7f5", size = 878147, upload-time = "2026-10-16T13:13:23.33Z" }, + { url = "https://files.pythonhosted.org/packages/43/bd/ad856238d24f4e32ab335ac76199afdeea6fec462e7e0a1f821ed8cb0d46/python_calamine-0.9.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:49c5323749f4168637795814ce8fe811b82fd2d0a7e41749c8d0d5f8d1a4c573", size = 857399, upload-time = "2026-10-16T13:13:24.782Z" }, + { url = "https://files.pythonhosted.org/packages/a5/df/d5cde5e51fdd42889ec6b47ae11bb72168066ef174602fb2c150985cf969/python_calamine-0.9.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b08fce785b9895193d0b1ddfa56b5ec878d76fde03fb71e23234399f26b489bb", size = 931639, upload-time = "2026-10-16T13:13:26.353Z" }, + { url = "https://files.pythonhosted.org/packages/e7/84/d0380e34154ba9a3833200988a8a95214681b20c14c5bbf8a121e49f9fac/python_calamine-0.9.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bf08852ce0ccb5e75aab4eef148e9d392208c12b9d8b4eb3257f8c0f98f12769", size = 924158, upload-time = "2026-10-16T13:13:27.757Z" }, + { url = "https://files.pythonhosted.org/packages/50/e4/031aac4668402b4d889e2aa208d8fc2ba7db63eda22535b2a5d67882abdb/python_calamine-0.9.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cfc19d1519f03161668cf06ead6d1cfbeed6562d5f6b67642598ad25b4f394ba", size = 1088934, upload-time = "2026-10-16T13:13:29.224Z" }, + { url = "https://files.pythonhosted.org/packages/74/e8/34aacf7de697d5d469d6c43bb38cd241a24dc0f2ac335dda265e68455781/python_calamine-0.9.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:53c5e26d5802f0245473faecf1cd87c2b06b7e8be3fcbd82731a3da5e96f05e0", size = 997994, upload-time = "2026-10-16T13:13:30.859Z" }, + { url = "https://files.pythonhosted.org/packages/38/b0/72cc759cb5fbbb6c5125e439b1b714e9cd47b9f76d594dc29d1493df31cb/python_calamine-0.9.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a3da3c0c385e8fc2e30ab0e2f9e2cbfe59aec1e2283b5698b2803196e88d42f", size = 936270, upload-time = "2026-10-16T13:13:32.271Z" }, + { url = "https://files.pythonhosted.org/packages/19/ea/82d06ae7c0a1a8f4d1562f4e27c24cdba59e80e40d00debe9c29cf311e31/python_calamine-0.9.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a02c615a1031f5446a000c344cd5de1753012340ab3beb335bbe22f82c2aae38", size = 996550, upload-time = "2026-10-16T13:13:33.865Z" }, + { url = "https://files.pythonhosted.org/packages/3a/80/f9872de90e3e7efc618dc3bb4069d75d597e3785f3c2c5f5ef80c59adb67/python_calamine-0.9.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:dba75466780569cab800ed33ec0d090e666a3de641dd8cf416bb19d2a5cb7419", size = 1109254, upload-time = "2026-10-16T13:13:35.293Z" }, + { url = "https://files.pythonhosted.org/packages/1d/bc/2f29966785d7be7aed9d344b79df6cfbe4904ad48269c93fd8a85e534049/python_calamine-0.9.0-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:fde608fba3e162c93eba9e5134a7f02d8cdf77ec9fda1d9b275e1e0b39a7f13e", size = 1197620, upload-time = "2026-10-16T13:13:36.798Z" }, + { url = "https://files.pythonhosted.org/packages/60/30/6f2712abed075a2e964c1d2878d2cef4fc0f792a8e9ad217b571733ee5f8/python_calamine-0.9.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:bf77390a53f481e84b973142dae9dc93cc6a85c8c4817ea9ba3fc2ca76f4842d", size = 1151236, upload-time = "2026-10-16T13:13:38.271Z" }, + { url = "https://files.pythonhosted.org/packages/10/84/208e3645f9a71b01e60d26b0b35f77838d41fa820667982b0be45a49df51/python_calamine-0.9.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:c049c7ab083460066da3f12525722c4b71ca4962a18fbfd1b74fcb735c196a15", size = 515983, upload-time = "2026-10-16T13:13:39.818Z" }, + { url = "https://files.pythonhosted.org/packages/82/6a/8b23052964a359bc9e756d84ca6527a36bba128b5f38da3951d43179cc41/python_calamine-0.9.0-cp314-cp314-win32.whl", hash = "sha256:cc15077a0bef42d95b50a36065dd4d79e90ce75629c262a89f9931a6441dcad2", size = 732172, upload-time = "2026-10-16T13:13:41.738Z" }, + { url = "https://files.pythonhosted.org/packages/e2/37/d30a4c25c0a31f30e139b02f30b5fc793ad373ff0d007cf9c84ca1aa0601/python_calamine-0.9.0-cp314-cp314-win_amd64.whl", hash = "sha256:4204c34ef13729e15826405b696e88038aad6109521944fb673e68dc987e9633", size = 784028, upload-time = "2026-10-16T13:13:43.284Z" }, + { url = "https://files.pythonhosted.org/packages/7e/65/9b4de889ce9badc08cb3b565f04b4ddf35f2c721d835263868cae104332c/python_calamine-0.9.0-cp314-cp314-win_arm64.whl", hash = "sha256:f14a10540b598a9aab68df01ec72da8155bdcc48678334a021bf70b51df966f9", size = 767125, upload-time = "2026-10-16T13:13:44.819Z" }, + { url = "https://files.pythonhosted.org/packages/51/d0/455f92cf965ff7c49f3b8bb02d07bbe845a7ad4fcf2ef61bc99ce3b768b8/python_calamine-0.9.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:0813a455fc3e241bf3c846772483675598e7e733456c5cfadc674045a5baa2da", size = 874169, upload-time = "2026-10-16T13:13:46.152Z" }, + { url = "https://files.pythonhosted.org/packages/3c/ab/eedf132e0cfe4c12f2a27beb08e7d14d23cb1f654bb140f3970fb6a252af/python_calamine-0.9.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0cc3e50cb1316fa5dbbfb750b3e690bc8a388ec2c8647be335b6e92f94b9b993", size = 853489, upload-time = "2026-10-16T13:13:47.594Z" }, + { url = "https://files.pythonhosted.org/packages/2c/7e/4abb224d6ce81b3cc89dac4e54579f429ca08b0c834443093f43da306d1c/python_calamine-0.9.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:286033fd3b0644548fd9b950c5648e2b3f7aa9b97f6b677e0ab78f073bd495a1", size = 927410, upload-time = "2026-10-16T13:13:49.103Z" }, + { url = "https://files.pythonhosted.org/packages/92/b3/92fc87241dd96f7559781ce10e7a02dc8b478417e9036a0c62db7a377a17/python_calamine-0.9.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6ffcbc785c380a21f2a30557601f70f3a79eaf1e613374d6db52097cc1dbf005", size = 918041, upload-time = "2026-10-16T13:13:50.487Z" }, + { url = "https://files.pythonhosted.org/packages/62/ae/accbf68096e524cdde26a58899a9c89274acb10a987f67bd01621f0f91b5/python_calamine-0.9.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a9a7a7125424478f8c9d996c4912cd0b62ab5cdfd5c070bf6f2b27705651eb41", size = 1086976, upload-time = "2026-10-16T13:13:51.902Z" }, + { url = "https://files.pythonhosted.org/packages/84/ba/37b9e813de4605fbe5394148cb7448ac6c00352c203151bd06f06657b2d0/python_calamine-0.9.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:476383df5824f6ef479269521dbc391a5f2b6525aff5ce7aefa529a224447eab", size = 993987, upload-time = "2026-10-16T13:13:53.355Z" }, + { url = "https://files.pythonhosted.org/packages/7e/ce/c5a4f748c69dd95be10ad184bbdca4b29bf0ba8578f34fbdb7614abc8fdf/python_calamine-0.9.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4b282cea34934b3710a1b82b506d98036bd6cc609739dc2e438297d4667b147", size = 934360, upload-time = "2026-10-16T13:13:54.989Z" }, + { url = "https://files.pythonhosted.org/packages/61/37/2c439ec6b95bcf9c70ae6c0d18af3dbe71eb57f47abad2e61b9d8d80f589/python_calamine-0.9.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f416a4407fe0c46f51842b59f1b5765cce44349e012a54709f495dc59d5e119c", size = 992197, upload-time = "2026-10-16T13:13:56.47Z" }, + { url = "https://files.pythonhosted.org/packages/ea/36/1debe7c98fdfc1b74f70bb5a4c9ad98c9db5c2da5b814e4113f6e65b4da9/python_calamine-0.9.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:2c188c8a9063d26f05fcc6d0c90a9104428e02da04d8b3ae12d3d496deafd7f8", size = 1104193, upload-time = "2026-10-16T13:13:58.048Z" }, + { url = "https://files.pythonhosted.org/packages/c3/2a/bacd41bc81ff8478e0146e0aed1aeb727147f97cc2f9b8913bebd3b5b9e9/python_calamine-0.9.0-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:ec2c7da970242d592f6c67c53055a0e973c492a118c2b976b0b2c8130f6c0b95", size = 1191403, upload-time = "2026-10-16T13:14:00.054Z" }, + { url = "https://files.pythonhosted.org/packages/3a/9c/436c50e542a0768aae094dc003be3400966118efc7bec82de27d2a6c91fa/python_calamine-0.9.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:5a86e98990e334f1d711a2295f6355b92363c9aa4fb2ff59ebf237658ee711cc", size = 1148759, upload-time = "2026-10-16T13:14:01.738Z" }, + { url = "https://files.pythonhosted.org/packages/67/54/1067bc4beac1e83e6173ae606b58592f9bd5be3d39bca72434713c9dafa5/python_calamine-0.9.0-cp314-cp314t-win_amd64.whl", hash = "sha256:701a92df322f26c0a68daf492d05fd2dad77a26e4ef5d85fd0ea8c0635718964", size = 781276, upload-time = "2026-10-16T13:14:03.358Z" }, + { url = "https://files.pythonhosted.org/packages/92/6f/18280b1dbc52ada01deb85daa21f4fa398393b7a6a3bca1db0936c934c95/python_calamine-0.9.0-cp314-cp314t-win_arm64.whl", hash = "sha256:98663d994147bf677b6090500d16429f15009fce737744134b67d83edfa7c5ed", size = 764938, upload-time = "2026-10-16T13:14:04.838Z" }, + { url = "https://files.pythonhosted.org/packages/ee/75/caa3a2fa421837dbbc5501c082195e9d45522e42ea25f0ac427e6747fc0f/python_calamine-0.9.0-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:591364759d3b878d1c3cfe88319ff58b6ea97f292b45796b16e01887ae9ef185", size = 877817, upload-time = "2026-10-16T13:14:07.114Z" }, + { url = "https://files.pythonhosted.org/packages/f6/6d/e3b5f6f904b1ea9fe8193ae54c058bc2ff1b99215b2541c8b0c38b27fdcf/python_calamine-0.9.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:7a1b86cfbb664af1c7e61e350b91f8d8cfdb5f08957985be76858b410a62892a", size = 857602, upload-time = "2026-10-16T13:14:08.538Z" }, + { url = "https://files.pythonhosted.org/packages/af/cf/d1308c01a5a3b34c82f2b40d06601d19568c4283e3a7efe85c41dff0c79c/python_calamine-0.9.0-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bd86b2cabb7079294ffdcb6d746265856f5b2c67b4dc892ff13bfd4efb15bc2", size = 932545, upload-time = "2026-10-16T13:14:10.186Z" }, + { url = "https://files.pythonhosted.org/packages/2f/d6/67dcea7ce139dc04be4feeb2c88021df502c4cc2b5243ba6bce24a50ac92/python_calamine-0.9.0-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5625ac2d7f1172ddd1d9e17d5dfc4f8f1694ea60ed06f6f18b768e6f64c71843", size = 924879, upload-time = "2026-10-16T13:14:12.278Z" }, + { url = "https://files.pythonhosted.org/packages/38/22/eed31d95c0e61f1778c96500bda03c5e1288a1846d3527ea14ad1f1a9670/python_calamine-0.9.0-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad25d200621145c530caabd39d6b2594b5e7c487dc45dcbe0c5d41c1a280b541", size = 1088905, upload-time = "2026-10-16T13:14:14.68Z" }, + { url = "https://files.pythonhosted.org/packages/10/59/64f79624ad51b5e4b64448a7954c0ddecdb40dc5eae3aed6abb33ec6c061/python_calamine-0.9.0-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a59693d81a4cc5e5bcadc7eeca7c12731f39c7b247f2ccca764e3429bfcee658", size = 997903, upload-time = "2026-10-16T13:14:16.409Z" }, + { url = "https://files.pythonhosted.org/packages/9b/1f/f47125df60c6b50ae679b7ffd6c68140fab7d0f5264cf46d4ba64e0ef930/python_calamine-0.9.0-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c981aa6a3007f5be8a9b8ef2a174e3fd031f73d6142d060bcf5087e8add25f6", size = 936776, upload-time = "2026-10-16T13:14:17.795Z" }, + { url = "https://files.pythonhosted.org/packages/ba/f0/cd22a7120d3f7d8097c9db5359d6f96b2f24fb92a042bb5f6454e8a5e920/python_calamine-0.9.0-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:06396ed00c0bfbf2570f53a6ac75d3d9216a04ffbe231e2c369269a3bb3748a8", size = 997555, upload-time = "2026-10-16T13:14:20.104Z" }, + { url = "https://files.pythonhosted.org/packages/9d/17/66b27d9cdb78163f0dccdddd886104d8e89a7e6ede147c9d1cd9237e3fc0/python_calamine-0.9.0-cp315-cp315-musllinux_1_1_aarch64.whl", hash = "sha256:c6a7478607b1bebf2be615a1f6ddfceda6b08f13d54e9c832948bf439cbc17b3", size = 1109224, upload-time = "2026-10-16T13:14:21.704Z" }, + { url = "https://files.pythonhosted.org/packages/71/62/963faa298afae4ed66630a6026de78ca620c4f096a07f5c6f81521dd5d0f/python_calamine-0.9.0-cp315-cp315-musllinux_1_1_armv7l.whl", hash = "sha256:6438e4409a135f8b038d8948bd0f590048c98fbdc2920a035e062eda8f98dc2a", size = 1197688, upload-time = "2026-10-16T13:14:23.127Z" }, + { url = "https://files.pythonhosted.org/packages/d3/72/fbe903542fa9fc06f35cabc68e461a32d122182404c1fb46869b6b235ba0/python_calamine-0.9.0-cp315-cp315-musllinux_1_1_x86_64.whl", hash = "sha256:34ced495159e9dfd1266a53d934712632cae8028dfa6ccb6dd5af9b7ab56c0b3", size = 1151100, upload-time = "2026-10-16T13:14:25.499Z" }, + { url = "https://files.pythonhosted.org/packages/06/ad/4186e6738c0afcd7a045e1a163a575fd88cd079f93ecbc610b3d0c9f2da1/python_calamine-0.9.0-cp315-cp315-pyemscripten_2026_5_wasm32.whl", hash = "sha256:e86a071102dca355b3f6a51d71c27c19a88435d684a2341cae104cb0e880d5a6", size = 516987, upload-time = "2026-10-16T13:14:27.116Z" }, + { url = "https://files.pythonhosted.org/packages/95/6b/89fead85dceb4080d2c793b84de8cad5465228add5b42e3d1198eb21f5d1/python_calamine-0.9.0-cp315-cp315-win32.whl", hash = "sha256:7513ff8f8bb956cffdec59a1c0ea182afce45dae0b51c1f9caf2a1af8e9ce94b", size = 732821, upload-time = "2026-10-16T13:14:28.467Z" }, + { url = "https://files.pythonhosted.org/packages/89/f3/783094548233b5433fae1da5cca5aa78c2b1004b4da9e554f18a46f6322e/python_calamine-0.9.0-cp315-cp315-win_amd64.whl", hash = "sha256:938987eaf260d01a0624230ff7959eac72b34cec37404e9a8f7c12dc30eabe9f", size = 784377, upload-time = "2026-10-16T13:14:29.829Z" }, + { url = "https://files.pythonhosted.org/packages/09/07/ac54af701bb59fc0747071d65504c8eb5d4394f84cb4350404664817bbb2/python_calamine-0.9.0-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:0a8a96c9505a69d16ddb1b17675ea61df5ca172905af5513c2b7d5ee0a6327bd", size = 874511, upload-time = "2026-10-16T13:14:31.164Z" }, + { url = "https://files.pythonhosted.org/packages/16/79/cf92f319aad4bd2f58a2ba74960e74b4f278027e4cd3f66254fcbb2c7009/python_calamine-0.9.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:0797237097b2cd324cc897ecf3f91d2f8e860b32ff36146b0a7fdbcc5e83daa4", size = 853445, upload-time = "2026-10-16T13:14:32.834Z" }, + { url = "https://files.pythonhosted.org/packages/57/0e/548802f6b219fc6a63699d9c4c75961246a754ab6f6d8398f124e8ec97b6/python_calamine-0.9.0-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04c09d11eeaa39c5d3fbb02eebccd3e9eab54786022d7e25c18b75a495c0a5a3", size = 927136, upload-time = "2026-10-16T13:14:34.299Z" }, + { url = "https://files.pythonhosted.org/packages/2e/d6/d7e6c4e1c11c05952f9bc960f78bc59fe3a062946f1199d27b8c0b156518/python_calamine-0.9.0-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0d0144180e5ca7d75569440d9619eb3f7564ecd2e5226527fbb5622870ad63a9", size = 918711, upload-time = "2026-10-16T13:14:35.69Z" }, + { url = "https://files.pythonhosted.org/packages/b9/e2/bed41f3a38340ddfa0ea4b02442f7e48b4528ca1d54ee3fa790c4082e9c9/python_calamine-0.9.0-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:635602612e44f1da0f9e472fbdd2fea63291d794421fa5432e7d56203d5d8407", size = 1085902, upload-time = "2026-10-16T13:14:37.268Z" }, + { url = "https://files.pythonhosted.org/packages/23/b8/3b9bdcaabc2e01cdcbfbcd84b4c58be93d27ceb17687d45f096ed8b531ee/python_calamine-0.9.0-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bb1f0d3906c30b9c874cefad279a2d966efcfd1cb9190e4cd04c136361b83831", size = 994833, upload-time = "2026-10-16T13:14:39.35Z" }, + { url = "https://files.pythonhosted.org/packages/a3/e3/2d8fe04e1b260cb3e3e265fa2a6cf15c5ab6751b1a694d77066587e991f0/python_calamine-0.9.0-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff72405bcac84b14aca29677ce33aeb2d2490ff59249a7b5e3752e779f4c6f96", size = 934943, upload-time = "2026-10-16T13:14:41.276Z" }, + { url = "https://files.pythonhosted.org/packages/b1/c7/1cd858c06d16ab5928697089dda694b81e5fda7ffbb198d2f036184b4ea3/python_calamine-0.9.0-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9996612d9e409c1d481f8c9c12dbf2e47363a9926676147f55a71843f69b0e2e", size = 992523, upload-time = "2026-10-16T13:14:43.047Z" }, + { url = "https://files.pythonhosted.org/packages/d4/f0/00be3e7047793b95493aead7b172e8dd8da3bd0b2733ae19574c6019fe51/python_calamine-0.9.0-cp315-cp315t-musllinux_1_1_aarch64.whl", hash = "sha256:e3eeca4941fe8c5e7c625de9c7d301b3129f2489289aa11cb5865a030f5b68ba", size = 1104456, upload-time = "2026-10-16T13:14:44.817Z" }, + { url = "https://files.pythonhosted.org/packages/0a/c7/6ee0a9d831771ba87be77d0560ed6077b8ca65e87823a39b9e63430d0837/python_calamine-0.9.0-cp315-cp315t-musllinux_1_1_armv7l.whl", hash = "sha256:e1d846450df95c414dccf4ec9419bdd6b7b9dd395b2675c9de237316c9cd41a7", size = 1191927, upload-time = "2026-10-16T13:14:46.282Z" }, + { url = "https://files.pythonhosted.org/packages/73/45/cfc41b6e9176df7c4982dec3fcac48e8cb0b24480c16cae55911740c51f8/python_calamine-0.9.0-cp315-cp315t-musllinux_1_1_x86_64.whl", hash = "sha256:e7a6e6dc61a4ddcd0e1f294f1a912c87d6291c5ffa9881284f3f0097bef7ed4f", size = 1149185, upload-time = "2026-10-16T13:14:47.782Z" }, + { url = "https://files.pythonhosted.org/packages/0b/22/5eb4cea0fa175fb6f15bfc8ab24c79affe6a34208f0a5d8e293abf26ca6a/python_calamine-0.9.0-cp315-cp315t-win_amd64.whl", hash = "sha256:9dc646440a7ba9e4c0b3bcc752b7efba5e4b0b9d46e7fab8a45c9ecd1b8d6b48", size = 781500, upload-time = "2026-10-16T13:14:49.18Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -2445,19 +2782,23 @@ wheels = [ name = "whyqd" source = { editable = "." } dependencies = [ + { name = "fastexcel" }, { name = "numpy" }, - { name = "openpyxl" }, { name = "packaging" }, { name = "pandas" }, + { name = "polars" }, { name = "pyarrow" }, { name = "pydantic" }, { name = "pydantic-settings" }, + { name = "python-calamine" }, { name = "randomname" }, { name = "rfc8785" }, - { name = "xlrd" }, ] [package.optional-dependencies] +pdf = [ + { name = "pdfplumber" }, +] verify-1x = [ { name = "openpyxl" }, { name = "pandas" }, @@ -2468,6 +2809,7 @@ verify-1x = [ [package.dev-dependencies] dev = [ { name = "jsonschema" }, + { name = "openpyxl" }, { name = "pytest" }, { name = "ruff" }, ] @@ -2487,26 +2829,29 @@ notebooks = [ [package.metadata] requires-dist = [ + { name = "fastexcel", specifier = ">=0.21.0,<0.22" }, { name = "numpy", specifier = ">=2.4.6,<3" }, - { name = "openpyxl", specifier = ">=3.1.5,<4" }, { name = "openpyxl", marker = "extra == 'verify-1x'", specifier = ">=3.1.5,<4" }, { name = "packaging", specifier = ">=26.3,<27" }, { name = "pandas", specifier = ">=3.0.6,<4" }, { name = "pandas", marker = "extra == 'verify-1x'", specifier = ">=3.0.6,<4" }, + { name = "pdfplumber", marker = "extra == 'pdf'", specifier = ">=0.11.10,<0.12" }, + { name = "polars", specifier = ">=2.0.0,<3" }, { name = "pyarrow", specifier = ">=24,<27" }, { name = "pyarrow", marker = "extra == 'verify-1x'", specifier = ">=24,<27" }, { name = "pydantic", specifier = ">=2.13.4,<3" }, { name = "pydantic-settings", specifier = ">=2.14.1,<3" }, + { name = "python-calamine", specifier = ">=0.9.0,<0.10" }, { name = "randomname", specifier = ">=0.2.1,<0.3" }, { name = "rfc8785", specifier = ">=0.1.4,<0.2" }, - { name = "xlrd", specifier = ">=2.0.2,<3" }, { name = "xlrd", marker = "extra == 'verify-1x'", specifier = ">=2.0.2,<3" }, ] -provides-extras = ["verify-1x"] +provides-extras = ["pdf", "verify-1x"] [package.metadata.requires-dev] dev = [ { name = "jsonschema", specifier = ">=4.26" }, + { name = "openpyxl", specifier = ">=3.1.5,<4" }, { name = "pytest", specifier = ">=7.2.2" }, { name = "ruff", specifier = "~=0.14.0" }, ] diff --git a/whyqd/config/patterns.py b/whyqd/config/patterns.py new file mode 100644 index 0000000..2d37f92 --- /dev/null +++ b/whyqd/config/patterns.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: BSD-3-Clause +"""Every regular expression whyqd runs, kept apart from the code that runs it, one class to each engine's syntax. + +Polars runs Rust's `regex` crate (https://docs.rs/regex/latest/regex/#syntax), whose syntax is not Python's `re`: +`\\p{Cc}` and the class intersection `&&` are Rust's alone. A pattern lives in the class whose engine runs it. +""" + +import re +from typing import ClassVar + + +class PolarsPatterns: + """Patterns Polars runs, in the syntax of Rust's `regex` crate. + + A hard space is U+00A0, U+2007, or U+202F; a zero-width character U+200B to U+200D, U+2060, or U+FEFF; a control + character any in Unicode's `Cc` but tab, line feed, and carriage return. A date-like text is three groups of digits + split by `/`, `-`, or `.`, or an English month's name, whole or short, among digits and separators. + """ + + HARD_SPACE: ClassVar[str] = "[   ]" + ZERO_WIDTH: ClassVar[str] = "[​-‍⁠]" + CONTROL: ClassVar[str] = r"[\p{Cc}&&[^\t\n\r]]" + WHITESPACE: ClassVar[str] = r"^\s+$" + DIGIT: ClassVar[str] = "[0-9]" + ZERO_PADDED: ClassVar[str] = r"^0[0-9]+$" + DATE: ClassVar[str] = r"^[0-9]+[/.-][0-9]+[/.-][0-9]+$" + MONTH: ClassVar[str] = ( + r"(?i)^[0-9\s/.,-]*\b(jan(uary)?|feb(ruary)?|mar(ch)?|apr(il)?|may|june?|july?|aug(ust)?|sep(t|tember)?" + r"|oct(ober)?|nov(ember)?|dec(ember)?)\b\.?[0-9\s/.,-]*$" + ) + + +class PythonPatterns: + """Patterns Python's `re` runs, compiled once.""" + + # An error cell as fastexcel reports it, naming calamine's `CellErrorType`: `Error(Div0)` + EXCEL_ERROR: ClassVar[re.Pattern] = re.compile(r"Error\((\w+)\)") + # A cell's reference in a sheet, its column's letters and its row's number: `AB12` + CELL_REFERENCE: ClassVar[re.Pattern] = re.compile(r"([A-Z]+)([0-9]+)") + # A line of text with its end, `\r\n`, `\n`, or `\r`, as `io.StringIO(newline="")` splits it; the last may have none + LINE: ClassVar[re.Pattern] = re.compile(r"[^\r\n]*(?:\r\n|\n|\r)|[^\r\n]+\Z") + # A part of the row key, its name given by `RowKeyParser.ROW`: `__row__`, `__row__1` + ROW_PART: ClassVar[str] = r"^{row}(\d*)$" diff --git a/whyqd/config/settings.py b/whyqd/config/settings.py index f8ff3fa..f457095 100644 --- a/whyqd/config/settings.py +++ b/whyqd/config/settings.py @@ -12,6 +12,28 @@ class Settings(BaseSettings): FORMAT_VERSIONS: ClassVar[tuple[str, ...]] = ("1.x", "2.0") FORMAT_CURRENT: ClassVar[str] = "2.0" CHECKSUM_ALGORITHM: ClassVar[str] = "BLAKE2b-512" + # The encodings a dialect may declare, by the WHATWG Encoding Standard's name in lower case, each with its labels + # (https://encoding.spec.whatwg.org/#names-and-labels) and the Python codec that decodes it as the standard does + ENCODINGS: ClassVar[dict[str, dict]] = { + "utf-8": { + "codec": "utf-8", + "labels": ("unicode-1-1-utf-8", "unicode11utf8", "unicode20utf8", "utf-8", "utf8", "x-unicode20utf8"), + }, + "utf-16le": { + "codec": "utf-16-le", + "labels": ("csunicode", "iso-10646-ucs-2", "ucs-2", "unicode", "unicodefeff", "utf-16", "utf-16le"), + }, + "utf-16be": {"codec": "utf-16-be", "labels": ("unicodefffe", "utf-16be")}, + "windows-1252": { + "codec": "cp1252", + "labels": ( + "ansi_x3.4-1968", "ascii", "cp1252", "cp819", "csisolatin1", "ibm819", "iso-8859-1", "iso-ir-100", + "iso8859-1", "iso88591", "iso_8859-1", "iso_8859-1:1987", "l1", "latin1", "us-ascii", + "windows-1252", "x-cp1252", + ), + }, + "macintosh": {"codec": "mac_roman", "labels": ("csmacintosh", "mac", "macintosh", "x-mac-roman")}, + } model_config = SettingsConfigDict(case_sensitive=True, env_file=".env", env_file_encoding="utf-8") diff --git a/whyqd/core/datasource.py b/whyqd/core/datasource.py index ae46658..3847674 100644 --- a/whyqd/core/datasource.py +++ b/whyqd/core/datasource.py @@ -5,7 +5,7 @@ from pydantic import Json from whyqd.dtypes import ChecksumContent, MimeType -from whyqd.models import ChecksumModel, DataSourceModel, CitationModel +from whyqd.models import ChecksumModel, DataSourceModel, CitationModel, DialectModel from whyqd.core.base import BaseDefinition from whyqd.config.settings import settings @@ -108,24 +108,30 @@ def derive_model( *, source: Path | str, mimetype: str | MimeType, - header: int | list[int | None] | None = 0, - **attributes, + dialect: DialectModel | dict | list[DialectModel | dict] | None = None, + names: list[str] | dict[str, list[str]] | None = None, + **settings, ) -> DataSourceModel | list[DataSourceModel]: - """Derive a data model schema (or list) from a data source. All columns will be coerced to `string` type to - preserve data quality even though this is far less efficient. + """Derive a data source from a file, or one a sheet from a workbook. Every cell is read as its text. Parameters: source: Source filename. - mimetype: Pandas can read a diversity of mimetypes. **whyqd** supports `xls`, `xlsx`, `csv`, `parquet` and `feather`. - header: Row (0-indexed) to use for the column labels of the parsed DataFrame. If there are multiple sheets, then - a list of integers should be provided. If `header` is `None`, row 0 will be treated as values and a - set of field names will be generated indexed to the number of data columns. - attributes: dict of specific `mimetype` related Pandas attributes. Use sparingly. + mimetype: **whyqd** supports `xls`, `xlsx`, `csv`, `parquet` and `feather`. + dialect: How the file is written, so it can be read: the dialect of CSV on the Web, as `DialectModel` + describes it. Undeclared, each setting takes its default; a CSV's encoding, undeclared, is settled by + whyqd's ladder and recorded. For a workbook, a list, each naming its `sheet`, or one a sheet in order. + names: One name per column, for a file with no header row or a header over several rows; for a workbook, a + mapping of sheet to names. + settings: Refused, naming the dialect: `header` and pandas' keywords are no longer read. Returns: List of DataSourceModel, or DataSourceModel """ - self.model = self.reader.derive_data_model(source=source, mimetype=mimetype, header=header, **attributes) + self.model, table = self.reader.derive_data_model( + source=source, mimetype=mimetype, dialect=dialect, names=names, **settings + ) + # Held so the first `get_data` need not read the file again + self.data = None if isinstance(self.model, list) else table.to_pandas() ######################################################################################### # VALIDATION UTILITIES @@ -135,7 +141,8 @@ def validate(self) -> bool: """Verify the source data against the data source's checksum. A checksum of the table as 1.x read it (`table-1x`) verifies through 1.x's own read, which needs the - `verify-1x` extra. Once it verifies, the file's byte hash is added beside it. + `verify-1x` extra. Once it verifies, the file's `fp0` and byte hash are added beside it. A data source without + one verifies by its byte hash, else by `fp0`, warning that the file was saved again with no cell changed. Raises: ImportError: If a 1.x checksum needs verifying and the `verify-1x` extra is not installed. @@ -152,21 +159,24 @@ def validate(self) -> bool: for m in models: table = m.get_checksum(ChecksumContent.TABLE_1X) if table is None: - raise ValueError(f"`{m.path}` has no checksum this whyqd can verify: it verifies `table-1x` only.") + self.reader.verify(source=m) + continue try: from whyqd.parsers.v1 import V1ChecksumParser except ImportError as e: raise ImportError("Verifying a 1.x checksum needs the extra: pip install whyqd[verify-1x]") from e if V1ChecksumParser().checksum(model=m) != table: raise AssertionError(f"{self.model_name.title()} model checksum does not match derived checksum.") - if m.get_checksum(ChecksumContent.BYTES) is None: - m.checksums.append( - ChecksumModel( - algorithm=settings.CHECKSUM_ALGORITHM, - checksumValue=self.core.get_checksum(source=m.path), - content=ChecksumContent.BYTES, + path = self.reader.locate(source=m.path) + found = {ChecksumContent.BYTES: self.reader.fingerprints.byte_hash(source=path)} + # One read with pandas' settings has no dialect to parse its grid by + if not (m.attributes and m.attributes.terms): + found[ChecksumContent.FP0] = self.reader.fingerprint(source=m, path=path) + for content, value in found.items(): + if m.get_checksum(content) is None and value is not None: + m.checksums.append( + ChecksumModel(algorithm=settings.CHECKSUM_ALGORITHM, checksumValue=value, content=content) ) - ) return True ######################################################################################### diff --git a/whyqd/core/schema.py b/whyqd/core/schema.py index 8eff795..4f26df7 100644 --- a/whyqd/core/schema.py +++ b/whyqd/core/schema.py @@ -103,5 +103,3 @@ def derive_model(self, *, data: DataSourceModel | dict) -> None: field["title"] = field["name"] self.fields.add(term=field) self._refresh_model_terms() - if data.index: - self.model.index = data.index diff --git a/whyqd/core/transform.py b/whyqd/core/transform.py index 08690d5..7afaca8 100644 --- a/whyqd/core/transform.py +++ b/whyqd/core/transform.py @@ -110,6 +110,7 @@ def process(self) -> None: self.model.whyqd = whyqd.__version__ # Validate crosswalk self.crosswalk.validate() + self.bound(source=self.model.dataSource) # Perform crosswalk, coercing to source schema prior to crosswalk df = self.reader.get(source=self.model.dataSource) df = self.reader.coerce_to_schema(df=df, schema=self.crosswalk.schema_source) @@ -158,29 +159,53 @@ def validate( Returns: A boolean `True` on successful validation. """ + # The checksums are 1.x's, so the files are read as 1.x read them + from whyqd.parsers.v1 import V1ChecksumParser + + v1 = V1ChecksumParser() + + def crosscheck(df, checksum: str) -> None: + if (generated := v1.table(df=df)) != checksum: + raise ValueError( + f"Checksum provided ({checksum}) is not equal to the generated checksum ({generated}). Check your data source." + ) + # Prepare self.set(transform=transform) - if data_source is not None: - source_df = self.reader.get( - source=data_source, mimetype=mimetype_source, preserve=self.model.dataSource.preserve - ) - else: - source_df = self.reader.get(source=self.model.dataSource) # Validate source + model = self.model.dataSource + if isinstance(data_source, DataSourceModel): + model = data_source + elif isinstance(data_source, dict): + model = DataSourceModel(**data_source) + elif data_source is not None: + model = model.model_copy(update={"path": str(data_source)}) source_checksum = self.model.dataSource.get_checksum(ChecksumContent.TABLE_1X) - if not source_checksum: - raise ValueError("Validation failed. Data source has no saved checksum.") - self.reader.get_checksum(df=source_df, crosscheck=source_checksum) + if source_checksum: + if data_source is not None: + source_df = v1.read( + source=data_source, mimetype=mimetype_source, preserve=self.model.dataSource.preserve + ) + else: + source_df = v1.read(source=self.model.dataSource) + crosscheck(source_df, source_checksum) + else: + try: + self.reader.verify(source=model) + except AssertionError as e: + raise ValueError(f"Validation failed. {e}") from e + source_df = self.reader.get(source=model) # Validate destination - destination_df = self.reader.get( + destination_df = v1.read( source=data_destination, mimetype=mimetype_destination, preserve=self.model.dataDestination.preserve ) destination_checksum = self.model.dataDestination.get_checksum(ChecksumContent.TABLE_1X) if not destination_checksum: raise ValueError("Validation failed. Data destination has no saved checksum.") - self.reader.get_checksum(df=destination_df, crosscheck=destination_checksum) + crosscheck(destination_df, destination_checksum) # Validate crosswalk, coercing to source schema prior to crosswalk self.crosswalk.validate() + self.bound(source=model) source_df = self.reader.coerce_to_schema(df=source_df, schema=self.crosswalk.schema_source) crosswalk_df = self.crosswalk.crud.transform_all(df=source_df) destination_names = [ @@ -194,9 +219,30 @@ def validate( raise ValueError( f"Validation failed. Missing required destination fields in crosswalked data: {set(required_names) - set(crosswalk_df.columns)}" ) - self.reader.get_checksum(df=crosswalk_df, crosscheck=destination_checksum) + crosscheck(crosswalk_df, destination_checksum) return True + def bound(self, *, source: DataSourceModel) -> None: + """Refuse a row-level action naming a row past the end of the data source's file, by its `index`, since a + schema carried from another file holds no row count. + + Raises: + ValueError: Naming the rows past its end, or where the data source has no row count to check them by. + """ + named = self.crosswalk.crud.rows() + if not named: + return + if source.index is None: + raise ValueError( + f"`{source.name}` has no row count, so the rows the crosswalk names cannot be checked. Derive it " + "again. [whyqd:rows-uncounted]" + ) + if past := sorted({r for r in named if not 0 <= r < source.index}): + raise ValueError( + f"The crosswalk names rows {past} of `{source.name}`, which has {source.index}, numbered from 0. " + "Check the rows each action names. [whyqd:row-past-end]" + ) + ######################################################################################### # SAVE UTILITIES ######################################################################################### diff --git a/whyqd/crud/action.py b/whyqd/crud/action.py index 5806190..7aa2264 100644 --- a/whyqd/crud/action.py +++ b/whyqd/crud/action.py @@ -208,6 +208,10 @@ def transform(self, *, df: pd.DataFrame, script: str | ActionScriptModel) -> pd. action_parser.set_schema(schema_source=self.schema_source, schema_destination=self.schema_destination) return action_parser.transform(df=df, **parsed) + def rows(self) -> list[int]: + """Every row a row-level action names, in script order.""" + return [r for script in self.get_all() for r in self.parse(script=script).get("rows") or []] + def transform_all(self, *, df: pd.DataFrame) -> pd.DataFrame: """Return a transformed dataframe after processing all scripts. diff --git a/whyqd/dtypes/cell.py b/whyqd/dtypes/cell.py index 6d33cc3..e1c7205 100644 --- a/whyqd/dtypes/cell.py +++ b/whyqd/dtypes/cell.py @@ -14,7 +14,8 @@ def describe(self): description = { "s": "Text, as stored. Text that is only whitespace stays text; empty text is an empty cell.", "n": "A number, as its shortest exact text; a whole number without `.0`.", - "d": "A date as ISO 8601, with its time only when that is not midnight; a time alone as ISO 8601.", + "d": "A date as ISO 8601, with its time only when that is not midnight; a time alone, or a duration, as " + "ISO 8601 (`PT25H30M`).", "b": "A boolean, as `TRUE` or `FALSE`.", "e": "An error cell, such as `#N/A`, as its reader reports it.", } diff --git a/whyqd/dtypes/encoding.py b/whyqd/dtypes/encoding.py index 5b27d5a..4481dee 100644 --- a/whyqd/dtypes/encoding.py +++ b/whyqd/dtypes/encoding.py @@ -4,18 +4,16 @@ class EncodingStep(str, Enum): BOM = "bom" - UTF8 = "utf-8" - DETECTED = "detected" - LATIN1 = "latin-1" DECLARED = "declared" + UTF8 = "utf-8" + WINDOWS_1252 = "windows-1252" @property def describe(self): description = { - "bom": "The file opens with a byte-order mark naming its encoding.", - "utf-8": "The file decodes as strict UTF-8.", - "detected": "Detected among a declared shortlist of encodings.", - "latin-1": "Read as Latin-1, which decodes any byte and loses nothing, since no other step settled it.", + "bom": "The file opens with a byte-order mark naming its encoding, which overrides any declaration.", "declared": "Declared by the wrangler.", + "utf-8": "The file decodes as strict UTF-8.", + "windows-1252": "Read as windows-1252, which decodes any byte and loses none, since no other step settled it.", } return description[self.value] diff --git a/whyqd/dtypes/mime.py b/whyqd/dtypes/mime.py index 245bec9..570a980 100644 --- a/whyqd/dtypes/mime.py +++ b/whyqd/dtypes/mime.py @@ -11,6 +11,7 @@ class MimeType(str, Enum): PRQ = "application/vnd.apache.parquet" FEATHER = "application/vnd.apache.feather" FTR = "application/vnd.apache.feather" + PDF = "application/pdf" @classmethod def value_of(cls, value): diff --git a/whyqd/formats/2.0/crosswalk.json b/whyqd/formats/2.0/crosswalk.json index 5499b7f..217ce6a 100644 --- a/whyqd/formats/2.0/crosswalk.json +++ b/whyqd/formats/2.0/crosswalk.json @@ -815,7 +815,7 @@ } ], "default": null, - "description": "Maximum value of a zero-base index for tabular data defined by this schema. Necessary where `actions` apply row-level transforms.", + "description": "A 1.x schema's row count, kept to save it as 1.x. It bounds nothing: a row-level action's rows are checked against its data source's `index`.", "title": "Index" }, "citation": { diff --git a/whyqd/formats/2.0/datasource.json b/whyqd/formats/2.0/datasource.json index 8c44d50..c8f1bb7 100644 --- a/whyqd/formats/2.0/datasource.json +++ b/whyqd/formats/2.0/datasource.json @@ -179,7 +179,7 @@ "type": "object" }, "ColumnModel": { - "description": "Pandas DataFrame column model for maintaining source reference. Necessary to perform term matching in\nparsing operations (there can be some *really* fruity column names.", + "description": "A data source's column, by its name as the source writes it, typed in whyqd's types only.", "properties": { "uuid": { "description": "Unique identity for the column. Automatically generated.", @@ -222,7 +222,7 @@ "type": "object" }, "DialectModel": { - "description": "How a file is written, so it can be read: the dialect of the W3C's CSV on the Web metadata vocabulary\n([Metadata Vocabulary for Tabular Data, section 5.9](https://www.w3.org/TR/tabular-metadata/#dialect-descriptions)),\nwith its names and defaults, and three of whyqd's own: `encodingStep`, `rowsPerRecord`, and `sheet`.\n\nOne default differs: CSV on the Web trims each cell, and whyqd keeps it as written (`trim` false). A blank line is\na row (`skipBlankRows` false), in both.", + "description": "How a file is written, so it can be read: the dialect of the W3C's CSV on the Web metadata vocabulary\n([Metadata Vocabulary for Tabular Data, section 5.9](https://www.w3.org/TR/tabular-metadata/#dialect-descriptions)),\nwith its names and defaults, and whyqd's own: `encodingStep`, `rowsPerRecord`, `sheet`, and, for a PDF, `cells`,\n`pages`, and `columns`.\n\nOne default differs: CSV on the Web trims each cell, and whyqd keeps it as written (`trim` false). A blank line is\na row (`skipBlankRows` false), in both.", "properties": { "encoding": { "default": "utf-8", @@ -332,6 +332,52 @@ "default": null, "description": "The spreadsheet's sheet, where the file has several.", "title": "Sheet" + }, + "cells": { + "anyOf": [ + { + "enum": [ + "lines", + "text" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "How a PDF's cells are found: by its ruling (`lines`), or by where its text sits (`text`). Read from the file where undeclared: `lines` where its ruling gives a table, else `text`.", + "title": "Cells" + }, + "pages": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A PDF's pages holding the table, counted from 1: `5-7`, or `1,3,5-7`. Absent for every page.", + "title": "Pages" + }, + "columns": { + "anyOf": [ + { + "items": { + "type": "number" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The left edge of each of a PDF table's columns, in points from the page's left, where its cells are found by `text` and the words do not show the columns.", + "title": "Columns" } }, "title": "DialectModel", @@ -340,10 +386,9 @@ "EncodingStep": { "enum": [ "bom", + "declared", "utf-8", - "detected", - "latin-1", - "declared" + "windows-1252" ], "title": "EncodingStep", "type": "string" @@ -376,7 +421,8 @@ "application/vnd.apache.parquet", "application/vnd.apache.parquet", "application/vnd.apache.feather", - "application/vnd.apache.feather" + "application/vnd.apache.feather", + "application/pdf" ], "title": "MimeType", "type": "string" @@ -444,6 +490,15 @@ "minimum": 0, "title": "Zeropadded", "type": "integer" + }, + "numberFormats": { + "additionalProperties": { + "type": "integer" + }, + "default": {}, + "description": "Number and date cells by the format code an `.xlsx` stores for them; empty for other files.", + "title": "Numberformats", + "type": "object" } }, "title": "ProfileModel", @@ -589,6 +644,19 @@ "default": null, "description": "How the file is written, so it can be read." }, + "hidden": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The publisher hid this sheet of the workbook; it is read like any other. Absent where visible.", + "title": "Hidden" + }, "attributes": { "anyOf": [ { diff --git a/whyqd/formats/2.0/schema.json b/whyqd/formats/2.0/schema.json index 355d92e..f94bb9d 100644 --- a/whyqd/formats/2.0/schema.json +++ b/whyqd/formats/2.0/schema.json @@ -733,7 +733,7 @@ } ], "default": null, - "description": "Maximum value of a zero-base index for tabular data defined by this schema. Necessary where `actions` apply row-level transforms.", + "description": "A 1.x schema's row count, kept to save it as 1.x. It bounds nothing: a row-level action's rows are checked against its data source's `index`.", "title": "Index" }, "citation": { diff --git a/whyqd/formats/2.0/transform.json b/whyqd/formats/2.0/transform.json index 9a8a4cc..fb3cf68 100644 --- a/whyqd/formats/2.0/transform.json +++ b/whyqd/formats/2.0/transform.json @@ -277,7 +277,7 @@ "type": "object" }, "ColumnModel": { - "description": "Pandas DataFrame column model for maintaining source reference. Necessary to perform term matching in\nparsing operations (there can be some *really* fruity column names.", + "description": "A data source's column, by its name as the source writes it, typed in whyqd's types only.", "properties": { "uuid": { "description": "Unique identity for the column. Automatically generated.", @@ -685,6 +685,19 @@ "default": null, "description": "How the file is written, so it can be read." }, + "hidden": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The publisher hid this sheet of the workbook; it is read like any other. Absent where visible.", + "title": "Hidden" + }, "attributes": { "anyOf": [ { @@ -741,7 +754,7 @@ "type": "object" }, "DialectModel": { - "description": "How a file is written, so it can be read: the dialect of the W3C's CSV on the Web metadata vocabulary\n([Metadata Vocabulary for Tabular Data, section 5.9](https://www.w3.org/TR/tabular-metadata/#dialect-descriptions)),\nwith its names and defaults, and three of whyqd's own: `encodingStep`, `rowsPerRecord`, and `sheet`.\n\nOne default differs: CSV on the Web trims each cell, and whyqd keeps it as written (`trim` false). A blank line is\na row (`skipBlankRows` false), in both.", + "description": "How a file is written, so it can be read: the dialect of the W3C's CSV on the Web metadata vocabulary\n([Metadata Vocabulary for Tabular Data, section 5.9](https://www.w3.org/TR/tabular-metadata/#dialect-descriptions)),\nwith its names and defaults, and whyqd's own: `encodingStep`, `rowsPerRecord`, `sheet`, and, for a PDF, `cells`,\n`pages`, and `columns`.\n\nOne default differs: CSV on the Web trims each cell, and whyqd keeps it as written (`trim` false). A blank line is\na row (`skipBlankRows` false), in both.", "properties": { "encoding": { "default": "utf-8", @@ -851,6 +864,52 @@ "default": null, "description": "The spreadsheet's sheet, where the file has several.", "title": "Sheet" + }, + "cells": { + "anyOf": [ + { + "enum": [ + "lines", + "text" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "How a PDF's cells are found: by its ruling (`lines`), or by where its text sits (`text`). Read from the file where undeclared: `lines` where its ruling gives a table, else `text`.", + "title": "Cells" + }, + "pages": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A PDF's pages holding the table, counted from 1: `5-7`, or `1,3,5-7`. Absent for every page.", + "title": "Pages" + }, + "columns": { + "anyOf": [ + { + "items": { + "type": "number" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The left edge of each of a PDF table's columns, in points from the page's left, where its cells are found by `text` and the words do not show the columns.", + "title": "Columns" } }, "title": "DialectModel", @@ -859,10 +918,9 @@ "EncodingStep": { "enum": [ "bom", + "declared", "utf-8", - "detected", - "latin-1", - "declared" + "windows-1252" ], "title": "EncodingStep", "type": "string" @@ -1048,7 +1106,8 @@ "application/vnd.apache.parquet", "application/vnd.apache.parquet", "application/vnd.apache.feather", - "application/vnd.apache.feather" + "application/vnd.apache.feather", + "application/pdf" ], "title": "MimeType", "type": "string" @@ -1205,6 +1264,15 @@ "minimum": 0, "title": "Zeropadded", "type": "integer" + }, + "numberFormats": { + "additionalProperties": { + "type": "integer" + }, + "default": {}, + "description": "Number and date cells by the format code an `.xlsx` stores for them; empty for other files.", + "title": "Numberformats", + "type": "object" } }, "title": "ProfileModel", @@ -1415,7 +1483,7 @@ } ], "default": null, - "description": "Maximum value of a zero-base index for tabular data defined by this schema. Necessary where `actions` apply row-level transforms.", + "description": "A 1.x schema's row count, kept to save it as 1.x. It bounds nothing: a row-level action's rows are checked against its data source's `index`.", "title": "Index" }, "citation": { diff --git a/whyqd/models/column.py b/whyqd/models/column.py index 80eece8..3e63a10 100644 --- a/whyqd/models/column.py +++ b/whyqd/models/column.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause from __future__ import annotations from typing import Optional -from pydantic import ConfigDict, BaseModel, Field, field_validator +from pydantic import ConfigDict, BaseModel, Field from whyqd.dtypes import FieldType from whyqd.models.identifier import Identifier @@ -9,8 +9,7 @@ class ColumnModel(BaseModel): - """Pandas DataFrame column model for maintaining source reference. Necessary to perform term matching in - parsing operations (there can be some *really* fruity column names.""" + """A data source's column, by its name as the source writes it, typed in whyqd's types only.""" uuid: Identifier = Field(description="Unique identity for the column. Automatically generated.") name: str = Field( @@ -24,12 +23,3 @@ class ColumnModel(BaseModel): ) profile: Optional[ProfileModel] = Field(None, description="What the column holds, as read, in counts.") model_config = ConfigDict(use_enum_values=True, validate_assignment=True, populate_by_name=True) - - @field_validator("dtype", mode="before") - @classmethod - def generate_dtype(cls, v: FieldType): - if v in ["float64", "int64", "Float64", "Int64", FieldType.NUMBER.value]: - return FieldType.NUMBER - if v in ["datetime64[ns]", FieldType.DATETIME.value]: - return FieldType.DATETIME - return FieldType.STRING diff --git a/whyqd/models/datasource.py b/whyqd/models/datasource.py index dfbbce3..2d280ce 100644 --- a/whyqd/models/datasource.py +++ b/whyqd/models/datasource.py @@ -60,6 +60,9 @@ class DataSourceModel(BaseModel): description="A list of key columns required before merging of multiple data sources into an interim data source.", ) dialect: Optional[DialectModel] = Field(None, description="How the file is written, so it can be read.") + hidden: Optional[bool] = Field( + None, description="The publisher hid this sheet of the workbook; it is read like any other. Absent where visible." + ) attributes: Optional[DataSourceAttributeModel] = Field( {}, description="Optional open dictionary for reader-specific attributes for Pandas' API. E.g. 'quoting' for the CSV library.", @@ -97,6 +100,7 @@ def get_checksum(self, content: str) -> str | None: @property def name(self) -> str: root = "".join(c for c in self.path.split("/")[-1] if c in f"-_. {string.ascii_letters}{string.digits}") - if self.sheet_name: - return f"{root}::{self.sheet_name}" + sheet = self.sheet_name or (self.dialect.sheet if self.dialect else None) + if sheet: + return f"{root}::{sheet}" return root diff --git a/whyqd/models/dialect.py b/whyqd/models/dialect.py index 95ab8c1..bdda610 100644 --- a/whyqd/models/dialect.py +++ b/whyqd/models/dialect.py @@ -9,7 +9,8 @@ class DialectModel(BaseModel): """How a file is written, so it can be read: the dialect of the W3C's CSV on the Web metadata vocabulary ([Metadata Vocabulary for Tabular Data, section 5.9](https://www.w3.org/TR/tabular-metadata/#dialect-descriptions)), - with its names and defaults, and three of whyqd's own: `encodingStep`, `rowsPerRecord`, and `sheet`. + with its names and defaults, and whyqd's own: `encodingStep`, `rowsPerRecord`, `sheet`, and, for a PDF, `cells`, + `pages`, and `columns`. One default differs: CSV on the Web trims each cell, and whyqd keeps it as written (`trim` false). A blank line is a row (`skipBlankRows` false), in both. @@ -30,4 +31,17 @@ class DialectModel(BaseModel): "`[47, 1]` is 47 fields on the first row and 1 on the next. Absent where a record is one row.", ) sheet: Optional[str] = Field(None, description="The spreadsheet's sheet, where the file has several.") + cells: Optional[Literal["lines", "text"]] = Field( + None, + description="How a PDF's cells are found: by its ruling (`lines`), or by where its text sits (`text`). Read " + "from the file where undeclared: `lines` where its ruling gives a table, else `text`.", + ) + pages: Optional[str] = Field( + None, description="A PDF's pages holding the table, counted from 1: `5-7`, or `1,3,5-7`. Absent for every page." + ) + columns: Optional[list[float]] = Field( + None, + description="The left edge of each of a PDF table's columns, in points from the page's left, where its cells " + "are found by `text` and the words do not show the columns.", + ) model_config = ConfigDict(use_enum_values=True, validate_assignment=True) diff --git a/whyqd/models/profile.py b/whyqd/models/profile.py index ebbe6a4..2cd7d48 100644 --- a/whyqd/models/profile.py +++ b/whyqd/models/profile.py @@ -16,4 +16,7 @@ class ProfileModel(BaseModel): control: int = Field(0, ge=0, description="Cells holding a control character.") dateLike: int = Field(0, ge=0, description="Text cells that look like a date.") zeroPadded: int = Field(0, ge=0, description="Text cells holding a number with leading zeros.") + numberFormats: dict[str, int] = Field( + {}, description="Number and date cells by the format code an `.xlsx` stores for them; empty for other files." + ) model_config = ConfigDict(use_enum_values=True, validate_assignment=True) diff --git a/whyqd/models/schema.py b/whyqd/models/schema.py index 5aabdd5..586c14f 100644 --- a/whyqd/models/schema.py +++ b/whyqd/models/schema.py @@ -37,7 +37,7 @@ class SchemaModel(BaseModel): ) index: Optional[int] = Field( None, - description="Maximum value of a zero-base index for tabular data defined by this schema. Necessary where `actions` apply row-level transforms.", + description="A 1.x schema's row count, kept to save it as 1.x. It bounds nothing: a row-level action's rows are checked against its data source's `index`.", ) citation: Optional[CitationModel] = Field(None, description="Optional full citation for the source data.") extensions: list[ExtensionModel] = Field([], description="The extension types the schema's fields use.") diff --git a/whyqd/parsers/__init__.py b/whyqd/parsers/__init__.py index 58f7533..2fc883f 100644 --- a/whyqd/parsers/__init__.py +++ b/whyqd/parsers/__init__.py @@ -7,6 +7,7 @@ from .category import CategoryParser from .cell import CellParser from .column import ColumnParser +from .fingerprint import FingerprintParser from .row_key import RowKeyParser diff --git a/whyqd/parsers/arrow_reader.py b/whyqd/parsers/arrow_reader.py new file mode 100644 index 0000000..dc16794 --- /dev/null +++ b/whyqd/parsers/arrow_reader.py @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: BSD-3-Clause +from pathlib import Path + +import polars as pl + +from whyqd.dtypes import CellType, MimeType +from whyqd.models import DialectModel +from whyqd.parsers.cell import CellParser +from whyqd.parsers.grid import GridParser + + +class ArrowParser: + """A Parquet or Feather file as the grid: its column names as the first row, then each value in whyqd's text. + + A columnar file stores types, not text, so each value is rendered as `CellParser.render` gives it, and the file + reads as the same table written as a CSV does. Its names are its header row, so the dialect's defaults read them. + """ + + PLACE = "row" + + def __init__(self) -> None: + self.cell = CellParser() + self.grids = GridParser() + + def grid( + self, *, source: Path, mime: MimeType, dialect: DialectModel + ) -> tuple[pl.DataFrame, pl.DataFrame, list[int], DialectModel]: + """The file's grid and each value's type (its names `s`), the row of the file each grid row is, counted from + 1, and the dialect unchanged. + + Raises + ------ + ValueError + For a column holding a type whyqd has no text for, naming it. + """ + df = pl.read_parquet(source) if mime in (MimeType.PARQUET, MimeType.PRQ) else pl.read_ipc(source) + columns, types = {}, {} + for i, name in enumerate(df.columns): + try: + cells = [self.cell.render(value=v) for v in df[name].to_list()] + except (TypeError, ValueError) as e: + raise ValueError( + f"`{source.name}` column `{name}` holds {df.schema[name]} values, which whyqd cannot read as text " + f"({e}). Write it as text, numbers, booleans, or dates. [whyqd:arrow-type]" + ) from e + columns[f"column_{i}"] = [name, *(None if c is None else c[1] for c in cells)] + types[f"column_{i}"] = [CellType.STRING.value, *(None if c is None else c[0].value for c in cells)] + schema = {c: pl.String for c in columns} + grid, types = self.grids.trim(grid=pl.DataFrame(columns, schema=schema), types=pl.DataFrame(types, schema=schema)) + return grid, types, list(range(1, grid.height + 1)), dialect diff --git a/whyqd/parsers/cell.py b/whyqd/parsers/cell.py index 3ae6c1f..a3b25ef 100644 --- a/whyqd/parsers/cell.py +++ b/whyqd/parsers/cell.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -from datetime import date, datetime, time +from datetime import date, datetime, time, timedelta from whyqd.dtypes import CellType @@ -13,7 +13,7 @@ class CellParser: """ def render( - self, *, value: str | int | float | bool | date | time | None, error: bool = False + self, *, value: str | int | float | bool | date | time | timedelta | None, error: bool = False ) -> tuple[CellType, str] | None: """A cell's type and canonical text, or `None` for an empty cell. @@ -21,7 +21,7 @@ def render( Parameters ---------- - value: str | int | float | bool | date | time | None + value: str | int | float | bool | date | time | timedelta | None The cell's stored value, as a Python type. error: bool, default False Whether the reader reports the cell as an error, such as `#N/A`. @@ -62,6 +62,27 @@ def render( return CellType.DATE, value.isoformat() if isinstance(value, (date, time)): return CellType.DATE, value.isoformat() + if isinstance(value, timedelta): + return CellType.DATE, self.duration(value=value) if isinstance(value, str): return CellType.STRING, value raise TypeError(f"No rendering is defined for a cell of type {type(value).__name__}.") + + def duration(self, *, value: timedelta) -> str: + """A length of time as ISO 8601, in hours, minutes, and seconds, as a spreadsheet's `[h]:mm` counts it. + + Example + ------- + ```python + CellParser().duration(value=timedelta(hours=25, minutes=30)) # "PT25H30M" + CellParser().duration(value=timedelta(0)) # "PT0S" + ``` + """ + sign, seconds = ("-", -value) if value < timedelta(0) else ("", value) + microseconds = seconds // timedelta(microseconds=1) + hours, rest = divmod(microseconds, 3_600_000_000) + minutes, rest = divmod(rest, 60_000_000) + whole, fraction = divmod(rest, 1_000_000) + second = f"{whole}.{fraction:06d}".rstrip("0") if fraction else str(whole) + parts = f"{f'{hours}H' if hours else ''}{f'{minutes}M' if minutes else ''}{f'{second}S' if rest else ''}" + return f"{sign}PT{parts or '0S'}" diff --git a/whyqd/parsers/core.py b/whyqd/parsers/core.py index 7885d2d..3955af9 100644 --- a/whyqd/parsers/core.py +++ b/whyqd/parsers/core.py @@ -4,7 +4,6 @@ from pydantic import Json import json import sys -import hashlib from urllib.parse import urlparse import urllib import posixpath @@ -21,7 +20,6 @@ from whyqd.parsers.format import FormatParser if TYPE_CHECKING: - import pandas as pd from whyqd.models import SchemaModel, CrosswalkModel @@ -176,30 +174,6 @@ def download_uri_source(self, source: str, directory: Path | str | None = None) urllib.request.urlretrieve(source, local_source) return local_source - ################################################################################################### - ### CHECKSUMS - ################################################################################################### - - def get_checksum(self, *, source: str) -> str: - # https://stackoverflow.com/a/47800021 - checksum = hashlib.blake2b() - with open(source, "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - checksum.update(chunk) - return checksum.hexdigest() - - def get_data_checksum(self, *, df: pd.DataFrame) -> str: - import numpy as np - - # The destination data does not have a valid checksum for the file itself, only the data. - # Need to ensure arrays are all of the same type. Easiest is to coerce np arrays to lists - for column in df.columns: - if isinstance(df.iloc[0][column], np.ndarray): - df[column] = df[column].apply(lambda x: x.tolist()) - df_string = df.astype("string").astype("string").to_string(index=False).encode("utf-8") - checksum = hashlib.blake2b - return checksum(df_string).hexdigest() - ################################################################################################### ### JSON & FILE LOAD & SAVE ################################################################################################### diff --git a/whyqd/parsers/csv_reader.py b/whyqd/parsers/csv_reader.py new file mode 100644 index 0000000..cd6fbea --- /dev/null +++ b/whyqd/parsers/csv_reader.py @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: BSD-3-Clause +from pathlib import Path +import csv + +import polars as pl + +from whyqd.config.patterns import PythonPatterns +from whyqd.dtypes import EncodingStep, MimeType +from whyqd.models import DialectModel +from whyqd.parsers.encoding import EncodingParser +from whyqd.parsers.grid import GridParser + + +class CsvParser: + """A CSV file as the grid: each cell's text as written, in rows as read, by the dialect and nothing else. + + Every line end (`\\r\\n`, `\\n`, `\\r`) ends a row, and a quoted cell keeps its line breaks. The grid is as wide + as its widest row, a shorter row's missing cells empty; trailing empty rows and columns are trimmed. An empty cell is + null; any other text, whitespace included, is kept. + + Python's `csv` parses, not Polars' reader, which fixes a table's width from its first row and has no escape for a + quote, so it cannot read every file as written. + """ + + PLACE = "line" + + def __init__(self) -> None: + self.encoding = EncodingParser() + self.grids = GridParser() + self.patterns = PythonPatterns() + + def grid( + self, *, source: Path, mime: MimeType, dialect: DialectModel + ) -> tuple[pl.DataFrame, pl.DataFrame, list[int], DialectModel]: + """The file's grid and its types, the line each of its rows starts on, and the dialect with the encoding + recorded. Every cell is text, so typed `s`; each is kept as written, `trim` being the table's. + + Parameters + ---------- + source: Path + mime: MimeType + Read only to share the interface every reader has. + dialect: DialectModel + How the file is written. Its encoding is read as declared where the wrangler declared it; otherwise the + ladder settles it, and the dialect returned records which step did. + + Raises + ------ + ValueError + Where the file cannot be read as the dialect describes it, naming the line and the setting. + + Returns + ------- + tuple[pl.DataFrame, pl.DataFrame, list[int], DialectModel] + """ + declared = None + if dialect.encodingStep == EncodingStep.DECLARED or ( + dialect.encodingStep is None and "encoding" in dialect.model_fields_set + ): + declared = dialect.encoding + text, encoding, step = self.encoding.decode(data=source.read_bytes(), declared=declared, source=f"`{source.name}`") + dialect = dialect.model_copy(update={"encoding": encoding, "encodingStep": step}) + rows, lines = self.rows(text=text, dialect=dialect, source=source) + grid, types = self.to_grid(rows=rows) + return grid, types, lines[: grid.height], dialect + + def rows(self, *, text: str, dialect: DialectModel, source: Path) -> tuple[list[list[str]], list[int]]: + settings = {"delimiter": dialect.delimiter, "doublequote": dialect.doubleQuote, "strict": True} + if dialect.quoteChar is None: + settings["quoting"] = csv.QUOTE_NONE + else: + settings["quotechar"] = dialect.quoteChar + if not dialect.doubleQuote: + settings["escapechar"] = "\\" + try: + # lines matched from the text, since `io.StringIO` copies it at four bytes a character + reader = csv.reader((m.group() for m in self.patterns.LINE.finditer(text)), **settings) + except TypeError as e: + raise ValueError( + f"`{source.name}` cannot be read with `delimiter` {dialect.delimiter!r} and `quoteChar` " + f"{dialect.quoteChar!r}: each must be one character ({e}). [whyqd:csv-character]" + ) from e + rows, lines, line = [], [], 1 + try: + for row in reader: + rows.append(row) + lines.append(line) + line = reader.line_num + 1 + except csv.Error as e: + raise ValueError( + f"`{source.name}` cannot be read from line {line}: {e}. If a quote there is text, declare `quoteChar` " + "as null. [whyqd:csv-quote]" + ) from e + return rows, lines + + def to_grid(self, *, rows: list[list[str]]) -> tuple[pl.DataFrame, pl.DataFrame]: + width = max((len(r) for r in rows), default=0) + grid = pl.DataFrame( + { + f"column_{i}": [r[i] if i < len(r) and r[i] != "" else None for r in rows] + for i in range(width) + }, + schema={f"column_{i}": pl.String for i in range(width)}, + ) + return self.grids.trim(grid=grid) diff --git a/whyqd/parsers/datasource.py b/whyqd/parsers/datasource.py index 6eee986..f157ab2 100644 --- a/whyqd/parsers/datasource.py +++ b/whyqd/parsers/datasource.py @@ -5,13 +5,22 @@ from datetime import date, datetime import pandas as pd import numpy as np -from pyarrow.parquet import ParquetFile -import pyarrow as pa import re import ast +import polars as pl + +from whyqd.parsers.arrow_reader import ArrowParser from whyqd.parsers.core import CoreParser -from whyqd.models import ColumnModel, DataSourceModel, DataSourceAttributeModel +from whyqd.parsers.csv_reader import CsvParser +from whyqd.parsers.excel_reader import ExcelParser +from whyqd.parsers.fingerprint import FingerprintParser +from whyqd.parsers.pdf_reader import PdfParser +from whyqd.parsers.profile import ProfileParser +from whyqd.parsers.row_key import RowKeyParser +from whyqd.parsers.table import TableParser +from whyqd.parsers.v1 import V1ChecksumParser +from whyqd.models import ColumnModel, DataSourceModel, DataSourceAttributeModel, DialectModel from whyqd.dtypes import ChecksumContent, MimeType, FieldType if TYPE_CHECKING: @@ -21,169 +30,84 @@ class DataSourceParser: """Get, review and restructure tabular source data.""" + # pandas' names for the dtypes a frame's column is recorded as a `number` by + NUMBERS = {"float64", "int64", "Float64", "Int64"} + def __init__(self): self.core = CoreParser() - self.DATE_FORMATS = { - "date": {"fmt": ["%Y-%m-%d"], "txt": ["YYYY-MM-DD"]}, - "datetime": { - "fmt": ["%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M:%S %Z%z"], - "txt": ["YYYY-MM-DD hh:mm:ss", "YYYY-MM-DD hh:mm:ss UTC+0000"], - }, - "year": {"fmt": ["%Y"], "txt": ["YYYY"]}, + self.fingerprints = FingerprintParser() + self.profiles = ProfileParser() + self.v1 = V1ChecksumParser() + self.tables = TableParser() + self.row_key = RowKeyParser() + arrow, excel = ArrowParser(), ExcelParser() + self.readers = { + MimeType.CSV: CsvParser(), + MimeType.PARQUET: arrow, + MimeType.PRQ: arrow, + MimeType.FEATHER: arrow, + MimeType.FTR: arrow, + MimeType.XLS: excel, + MimeType.XLSX: excel, + MimeType.PDF: PdfParser(), } ################################################################################################### ### TABULAR DATA READERS AND WRITERS ################################################################################################### - def read_excel(self, *, source: str | Path, **kwargs) -> dict[str, pd.DataFrame] | pd.DataFrame: - # This will default to returning a dictionary of dataframes for each sheet - # https://pandas.pydata.org/docs/reference/api/pandas.read_excel.html - if kwargs.get("sheet_name"): - # Only going to return a single dataframe - return pd.read_excel(source, **kwargs) - df = pd.read_excel(source, **kwargs) - keys = list(df.keys()) - for k in keys: - if df[k].empty: - del df[k] - if len(df.keys()) == 1: - df = df[keys[0]] - return df - - def read_csv(self, *, source: str | Path, **kwargs) -> pd.DataFrame: - # New in pandas 1.3: will ignore encoding errors - perfect for this initial wrangling process - # https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html - kwargs["encoding_errors"] = "ignore" - # Supposed to help with fruity separater guessing - kwargs["engine"] = "python" - if not kwargs.get("nrows"): - df = pd.read_csv(source, **kwargs) - else: - kwargs["iterator"] = True - kwargs["chunksize"] = 10000 - df_iterator = pd.read_csv(source, **kwargs) - df = pd.concat(df_iterator, ignore_index=True) - return df - - def read_parquet(self, *, source: str | Path, nrows: int | None = None, **kwargs) -> pd.DataFrame: - # https://pandas.pydata.org/docs/reference/api/pandas.read_parquet.html - if nrows: - # https://stackoverflow.com/a/69888274/295606 - pf = ParquetFile(str(source)) - pf = next(pf.iter_batches(batch_size=nrows)) - # https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table.to_pandas - return pa.Table.from_batches([pf]).to_pandas(**kwargs) - if "engine" not in kwargs: - kwargs["engine"] = "pyarrow" - return pd.read_parquet(source, **kwargs) - - def read_feather(self, *, source: str | Path, **kwargs) -> pd.DataFrame: - # https://pandas.pydata.org/docs/reference/api/pandas.read_feather.html - return pd.read_feather(source, **kwargs) - def get( self, *, source: str | Path | DataSourceModel, mimetype: str | MimeType | None = None, - header: int | None = 0, - preserve: str | list[str] | bool = False, - names: list[str] | None = None, - nrows: int | None = None, - sheet_name: str | None = None, directory: str | Path | None = None, - **attributes, - ) -> dict[str, pd.DataFrame] | pd.DataFrame | None: - """Return a Pandas dataframe from a given source. - - Accepts default pandas parameters for Excel and CSV, but the objective is to preserve the source data with - little data conversion outside of the data wrangling process. + **settings, + ) -> pd.DataFrame: + """A data source's table, read by its `dialect`, every cell as its text, keyed by `__row__`. Parameters ---------- source: str | Path | DataSourceModel - Path to source data file. - mimetype: str | MimeType - **whyqd** supports reading from CSV, XLS, XLSX, Feather and Parquet files. - header: int, default 0 - Row to use as the column headings. Default is 0. - preserve: str or list of str, default None - Column names where variable type guessing must be prevented and the original data preserved. - Critical for foreign key references with weird formats, like integers with leading `0`. - Only works for CSV and Excel. - names: list of str, default None - If the source data has no header row, explicitly pass a list of names - in the correct order - to address - the data. Only works for CSV and Excel. - nrows: int, default None - A specified number of rows to return. For review, it is faster to load only a small number. - Only works for CSV and Excel. - sheet_name: str, default None - If the source is a multi-sheet Excel file, and you know the sheet, you can load that specifically. If you - need to apply parameters specific to each sheet, it's best to load individually. Default, load all. - directory: str | Path, default None - An optional working directory for use for remote source data. Default will be from `settings`. - attributes: dict of specific `mimetype` related Pandas attributes. Use sparingly. + The data source, or a file's path, read by the default dialect; a workbook's by path, its only sheet. + mimetype: str | MimeType | None, default None + For a path: CSV, XLS, XLSX, Parquet, or Feather. + directory: str | Path | None, default None + Where to download a source given by URI. Default will be from `settings`. + + Raises + ------ + TypeError + For any other setting, since a file is read by its dialect: derive its data source, then get it by its model. Returns ------- - DataFrame or dict of DataFrame + DataFrame """ - kwargs = {} - kwargs |= attributes + if settings: + raise TypeError( + f"`{'`, `'.join(settings)}` {'is not a setting' if len(settings) == 1 else 'are not settings'} of " + "`get`. Derive the data source with a `dialect`, then get it by its model. [whyqd:get-settings]" + ) if isinstance(source, DataSourceModel): - # Additional attributes stored in the model, e.g. End-of-line errors require `quoting=csv.QUOTE_NONE` - # quoting: int, default None - # CSV errors in opening a source file can be fixed by referencing quotation and end-of-line errors using - # `quoting=csv.QUOTE_NONE` and `import csv`. - if source.attributes: - kwargs |= source.attributes.terms - mimetype = source.mime - preserve = source.preserve - sheet_name = source.sheet_name - header = source.header - if source.names: - names = source.names - source = source.path - if self.core.check_uri(source=source): - source = self.core.download_uri_source(source=source, directory=directory) - if not self.core.check_source(source=source): - e = f"Source at `{source}` not found." - raise FileNotFoundError(e) - # These, currently, don't accept any additional parameters. - mimetype = self.get_mimetype(mimetype=mimetype) - if mimetype in [MimeType.PRQ, MimeType.PARQUET]: - return self.read_parquet(source=source, nrows=nrows, **kwargs) - if mimetype in [MimeType.FTR, MimeType.FEATHER]: - return self.read_feather(source=source, **kwargs) - # If the dtypes have not been set, then ensure that any provided preserved columns remain untouched - # i.e. no forcing of text to numbers - # defaulting to `dtype = object` ... - if preserve: - if isinstance(preserve, bool): - kwargs["dtype"] = pd.StringDtype() - # kwargs["keep_default_na"] = False - # kwargs["na_values"] = "" - else: - if not isinstance(preserve, list): - preserve = [preserve] - # kwargs["dtype"] = {k: object for k in preserve} - kwargs["dtype"] = {k: pd.StringDtype() for k in preserve} - kwargs["header"] = header - if names: - # Preserve all rows - kwargs["header"] = None - kwargs["names"] = names - if nrows: - kwargs["nrows"] = nrows - # Check mimetype - if mimetype in [MimeType.XLS, MimeType.XLSX]: - kwargs["sheet_name"] = sheet_name - return self.read_excel(source=source, **kwargs) - if mimetype == MimeType.CSV: - return self.read_csv(source=source, **kwargs) - # Unknown mimetype - raise FileExistsError(f"Source at `{source}` does not have a recognised mimetype `{mimetype}`.") + mime = self.get_mimetype(mimetype=source.mime) + path = self.locate(source=source.path, directory=directory) + table, _ = self.read( + source=path, mime=mime, dialect=self.dialect(source=source), names=source.names or None + ) + if ( + source.get_checksum(ChecksumContent.TABLE_1X) is not None + and source.index is not None + and table.height != source.index + ): + self.core.show_warning( + f"`{source.name}` reads as {table.height} rows where 1.x read {source.index}, so its rows are " + "numbered differently. Check any stored action that names rows. [whyqd:rows-renumbered]" + ) + return table.to_pandas() + path = self.locate(source=source, directory=directory) + table, _ = self.read(source=path, mime=self.get_mimetype(mimetype=mimetype), dialect=DialectModel()) + return table.to_pandas() def set(self, *, df: pd.DataFrame, source: str | Path, mimetype: MimeType) -> None: """Save a Pandas dataframe from a given source. @@ -217,7 +141,7 @@ def set(self, *, df: pd.DataFrame, source: str | Path, mimetype: MimeType) -> No def coerce_to_schema(self, *, df: pd.DataFrame, schema: Type[SchemaDefinition]) -> pd.DataFrame: validate = {"matched": [], "unmatched": [], "coerced": []} - columns = self.get_header_columns(df=df) + columns = [c for c in self.get_header_columns(df=df) if not self.row_key.is_part(c.name)] for c in columns: prospect = schema.fields.get(name=c.name) if prospect: @@ -261,282 +185,325 @@ def coerce_column_to_dtype(self, *, column: pd.Series, coerce: str) -> pd.Series } return column.apply(lambda x: parser[coerce](x)) - def validate_schema_coersion(self, *, df: pd.DataFrame, schema: Type[SchemaDefinition]) -> bool: - """Returns the MimeType representation of of a submitted source datatype.""" - for field in schema.fields.get_all(): - # Tests conversion to type - series = df[field.name].astype(FieldType(field.dtype).astype) - # Tests each constraint - if field.constraints: - if field.constraints.category: - # https://python-reference.readthedocs.io/en/latest/docs/comprehensions/set_comprehension.html - constraints = {c.name for c in field.constraints.category} - difference = { - term for term in series.unique() if not (term is pd.NA or pd.isnull(term) or term is None) - } - constraints - if difference: - raise ValueError( - f"Terms in `{field.name}` not defined as a category constraint: `{list(difference)}`" - ) - if field.constraints.required and len(series) != series.count(): - raise ValueError( - f"Required constraint in `{field.name}` has `{len(series) - series.count()}` null values in `{len(series)}` rows." - ) - if field.constraints.unique and series.count() != len(series.unique()): - raise ValueError( - f"Unique constraint in `{field.name}` has `{series.count() - len(series.unique())}` non-unique values in `{series.count()}` rows." - ) - if field.constraints.minimum and series.min() < field.constraints.minimum: - raise ValueError( - f"Minimum constraint in `{field.name}` has value `{series.min()}` < `{field.constraints.minimum}`" - ) - if field.constraints.maximum and series.max() > field.constraints.maximum: - raise ValueError( - f"Minimum constraint in `{field.name}` has value `{series.max()}` > `{field.constraints.maximum}`" - ) - return True - ################################################################################################### ### DATA MODEL SCHEMA DERIVATION AND WRITER UTILITIES ################################################################################################### - def derive_data_model( - self, - *, - source: Path | str, - mimetype: str | MimeType, - sheet_name: str | None = None, - header: int | list[int] | None = 0, - **attributes, - ) -> DataSourceModel | list[DataSourceModel]: - """Derive a data model schema (or list) from a data source. All columns will be coerced to `string` type to - preserve data quality even though this is far less efficient. - - Parameters - ---------- - source: str - Source filename. - mimetype: str or MimeType - Pandas can read a diversity of mimetypes. **whyqd** supports `xls`, `xlsx`, `csv`, `parquet` and `feather`. - sheet_name: str, default None. - If `mimetype` is a multi-sheeted Excel file, specify which one to use. - header: int | list[int | None] | None = 0, - Row (0-indexed) to use for the column labels of the parsed DataFrame. If there are multiple sheets, then - a list of integers should be provided. - If `header` is `None`, row 0 will be treated as values and a set of field names will be generated indexed - to the number of data columns. - attributes: dict of specific `mimetype` related Pandas attributes. Use sparingly. + def locate(self, *, source: str | Path, directory: str | Path | None = None) -> Path: + """The local path of a source, downloading one given by URI.""" + if self.core.check_uri(source=source): + source = self.core.download_uri_source(source=source, directory=directory) + if not self.core.check_source(source=source): + raise FileNotFoundError(f"No file at `{source}`. [whyqd:source-missing]") + return Path(source) + + def declare(self, *, dialect: DialectModel | dict | None) -> DialectModel: + """A dialect as the wrangler declares it, refusing a setting whyqd does not know, naming those it does.""" + if dialect is None: + return DialectModel() + if isinstance(dialect, DialectModel): + return dialect + known = [k for k in DialectModel.model_fields if k != "encodingStep"] + unknown = [k for k in dialect if k not in known] + if unknown: + raise ValueError( + f"`{'`, `'.join(unknown)}` {'is not a setting' if len(unknown) == 1 else 'are not settings'} whyqd " + f"reads a file by. A `dialect`'s settings are {', '.join(known)}. [whyqd:dialect-unknown]" + ) + return DialectModel(**dialect) - Returns - ------- - List of DataSourceModel, or DataSourceModel + def dialect(self, *, source: DataSourceModel) -> DialectModel: + """The dialect a data source is read by: its own, or, for a 1.x data source built without one, the one that + reads its rows as 1.x read them (`FormatParser.dialect_1x`). Its `attributes` were pandas' keywords, which no + dialect holds, so a data source with any is refused. """ - response = [] - mimetype = self.get_mimetype(mimetype=mimetype) - sheet_names = [sheet_name] - if sheet_name is None and mimetype in [MimeType.XLS, MimeType.XLSX]: - # Check for multiple sheets - df = self.read_excel(source=source, sheet_name=sheet_name, nrows=1, **attributes) - if isinstance(df, dict): - # Multiple sheets - sheet_names = list(df.keys()) - for i, sheet_name in enumerate(sheet_names): - header_i = header - if isinstance(header, list): - header_i = header[i] - if header_i is None: - header_names = self.create_header_names( - source=source, mimetype=mimetype, sheet_name=sheet_name, **attributes - ) - else: - header_names = self.get_header_names( - source=source, mimetype=mimetype, sheet_name=sheet_name, header=header_i, **attributes - ) - response.append( - self.get_source_data_model( - source=source, - mimetype=mimetype, - names=header_names, - sheet_name=sheet_name, - header=header_i, - **attributes, - ) + if source.dialect is not None: + return source.dialect + settings = {"header": source.header, "sheet_name": source.sheet_name} + if source.attributes and source.attributes.terms: + raise ValueError( + f"`{source.name}` was read in 1.x with pandas' `{'`, `'.join(source.attributes.terms)}`, which whyqd " + "no longer reads. Derive it again with a `dialect`. [whyqd:attributes-1x]" ) - if len(response) > 1: - return response - return response[0] + return DialectModel(**self.core.formats.dialect_1x(settings)) - def get_source_data_model( + def reader(self, *, mime: MimeType) -> CsvParser | ExcelParser | ArrowParser | PdfParser: + if mime not in self.readers: + raise ValueError( + f"whyqd does not read `{mime.value}` files. It reads {', '.join(sorted({m.name for m in self.readers}))}. " + "[whyqd:format-unknown]" + ) + return self.readers[mime] + + def read( + self, *, source: Path, mime: MimeType, dialect: DialectModel, names: list[str] | None = None + ) -> tuple[pl.DataFrame, DialectModel]: + """A file's table, keyed by `__row__`, and its dialect with what the read settled recorded.""" + reader = self.reader(mime=mime) + grid, _, lines, dialect = reader.grid(source=source, mime=mime, dialect=dialect) + table, _ = self.tables.table(grid=grid, lines=lines, dialect=dialect, names=names, place=reader.PLACE) + return table, dialect + + def derive( self, *, - df: pd.DataFrame | None = None, source: Path | str, - mimetype: str | MimeType, + mime: MimeType, + dialect: DialectModel | dict | None = None, names: list[str] | None = None, - sheet_name: str | None = None, - header: int | None = 0, - **attributes, - ) -> DataSourceModel: - """Derive a data model schema from a SINGLE tabular data source. All columns will be coerced to `string` type to - preserve data quality even though this is far less efficient. - - Parameters - ---------- - source: str | Path - Source filename. - mimetype: str or MimeType - Pandas can read a diversity of mimetypes. **whyqd** supports `xls`, `xlsx`, `csv`, `parquet` and `feather`. - names: list of str - A de-conflicted list of header-row names. - sheet_name: str, default None. - If `mimetype` is a multi-sheeted Excel file, specify which one to use. - header: int, default 0 - Row (0-indexed) to use for the column labels of the parsed DataFrame. - attributes: dict of specific `mimetype` related Pandas attributes. Use sparingly. + ) -> tuple[DataSourceModel, pl.DataFrame]: + """A file's data source, its columns, row count, `fp0` and byte hash, and the dialect that reads it; and its + table, as `get` would read it.""" + path = self.locate(source=source) + read = self.reader(mime=mime).grid(source=path, mime=mime, dialect=self.declare(dialect=dialect)) + fp0 = None if mime == MimeType.PDF else self.fingerprints.fp0(sheets=[("", read[0], read[1])]) + return self.model(source=source, path=path, mime=mime, read=read, names=names, fp0=fp0) + + def model( + self, + *, + source: Path | str, + path: Path, + mime: MimeType, + read: tuple[pl.DataFrame, pl.DataFrame, list[int], DialectModel], + names: list[str] | None, + fp0: str | None, + hidden: bool = False, + ) -> tuple[DataSourceModel, pl.DataFrame]: + """The data source of a grid as its reader gave it, each column profiled, and its table.""" + grid, types, lines, dialect = read + reader = self.reader(mime=mime) + beside = [grid, types] + if mime == MimeType.XLSX: + beside.append(reader.formats(source=path, sheet=dialect.sheet, height=grid.height, width=grid.width)) + table, (text, types, *formats) = self.tables.table( + grid=grid, lines=lines, dialect=dialect, names=names, place=reader.PLACE, beside=beside + ) + formats = formats[0] if formats else None + checksums = [ + self.core.formats.checksum(value=self.fingerprints.byte_hash(source=path), content=ChecksumContent.BYTES) + ] + if fp0 is not None: + checksums.append(self.core.formats.checksum(value=fp0, content=ChecksumContent.FP0)) + model = DataSourceModel( + path=str(source), + mime=mime, + hidden=hidden or None, + attributes=DataSourceAttributeModel({}), + columns=[ + ColumnModel( + name=c, + profile=self.profiles.profile( + text=text[c], types=types[c], formats=None if formats is None else formats[c] + ), + ) + for c in text.columns + ], + names=names or [], + dialect=dialect, + checksums=checksums, + index=table.height, + ) + return model, table + + def fingerprint(self, *, source: DataSourceModel, path: Path) -> str | None: + """The `fp0` of a data source's file as it is now: every sheet of a workbook, or the grid its dialect parses. + `None` for a PDF, which has none.""" + mime = self.get_mimetype(mimetype=source.mime) + if mime == MimeType.PDF: + return None + reader = self.reader(mime=mime) + if mime in (MimeType.XLS, MimeType.XLSX): + sheets = [ + (name, *reader.grid(source=path, mime=mime, dialect=DialectModel(sheet=name))[:2]) + for name, _ in reader.sheets(source=path) + ] + return self.fingerprints.fp0(sheets=sheets) + grid, types, _, _ = reader.grid(source=path, mime=mime, dialect=self.dialect(source=source)) + return self.fingerprints.fp0(sheets=[("", grid, types)]) + + def verify(self, *, source: DataSourceModel, directory: str | Path | None = None) -> None: + """Verify a data source's file by its byte hash, else by `fp0`, warning that the file was saved again with no + cell changed. + + Raises + ------ + ValueError + For a data source with neither. + AssertionError + Where the file's cells are not those its data source was derived from. + """ + hashed, fp0 = source.get_checksum(ChecksumContent.BYTES), source.get_checksum(ChecksumContent.FP0) + if hashed is None and fp0 is None: + raise ValueError(f"`{source.path}` has no checksum whyqd can verify. [whyqd:checksum-none]") + path = self.locate(source=source.path, directory=directory) + if hashed is not None and self.fingerprints.byte_hash(source=path) == hashed: + return + if fp0 is None or self.fingerprint(source=source, path=path) != fp0: + raise AssertionError( + f"`{source.path}` has changed: its cells are not those its data source was derived from. " + "[whyqd:source-changed]" + ) + self.core.show_warning( + f"`{source.path}` was saved again: its bytes differ, but its cells (`fp0`) do not. " + "[whyqd:source-saved-again]" + ) - Returns - ------- - DataSourceModel + def derive_sheets( + self, + *, + source: Path | str, + mime: MimeType, + dialect: DialectModel | dict | list[DialectModel | dict] | None = None, + names: list[str] | dict[str, list[str]] | None = None, + ) -> tuple[DataSourceModel | list[DataSourceModel], pl.DataFrame | list[pl.DataFrame]]: + """A workbook's data sources, one a sheet, every sheet read and a hidden one flagged, and their tables. + + Each sheet is its own table, with its own dialect and names. A list of dialects names each one's `sheet`, a + sheet named by none taking the defaults; or names none, giving one a sheet in workbook order. `names` maps a + sheet to its names, or, for a workbook of one sheet, is that sheet's names. + + Raises + ------ + ValueError + For a dialect or names given for a sheet the workbook does not have, or dialects that cannot be matched to + its sheets. """ - del_df = False - if df is None: - del_df = True - if header is None or header != 0: - # If names are specified, then header is None - df = self.get( - source=source, - mimetype=mimetype, - names=names, - preserve=names, - sheet_name=sheet_name, - header=None, - **attributes, + path = self.locate(source=source) + reader = self.reader(mime=mime) + sheets = reader.sheets(source=path) + order = [name for name, _ in sheets] + dialects = self.sheet_dialects(dialect=dialect, sheets=order, source=source) + if isinstance(names, list): + if len(order) != 1: + raise ValueError( + f"`{Path(source).name}` has {len(order)} sheets, so give `names` by sheet: " + f"{{'{order[0]}': [...]}}. [whyqd:names-by-sheet]" ) - else: - df = self.get(source=source, mimetype=mimetype, preserve=names, sheet_name=sheet_name, **attributes) - columns = self.get_header_columns(df=df) - preserve = [c.name for c in columns if c.dtype == "string"] - source_data_model = { - "path": str(source), - "mime": self.get_mimetype(mimetype=mimetype), - "sheet_name": sheet_name, - "columns": columns, - "preserve": preserve, - "checksums": [ - self.core.formats.checksum(value=self.get_checksum(df=df), content=ChecksumContent.TABLE_1X) - ], - "header": header, - "index": df.shape[0], - "attributes": DataSourceAttributeModel.model_validate(attributes), + names = {order[0]: names} + names = names or {} + self.check_sheets(given=names, sheets=order, setting="names", source=source) + reads = { + name: reader.grid(source=path, mime=mime, dialect=dialects[name].model_copy(update={"sheet": name})) + for name in order } - if header is None or header != 0: - source_data_model["names"] = names - if del_df: - del df - return DataSourceModel(**source_data_model) + fp0 = self.fingerprints.fp0(sheets=[(name, grid, types) for name, (grid, types, _, _) in reads.items()]) + models, tables = zip( + *( + self.model( + source=source, path=path, mime=mime, read=reads[name], names=names.get(name), fp0=fp0, hidden=hidden + ) + for name, hidden in sheets + ), + strict=True, + ) + return (models[0], tables[0]) if len(models) == 1 else (list(models), list(tables)) + + def sheet_dialects( + self, *, dialect: DialectModel | dict | list[DialectModel | dict] | None, sheets: list[str], source: Path | str + ) -> dict[str, DialectModel]: + given = [] if dialect is None else dialect if isinstance(dialect, list) else [dialect] + given = [self.declare(dialect=d) for d in given] + named = [d for d in given if d.sheet is not None] + if not named: + if given and len(given) != len(sheets): + raise ValueError( + f"`{Path(source).name}` has {len(sheets)} sheets but {len(given)} dialects without a `sheet`. " + "Give one a sheet, in order, or name each one's `sheet`. [whyqd:dialects-count]" + ) + return {name: given[i] if given else DialectModel() for i, name in enumerate(sheets)} + if len(named) != len(given): + raise ValueError("Name every dialect's `sheet`, or none. [whyqd:dialects-named]") + by_name = {d.sheet: d for d in named} + self.check_sheets(given=by_name, sheets=sheets, setting="dialect", source=source) + return {name: by_name.get(name, DialectModel()) for name in sheets} + + def check_sheets(self, *, given: dict, sheets: list[str], setting: str, source: Path | str) -> None: + if unknown := [s for s in given if s not in sheets]: + raise ValueError( + f"`{setting}` is given for `{'`, `'.join(unknown)}`, which `{Path(source).name}` does not have. Its " + f"sheets are `{'`, `'.join(sheets)}`. [whyqd:sheet-given]" + ) - def create_header_names( + def derive_data_model( self, *, source: Path | str, mimetype: str | MimeType, - sheet_name: str | None = None, - nrows: int = 50, - **attributes, - ) -> list[str]: - """ - Return a list of default column names - starting with `column-0` - to apply to a given SINGULAR tabular data - source. + dialect: DialectModel | dict | list[DialectModel | dict] | None = None, + names: list[str] | dict[str, list[str]] | None = None, + **settings, + ) -> tuple[DataSourceModel | list[DataSourceModel], pl.DataFrame | list[pl.DataFrame]]: + """A data source derived from a file, and its table: a workbook's one a sheet, as lists where it has several. Parameters ---------- - source: str - Source filename. - mimetype: MimeType, default MimeType.CSV - Pandas can read a diversity of mimetypes, but whyqd has only been tested on `xls`, `xlsx` and `csv`. - sheet_name: str, default None. - If `mimetype` is a multi-sheeted Excel file, specify which one to use. - nrows: int, default 50 - A specified number of rows to return. For review, it is faster to load only a small number. - attributes: dict of specific `mimetype` related Pandas attributes. Use sparingly. - - Returns - ------- - List of str + source: Path | str + mimetype: str | MimeType + CSV, XLS, XLSX, Parquet, or Feather. + dialect: DialectModel | dict | list[DialectModel | dict] | None, default None + How the file is written, so it can be read; for a workbook, one a sheet (`derive_sheets`). + names: list[str] | dict[str, list[str]] | None, default None + One name per column; for a workbook, by sheet. + + Raises + ------ + TypeError + For `header` or any reader's keyword, naming the dialect's settings to declare instead. """ - df = self.get(source=source, mimetype=mimetype, sheet_name=sheet_name, nrows=nrows, **attributes) - if (isinstance(df, dict) and not df) or df.empty: - return [] - return [f"column_{i}" for i in range(len(df.columns))] + if settings: + raise TypeError( + f"`{'`, `'.join(settings)}` {'is a pandas setting' if len(settings) == 1 else 'are pandas settings'}, " + "which whyqd no longer reads. Declare the file in `dialect`, and its column names in `names`. " + "[whyqd:derive-settings]" + ) + mimetype = self.get_mimetype(mimetype=mimetype) + if mimetype in (MimeType.XLS, MimeType.XLSX): + return self.derive_sheets(source=source, mime=mimetype, dialect=dialect, names=names) + return self.derive(source=source, mime=mimetype, dialect=dialect, names=names) - def get_header_names( - self, - *, - source: Path | str, - mimetype: str | MimeType, - sheet_name: str | None = None, - header: int | None = 0, - nrows: int = 1, - **attributes, - ) -> list[str]: - """Return a list of column names to apply to a given SINGULAR tabular data source. + def get_source_data_model(self, *, df: pd.DataFrame, source: Path | str, mimetype: str | MimeType) -> DataSourceModel: + """The data source of a transform's output, from the frame it was saved from, with its `table-1x` checksum. Parameters ---------- - source: str - Source filename. - mimetype: MimeType, default MimeType.CSV - Pandas can read a diversity of mimetypes, but whyqd has only been tested on `xls`, `xlsx` and `csv`. - sheet_name: str, default None. - If `mimetype` is a multi-sheeted Excel file, specify which one to use. - header: int, default 0 - Row (0-indexed) to use for the column labels of the parsed DataFrame. - nrows: int, default 1 - A specified number of rows to return. For review, it is faster to load only a small number. - attributes: dict of specific `mimetype` related Pandas attributes. Use sparingly. + df: pd.DataFrame + The output, as saved. + source: Path | str + Where it was saved. + mimetype: str | MimeType Returns ------- - List of str + DataSourceModel """ - if header is not None and header > nrows: - nrows = header + 1 - df = self.get(source=source, mimetype=mimetype, sheet_name=sheet_name, header=header, nrows=nrows, **attributes) - return df.columns.tolist() + columns = self.get_header_columns(df=df) + return DataSourceModel( + path=str(source), + mime=self.get_mimetype(mimetype=mimetype), + columns=columns, + preserve=[c.name for c in columns if c.dtype == "string"], + checksums=[self.core.formats.checksum(value=self.v1.table(df=df), content=ChecksumContent.TABLE_1X)], + index=df.shape[0], + attributes=DataSourceAttributeModel({}), + ) def get_header_columns(self, *, df: pd.DataFrame) -> List[ColumnModel]: - """Returns a list of ColumnModels from a source DataFrame. + """A frame's columns, each typed `number` where pandas holds it as one of `NUMBERS`, else `string`, a date + column included, as 1.x recorded them. Parameters ---------- df: pd.DataFrame - Should be derived from `get` with a sensible default for `nrows` being 50. Returns ------- List of ColumnModel """ - # Prepare summary - try: - columns = [ - {"name": k, "type": "number"} - if v in ["float64", "int64", "Float64", "Int64"] - else {"name": k, "type": "date"} - if v in ["datetime64[ns]"] - else {"name": k, "type": "string"} - for k, v in df.dtypes.apply(lambda x: x.name).to_dict().items() - ] - except AttributeError: - # No idea ... some seem to give shit - columns = [ - {"name": k, "type": "number"} - if v in ["float64", "int64", "Float64", "Int64"] - else {"name": k, "type": "date"} - if v in ["datetime64[ns]"] - else {"name": k, "type": "string"} - for k, v in df.dtypes.apply(lambda x: str(x)).to_dict().items() - ] - return [ColumnModel(**c) for c in columns] + return [ + ColumnModel(name=k, dtype=FieldType.NUMBER if str(v) in self.NUMBERS else FieldType.STRING) + for k, v in df.dtypes.items() + ] ################################################################################################### ### GENERAL UTILITIES @@ -562,25 +529,6 @@ def get_mimetype(self, mimetype: str | MimeType) -> MimeType: return MimeType(mimetype) raise ValueError(f"MimeType not found for '{mimetype}'") - def get_checksum(self, *, df: pd.DataFrame, crosscheck: str = None) -> str: - checksum = self.core.get_data_checksum(df=df) - if crosscheck and checksum != crosscheck: - raise ValueError( - f"Checksum provided ({crosscheck}) is not equal to the generated checksum ({checksum}). Check your data source." - ) - return checksum - - def check_date_format(self, *, date_type: str, date_value: str) -> bool: - # https://stackoverflow.com/a/37045601 - # https://www.saltycrane.com/blog/2009/05/converting-time-zones-datetime-objects-python/ - for fmt in self.DATE_FORMATS[date_type]["fmt"]: - try: - if date_value == datetime.strptime(date_value, fmt).strftime(fmt): - return True - except ValueError: - continue - raise ValueError(f"Incorrect date format, should be: `{self.DATE_FORMATS[date_type]['txt']}`") - ################################################################################################### ### PANDAS TYPE PARSERS ################################################################################################### diff --git a/whyqd/parsers/encoding.py b/whyqd/parsers/encoding.py new file mode 100644 index 0000000..a1b8fde --- /dev/null +++ b/whyqd/parsers/encoding.py @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: BSD-3-Clause +import codecs +import warnings +from collections import Counter + +from whyqd.config.settings import settings +from whyqd.dtypes import EncodingStep + + +class EncodingParser: + """A file's bytes as text, decoded as the WHATWG Encoding Standard decodes them, which CSV on the Web names its + encodings by ([Encoding Standard](https://encoding.spec.whatwg.org/)). + + Nothing is guessed. The ladder: a byte-order mark, over any declaration; otherwise the declared encoding; otherwise + strict UTF-8; otherwise `windows-1252`, which decodes every byte and loses none. A file in another encoding then + loads with every byte kept, and reads as its text once the wrangler declares its encoding. + """ + + BOMS = ((b"\xef\xbb\xbf", "utf-8"), (b"\xfe\xff", "utf-16be"), (b"\xff\xfe", "utf-16le")) + # Python's `cp1252` leaves five bytes undefined, which the standard decodes to the code point of their value + WINDOWS_1252 = "".join( + chr(b) if b in (0x81, 0x8D, 0x8F, 0x90, 0x9D) else bytes([b]).decode("cp1252") for b in range(256) + ) + + def name(self, *, label: str) -> str: + """The standard's name, in lower case, for an encoding's label. + + Raises + ------ + ValueError + For a label of an encoding this whyqd does not decode, naming those it does. + + Example + ------- + ```python + EncodingParser().name(label="Latin1") # "windows-1252" + ``` + """ + label = label.strip().lower() + for name, encoding in settings.ENCODINGS.items(): + if label in encoding["labels"]: + return name + raise ValueError( + f"whyqd does not decode `{label}`. Declare one of {', '.join(settings.ENCODINGS)}, or no encoding. " + "[whyqd:encoding-unknown]" + ) + + def decode(self, *, data: bytes, declared: str | None = None, source: str = "The file") -> tuple[str, str, EncodingStep]: + """A file's text, the encoding that decoded it, and the step that settled it. + + A declared encoding that cannot decode some bytes still loads: each such byte reads as U+FFFD, as the standard's + replacement mode reads it, and one warning says how many, and what to declare instead. + + Parameters + ---------- + data: bytes + The file. + declared: str | None, default None + The encoding the wrangler declared, by any of its labels. + source: str, default "The file" + The file's name, for the warning. + + Returns + ------- + tuple[str, str, EncodingStep] + The text, the encoding's name, and the step. + """ + for mark, name in self.BOMS: + if data.startswith(mark): + return self.read(data=data[len(mark):], name=name, source=source), name, EncodingStep.BOM + if declared is not None: + name = self.name(label=declared) + return self.read(data=data, name=name, source=source), name, EncodingStep.DECLARED + try: + return data.decode("utf-8"), "utf-8", EncodingStep.UTF8 + except UnicodeDecodeError: + return self.read(data=data, name="windows-1252"), "windows-1252", EncodingStep.WINDOWS_1252 + + def read(self, *, data: bytes, name: str, source: str = "The file") -> str: + if name == "windows-1252": + return codecs.charmap_decode(data, "strict", self.WINDOWS_1252)[0] + codec = settings.ENCODINGS[name]["codec"] + text, replaced, start, view = [], [], 0, memoryview(data) + while True: + try: + text.append(codecs.decode(view[start:], codec)) + break + except UnicodeDecodeError as e: + text.append(codecs.decode(view[start : start + e.start], codec)) + replaced.append((start + e.start, bytes(view[start + e.start : start + e.end]))) + text.append("�") + start += e.end + if replaced: + self.warn(data=data, name=name, replaced=replaced, source=source) + return "".join(text) + + def warn(self, *, data: bytes, name: str, replaced: list[tuple[int, bytes]], source: str) -> None: + line = {} + for position, value in replaced: + if value not in line: + line[value] = data.count(b"\n", 0, position) + 1 + commonest = ", ".join( + f"{value.hex(' ').upper()} ({count}, first on line {line[value]})" + for value, count in Counter(v for _, v in replaced).most_common(3) + ) + warnings.warn( + f"{source} is declared `{name}`, but {len(replaced)} of its bytes are not, and read as � (commonest: " + f"{commonest}). Declare its own encoding, most often `windows-1252`. [whyqd:encoding-replaced]", + UserWarning, + stacklevel=2, + ) diff --git a/whyqd/parsers/excel_reader.py b/whyqd/parsers/excel_reader.py new file mode 100644 index 0000000..dfbe012 --- /dev/null +++ b/whyqd/parsers/excel_reader.py @@ -0,0 +1,211 @@ +# SPDX-License-Identifier: BSD-3-Clause +import posixpath +import zipfile +from pathlib import Path +from xml.etree import ElementTree + +import fastexcel +import polars as pl +import python_calamine + +from whyqd.config.patterns import PythonPatterns +from whyqd.dtypes import MimeType +from whyqd.models import DialectModel +from whyqd.parsers.cell import CellParser +from whyqd.parsers.grid import GridParser + + +class ExcelParser: + """A workbook's sheet as the grid: each cell its stored value in whyqd's text, never as displayed. + + Every sheet is read, a hidden one included, since it is the publisher's data; each is its own table. A cell is read + from A1, so a sheet's rows are numbered as the spreadsheet numbers them. python-calamine gives each cell's value, + but reads an error cell as empty, so fastexcel names the error cells, each read as its code (`#N/A`). + """ + + PLACE = "row" + # Each error's code, as Excel shows it: calamine's `CellErrorType` (https://github.com/tafia/calamine, `lib.rs`) + ERRORS = { + "Div0": "#DIV/0!", + "NA": "#N/A", + "Name": "#NAME?", + "Null": "#NULL!", + "Num": "#NUM!", + "Ref": "#REF!", + "Value": "#VALUE!", + "GettingData": "#DATA!", + } + # Each built-in format's code, stored as an id alone, for the ids every language shares: ECMA-376 Part 1, + # section 18.8.30 (https://ecma-international.org/publications-and-standards/standards/ecma-376/) + BUILT_IN = { + 0: "General", + 1: "0", + 2: "0.00", + 3: "#,##0", + 4: "#,##0.00", + 9: "0%", + 10: "0.00%", + 11: "0.00E+00", + 12: "# ?/?", + 13: "# ??/??", + 14: "mm-dd-yy", + 15: "d-mmm-yy", + 16: "d-mmm", + 17: "mmm-yy", + 18: "h:mm AM/PM", + 19: "h:mm:ss AM/PM", + 20: "h:mm", + 21: "h:mm:ss", + 22: "m/d/yy h:mm", + 37: "#,##0 ;(#,##0)", + 38: "#,##0 ;[Red](#,##0)", + 39: "#,##0.00;(#,##0.00)", + 40: "#,##0.00;[Red](#,##0.00)", + 45: "mm:ss", + 46: "[h]:mm:ss", + 47: "mmss.0", + 48: "##0.0E+0", + 49: "@", + } + + def __init__(self) -> None: + self.cell = CellParser() + self.grids = GridParser() + self.patterns = PythonPatterns() + + def sheets(self, *, source: Path) -> list[tuple[str, bool]]: + """Each worksheet's name, in workbook order, and whether its publisher hid it.""" + workbook = python_calamine.CalamineWorkbook.from_path(str(source)) + return [ + (m.name, m.visible != python_calamine.SheetVisibleEnum.Visible) + for m in workbook.sheets_metadata + if m.typ == python_calamine.SheetTypeEnum.WorkSheet + ] + + def grid( + self, *, source: Path, mime: MimeType, dialect: DialectModel + ) -> tuple[pl.DataFrame, pl.DataFrame, list[int], DialectModel]: + """A sheet's grid and each cell's stored type, the spreadsheet's number for each grid row, and the dialect + naming the sheet. + + Parameters + ---------- + source: Path + mime: MimeType + Read only to share the interface every reader has. + dialect: DialectModel + Its `sheet` names the sheet; a workbook of one sheet needs none. + + Raises + ------ + ValueError + For a sheet the workbook does not have, naming those it has. + """ + names = [name for name, _ in self.sheets(source=source)] + sheet = dialect.sheet + if sheet is None and len(names) == 1: + sheet = names[0] + if sheet not in names: + raise ValueError( + f"`{source.name}` has no sheet `{sheet}`. Name one of `{'`, `'.join(names)}` in the dialect's `sheet`. " + "[whyqd:sheet-unknown]" + ) + read = python_calamine.CalamineWorkbook.from_path(str(source)).get_sheet_by_name(sheet) + rows = read.to_python(skip_empty_area=False) + errors = self.errors(source=source, sheet=sheet, start=read.start) + width = max((len(r) for r in rows), default=0) + columns = {f"column_{j}": [] for j in range(width)} + types = {f"column_{j}": [] for j in range(width)} + for i, row in enumerate(rows): + for j in range(width): + value = row[j] if j < len(row) else None + code = errors.get((i, j)) + rendered = self.cell.render(value=code, error=True) if code else self.cell.render(value=value) + columns[f"column_{j}"].append(None if rendered is None else rendered[1]) + types[f"column_{j}"].append(None if rendered is None else rendered[0].value) + schema = {c: pl.String for c in columns} + grid, types = self.grids.trim(grid=pl.DataFrame(columns, schema=schema), types=pl.DataFrame(types, schema=schema)) + return grid, types, list(range(1, grid.height + 1)), dialect.model_copy(update={"sheet": sheet}) + + def formats(self, *, source: Path, sheet: str, height: int, width: int) -> pl.DataFrame: + """Each cell's number format code in a sheet of an `.xlsx`, in the shape of its grid. + + No engine whyqd reads a workbook with gives a cell's format, so it is read from the workbook's XML: the code + the workbook stores, a built-in id's from ECMA-376, or `id 5` for an id the standard leaves to the application. + An `.xls` stores its formats in another form, and is not read for them. + """ + codes = {f"column_{j}": [None] * height for j in range(width)} + with zipfile.ZipFile(source) as book: + workbook = self.parts(book=book, path="_rels/.rels", where="")["officeDocument"][0][0] + folder, name = posixpath.split(workbook) + parts = self.parts(book=book, path=posixpath.join(folder, "_rels", f"{name}.rels"), where=folder) + ids = {} + for element in ElementTree.fromstring(book.read(workbook)).iter(): + if self.local(element.tag) == "sheet": + ids[element.get("name")] = next(v for k, v in element.attrib.items() if self.local(k) == "id") + path = next(target for target, i in parts["worksheet"] if i == ids[sheet]) + styles = self.styles(book=book, path=parts["styles"][0][0]) if "styles" in parts else [] + row = column = -1 + with book.open(path) as xml: + for event, element in ElementTree.iterparse(xml, events=("start", "end")): + tag = self.local(element.tag) + if event == "start" and tag == "row": + row, column = (int(element.get("r")) - 1 if element.get("r") else row + 1), -1 + elif event == "end" and tag == "c": + if reference := element.get("r"): + letters, number = self.patterns.CELL_REFERENCE.fullmatch(reference).groups() + row, column = int(number) - 1, self.column(letters=letters) + else: + column += 1 + style = int(element.get("s", 0)) + if row < height and column < width: + codes[f"column_{column}"][row] = styles[style] if style < len(styles) else self.BUILT_IN[0] + element.clear() + return pl.DataFrame(codes, schema={c: pl.String for c in codes}) + + def parts(self, *, book: zipfile.ZipFile, path: str, where: str) -> dict[str, list[tuple[str, str]]]: + """A relationships part's targets by their type's last word (`worksheet`, `styles`), each with its id.""" + found = {} + for element in ElementTree.fromstring(book.read(path)).iter(): + if self.local(element.tag) == "Relationship": + target = element.get("Target") + target = target.lstrip("/") if target.startswith("/") else posixpath.normpath(posixpath.join(where, target)) + found.setdefault(element.get("Type").rsplit("/", 1)[-1], []).append((target, element.get("Id"))) + return found + + def styles(self, *, book: zipfile.ZipFile, path: str) -> list[str]: + """Each cell style's format code, by the style's index (`cellXfs`).""" + root = ElementTree.fromstring(book.read(path)) + custom = { + int(e.get("numFmtId")): e.get("formatCode") for e in root.iter() if self.local(e.tag) == "numFmt" + } + cell_styles = next((e for e in root.iter() if self.local(e.tag) == "cellXfs"), []) + codes = [] + for xf in cell_styles: + number = int(xf.get("numFmtId", 0)) + codes.append(custom.get(number, self.BUILT_IN.get(number, f"id {number}"))) + return codes + + def column(self, *, letters: str) -> int: + """A column's index from 0, by its letters: `A` is 0, `AA` 26.""" + index = 0 + for letter in letters: + index = index * 26 + ord(letter) - 64 + return index - 1 + + def local(self, tag: str) -> str: + """An XML name without its namespace, so a workbook in either of ECMA-376's namespaces reads.""" + return tag.rsplit("}", 1)[-1] + + def errors(self, *, source: Path, sheet: str, start: tuple[int, int] | None) -> dict[tuple[int, int], str]: + """Each error cell's code, by its row and column from A1.""" + if start is None: + return {} + read = fastexcel.read_excel(str(source)).load_sheet(sheet, header_row=None, dtypes="string") + _, found = read.to_arrow_with_errors() + codes = {} + for error in found.errors if found else []: + if kind := self.patterns.EXCEL_ERROR.search(error.detail): + row, column = error.position + codes[(start[0] + row, start[1] + column)] = self.ERRORS.get(kind.group(1), kind.group(1)) + return codes diff --git a/whyqd/parsers/fingerprint.py b/whyqd/parsers/fingerprint.py new file mode 100644 index 0000000..6950757 --- /dev/null +++ b/whyqd/parsers/fingerprint.py @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: BSD-3-Clause +from __future__ import annotations +from pathlib import Path +from typing import TYPE_CHECKING +import hashlib +import json + +if TYPE_CHECKING: + import polars as pl + + +class FingerprintParser: + """Two proofs of a source file: its byte hash, which any byte changes, and `fp0`, which only a cell changes. + + `fp0` hashes the file's grid as its reader parses it, every sheet in workbook order, a hidden one included, before + any row is skipped, joined, or taken as the header, and before any cell is trimmed: it proves the file, not what + is read from it. Each cell is its stored type and whyqd's text (`CellType`, `CellParser`), so `fp0` is the same + through any reader and in any encoding, and a file saved again with no cell changed, such as after a column is + widened, keeps it. Both are BLAKE2b-512. + + The form hashed: `fp0`, then for each sheet a line holding `{"sheet": name}` (`""` for a file without sheets), + then a line for each row of its grid, trailing empty rows and columns trimmed, a JSON array of each cell as + `[type, text]`, or `null` where empty; UTF-8, lines joined by `\\n`. + """ + + VERSION = "fp0" + + def byte_hash(self, *, source: str | Path) -> str: + """The file's bytes, hashed.""" + checksum = hashlib.blake2b() + with open(source, "rb") as f: + for chunk in iter(lambda: f.read(65536), b""): + checksum.update(chunk) + return checksum.hexdigest() + + def fp0(self, *, sheets: list[tuple[str, pl.DataFrame, pl.DataFrame]]) -> str: + """The file's `fp0`, from each sheet's name, grid, and types, in workbook order. + + Example + ------- + ```python + grid = pl.DataFrame({"column_0": ["Code", "12"]}) + types = pl.DataFrame({"column_0": ["s", "n"]}) + FingerprintParser().fp0(sheets=[("Data", grid, types)]) + ``` + """ + checksum = hashlib.blake2b() + checksum.update(self.VERSION.encode()) + for name, grid, types in sheets: + checksum.update(b"\n" + json.dumps({"sheet": name}, ensure_ascii=False).encode("utf-8")) + for texts, tags in zip(grid.iter_rows(), types.iter_rows(), strict=True): + row = [None if text is None else [tag, text] for text, tag in zip(texts, tags, strict=True)] + checksum.update(b"\n" + json.dumps(row, ensure_ascii=False, separators=(",", ":")).encode("utf-8")) + return checksum.hexdigest() diff --git a/whyqd/parsers/format.py b/whyqd/parsers/format.py index 58f9409..f2d0915 100644 --- a/whyqd/parsers/format.py +++ b/whyqd/parsers/format.py @@ -38,6 +38,7 @@ class FormatParser: SCHEMAS = ("schemaSource", "schemaDestination") NEW = ( "dialect", + "hidden", "profile", "policy", "termList", @@ -131,9 +132,9 @@ def write(self, *, model: BaseModel, format: str | None = None, hide_uuid: bool return {self.KEY: version} | self.name_schemas(self.drop_empty_attributes(written)) def upgrade(self, value): - """1.x to the current format: a data source's bare checksum names what it hashed; a field's `time` is a - `duration`; and a string field's `minimum` and `maximum`, which 1.x described as lengths, are `minLength` and - `maxLength`.""" + """1.x to the current format: a data source's bare checksum names what it hashed, and it gains the dialect that + reads it as 1.x did; a field's `time` is a `duration`; and a string field's `minimum` and `maximum`, which 1.x + described as lengths, are `minLength` and `maxLength`.""" if isinstance(value, list): return [self.upgrade(v) for v in value] if not isinstance(value, dict): @@ -141,6 +142,8 @@ def upgrade(self, value): value = {k: self.upgrade(v) for k, v in value.items()} if "mime" in value and isinstance(value.get("checksum"), str): value["checksums"] = [self.checksum(value=value.pop("checksum"), content=ChecksumContent.TABLE_1X)] + if "dialect" not in value and (dialect := self.dialect_1x(value)) is not None: + value["dialect"] = dialect for field in value.get("fields") or []: if field.get("type") == FieldType.TIME.value: field["type"] = FieldType.DURATION.value @@ -151,14 +154,17 @@ def upgrade(self, value): def downgrade(self, value): """The current format to 1.x, which holds one checksum per data source: the table as 1.x read it. - The byte hash and an embedded schema's content checksum are left out, since whyqd recomputes both. An edit's - time is written without its zone, as 1.x wrote it, being UTC. A part new in 2.0 is refused, naming it. + The byte hash and an embedded schema's content checksum are left out, since whyqd recomputes both, and so is a + dialect that says only what the data source's 1.x settings say. An edit's time is written without its zone, as + 1.x wrote it, being UTC. A part new in 2.0 is refused, naming it. """ if isinstance(value, list): return [self.downgrade(v) for v in value] if not isinstance(value, dict): return value value = {k: self.downgrade(v) for k, v in value.items()} + if "mime" in value and "dialect" in value and value["dialect"] == self.dialect_1x(value): + value.pop("dialect") for key in self.NEW: if key in value: raise ValueError( @@ -180,6 +186,20 @@ def downgrade(self, value): value["checksum"] = table[0]["checksumValue"] return value + def dialect_1x(self, value: dict) -> dict | None: + """The dialect that reads a 1.x data source's rows as 1.x read them, as written, or `None` for one read with + pandas' `attributes`, which no dialect holds. + + 1.x named the columns from row `header` and kept every row as data, so a `header` below row 0, or none, reads as + a file with no header row, by the names 1.x read and stored; `sheet_name` is the dialect's `sheet`. + """ + if value.get("attributes"): + return None + dialect = {} if value.get("header", 0) == 0 else {"headerRowCount": 0} + if value.get("sheet_name"): + dialect["sheet"] = value["sheet_name"] + return dialect + def downgrade_field(self, field: dict) -> None: """Write a field's type and bounds as 1.x names them, refusing a time of day, which 1.x's `time` is not, and a length on anything but a string.""" diff --git a/whyqd/parsers/grid.py b/whyqd/parsers/grid.py new file mode 100644 index 0000000..7e4f275 --- /dev/null +++ b/whyqd/parsers/grid.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: BSD-3-Clause +import polars as pl + +from whyqd.dtypes import CellType + + +class GridParser: + """The grid every reader gives: each cell's text, null where empty, in columns `column_0` on, and beside it each + cell's stored type, a `CellType`, in a frame of the same shape. + + A format that stores no types (CSV, PDF) holds text only, so each of its cells is `s`. A reader's grid is trimmed + of trailing empty rows and columns, so a file's grid does not depend on how far a program that saved it left empty + cells: a trailing delimiter, or a workbook's used range past its data. + """ + + def trim(self, *, grid: pl.DataFrame, types: pl.DataFrame | None = None) -> tuple[pl.DataFrame, pl.DataFrame]: + """The grid and its types without the grid's trailing empty columns and rows. + + Parameters + ---------- + grid: pl.DataFrame + types: pl.DataFrame | None, default None + Each cell's `CellType` value, in the grid's shape; where `None`, every cell holding text is `s`. + + Example + ------- + ```python + grid = pl.DataFrame({"column_0": ["a", None], "column_1": [None, None]}) + GridParser().trim(grid=grid) # one column, one row: "a", typed "s" + ``` + """ + filled = [c for c in grid.columns if grid[c].null_count() < grid.height] + if not filled: + grid = grid.clear().select([]) + else: + grid = grid.select(grid.columns[: grid.columns.index(filled[-1]) + 1]) + rows = grid.select(pl.any_horizontal(pl.all().is_not_null())).to_series() + grid = grid.head(rows.arg_true().max() + 1) + if types is None: + types = grid.select( + pl.when(pl.col(c).is_not_null()).then(pl.lit(CellType.STRING.value)).alias(c) for c in grid.columns + ) + else: + types = types.select(grid.columns).head(grid.height) + return grid, types diff --git a/whyqd/parsers/morph.py b/whyqd/parsers/morph.py index 9dece41..021c55a 100644 --- a/whyqd/parsers/morph.py +++ b/whyqd/parsers/morph.py @@ -103,15 +103,10 @@ def parse( raise ValueError(f"Script error for {action.name} Morph. Fields are not a valid input.") if not (source or destination) and "fields" in action.structure: raise ValueError(f"Script error for {action.name} Morph. No valid input fields provided.") + # The rows are checked against the data source's row count where the crosswalk meets it, in the transform if rows: - if not self.row_indices: - raise ValueError( - f"Script error for {action.name} Morph. Schema does not define row indices for a row-level action." - ) if "rows" not in action.structure: raise ValueError(f"Script error for {action.name} Morph. Rows are not a valid input.") - elif not set(rows) <= set(self.row_indices): - raise ValueError("Not all row indices found in source data.") else: if "rows" in action.structure: raise ValueError(f"Script error for {action.name} Morph. No valid input rows provided.") @@ -189,9 +184,6 @@ def set_schema( if not schema_source or not schema_destination: raise ValueError("Schema for both source and destination has not been provided.") self.schema = [schema_source, schema_destination] - self.row_indices = None - if schema_source.get.index: - self.row_indices = list(range(schema_source.get.index)) def get_hexed_script(self, *, script: str) -> str: # Changes fields to uuid hexes diff --git a/whyqd/parsers/pdf_reader.py b/whyqd/parsers/pdf_reader.py new file mode 100644 index 0000000..60dfe58 --- /dev/null +++ b/whyqd/parsers/pdf_reader.py @@ -0,0 +1,125 @@ +# SPDX-License-Identifier: BSD-3-Clause +from pathlib import Path + +import polars as pl + +from whyqd.dtypes import MimeType +from whyqd.models import DialectModel +from whyqd.parsers.grid import GridParser + + +class PdfParser: + """A PDF's table as the grid, at best effort: a PDF holds a printed page, not a table. + + What the file shows is read: whether it holds text, and whether its ruling gives a table, which sets how cells are + found (`cells`). What it cannot show is declared: the pages, an unruled table's columns, and the names. A cell keeps + its line breaks, since a wrap and a break look the same on a page. A row equal to a header row is that header + repeated atop a later page, and is left out. Read through pdfplumber, the `pdf` extra. + """ + + PLACE = "row" + # Points between the tops of two words on one line: pdfplumber's own `y_tolerance` + LINE = 3 + + def __init__(self) -> None: + self.grids = GridParser() + + def grid( + self, *, source: Path, mime: MimeType, dialect: DialectModel + ) -> tuple[pl.DataFrame, pl.DataFrame, list[int], DialectModel]: + """The table's grid and its types, every cell text, each grid row's number in the order read, and the dialect + with `cells` recorded. It has no `fp0`, since the grid depends on how the PDF is read: its `cells`, `columns`, + and `pages`. + + Raises + ------ + ImportError + Without the `pdf` extra. + ValueError + For a PDF with no text, such as a scan, or pages it does not have. + """ + try: + import pdfplumber + except ImportError as e: + raise ImportError("Reading a PDF needs the extra: pip install whyqd[pdf] [whyqd:pdf-extra]") from e + with pdfplumber.open(source) as pdf: + pages = self.pages(pages=pdf.pages, declared=dialect.pages, source=source) + if not any(page.chars for page in pages): + raise ValueError( + f"`{source.name}` holds no text on {'its pages' if dialect.pages is None else 'pages ' + dialect.pages}, " + "only images: it is a scan. Recognise its text first (OCR), or load the file it was printed from. " + "[whyqd:pdf-scan]" + ) + if dialect.columns: + cells = "text" + rows = [row for page in pages for row in self.lines(page=page, columns=dialect.columns)] + else: + cells = dialect.cells or ("lines" if any(page.find_tables() for page in pages) else "text") + rows = [] + for page in pages: + for table in page.extract_tables(self.settings(cells=cells)): + rows.extend(table) + rows = self.unrepeated(rows=rows, dialect=dialect) + width = max((len(r) for r in rows), default=0) + grid = pl.DataFrame( + {f"column_{i}": [r[i] if i < len(r) and r[i] else None for r in rows] for i in range(width)}, + schema={f"column_{i}": pl.String for i in range(width)}, + ) + grid, types = self.grids.trim(grid=grid) + return grid, types, list(range(1, grid.height + 1)), dialect.model_copy(update={"cells": cells}) + + def pages(self, *, pages: list, declared: str | None, source: Path) -> list: + """The pages a dialect's `pages` names, counted from 1, or every page.""" + if declared is None: + return pages + chosen = [] + for part in declared.replace(" ", "").split(","): + first, _, last = part.partition("-") + try: + numbers = range(int(first), int(last or first) + 1) + except ValueError as e: + raise ValueError( + f"`pages` `{declared}` names pages by number from 1: `5-7`, or `1,3,5-7`. [whyqd:pdf-pages]" + ) from e + if not numbers or numbers.start < 1 or numbers.stop - 1 > len(pages): + raise ValueError( + f"`pages` `{declared}` names a page `{source.name}` does not have: it has {len(pages)}. " + "[whyqd:pdf-pages]" + ) + chosen.extend(pages[n - 1] for n in numbers) + return chosen + + def settings(self, *, cells: str) -> dict: + """pdfplumber's table settings for how cells are found.""" + if cells == "lines": + return {} + return {"vertical_strategy": "text", "horizontal_strategy": "text"} + + def lines(self, *, page, columns: list[float]) -> list[list[str | None]]: + """A page's rows under declared columns: each line of text a row, each word in the column it starts in. + + A word belongs to the last column whose left edge it starts at or after, one left of the first edge to the + first; words whose tops are within `LINE` points of each other are one line, and a cell's words are joined by + one space, so a run of spaces in the PDF reads as one. pdfplumber finds no cells where a mostly empty column + meets the page's edge, so the words are placed here, by the declared edges alone. + """ + rows, top = [], None + for word in sorted(page.extract_words(), key=lambda w: (round(w["top"]), w["x0"])): + if top is None or word["top"] - top > self.LINE: + rows.append([[] for _ in columns]) + top = word["top"] + column = max([0, *(i for i, edge in enumerate(columns) if word["x0"] >= edge)]) + rows[-1][column].append(word) + return [[" ".join(w["text"] for w in sorted(cell, key=lambda w: w["x0"])) or None for cell in row] for row in rows] + + def unrepeated(self, *, rows: list[list[str | None]], dialect: DialectModel) -> list[list[str | None]]: + """The rows without any equal to a header row after the header, each being that header atop a later page.""" + start, end = dialect.skipRows, dialect.skipRows + dialect.headerRowCount + header = [self.row(r) for r in rows[start:end]] + return rows[:end] + [r for r in rows[end:] if self.row(r) not in header] + + def row(self, row: list[str | None]) -> tuple[str, ...]: + cells = [c or "" for c in row] + while cells and not cells[-1]: + cells.pop() + return tuple(cells) diff --git a/whyqd/parsers/profile.py b/whyqd/parsers/profile.py new file mode 100644 index 0000000..4280250 --- /dev/null +++ b/whyqd/parsers/profile.py @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: BSD-3-Clause +import polars as pl + +from whyqd.config.patterns import PolarsPatterns +from whyqd.dtypes import CellType +from whyqd.models import ProfileModel + + +class ProfileParser: + """What a column holds, as read, in counts. Profiling never changes a cell. + + Any cell may hold a hard space, a zero-width character, or a control character (`PolarsPatterns`); only a text + cell is zero-padded (`0012`) or date-like. Format codes are counted for number and date cells alone. + """ + + FORMATTED = [CellType.NUMBER.value, CellType.DATE.value] + + def __init__(self) -> None: + self.patterns = PolarsPatterns() + + def profile(self, *, text: pl.Series, types: pl.Series, formats: pl.Series | None = None) -> ProfileModel: + """A column's profile from its cells' text, their stored types, and, for an `.xlsx`, their format codes. + + Parameters + ---------- + text: pl.Series + Each cell's text as stored, null where empty. + types: pl.Series + Each cell's `CellType` tag. + formats: pl.Series | None, default None + Each cell's number format code; none for a file that stores none. + """ + p = self.patterns + found = dict(types.drop_nulls().value_counts().iter_rows()) + is_text = types == CellType.STRING.value + counts = {} + if formats is not None: + coded = dict(formats.filter(types.is_in(self.FORMATTED)).drop_nulls().value_counts().iter_rows()) + counts = dict(sorted(coded.items(), key=lambda c: (-c[1], c[0]))) + return ProfileModel( + cellTypes={t.value: found[t.value] for t in CellType if t.value in found}, + blank=text.null_count(), + whitespace=text.str.contains(p.WHITESPACE).sum(), + hardSpace=text.str.contains(p.HARD_SPACE).sum(), + zeroWidth=text.str.contains(p.ZERO_WIDTH).sum(), + control=text.str.contains(p.CONTROL).sum(), + dateLike=( + is_text & (text.str.contains(p.DATE) | (text.str.contains(p.MONTH) & text.str.contains(p.DIGIT))) + ).sum(), + zeroPadded=(is_text & text.str.contains(p.ZERO_PADDED)).sum(), + numberFormats=counts, + ) diff --git a/whyqd/parsers/row_key.py b/whyqd/parsers/row_key.py index b343e56..55e5189 100644 --- a/whyqd/parsers/row_key.py +++ b/whyqd/parsers/row_key.py @@ -2,16 +2,18 @@ import re from typing import Iterable +from whyqd.config.patterns import PythonPatterns +from whyqd.models.dialect import DialectModel from whyqd.parsers.column import ColumnParser class RowKeyParser: """The row key: every row's number from the read, carried through every action. - Rows are numbered from `0` by their position in the source as read, each sheet from `0`. With the header at row - `0`, row `0` is the first row after it; with no header, another header row, or given names, row `0` is the file's - first row. An action never renumbers a row, so `DELETE_ROWS` and `PIVOT_CATEGORIES` address the same rows wherever - they run. + Rows are numbered from `0` by their position in the table as read, each sheet from `0`: row `0` is the first row + after the header, or the file's first where its dialect declares no header row (`headerRowCount` 0). A 1.x data + source reads by a dialect with no header row, so its rows are numbered as 1.x numbered them. An action never + renumbers a row, so `DELETE_ROWS` and `PIVOT_CATEGORIES` address the same rows wherever they run. The key is the column `__row__`. An action that turns one row into several adds a part, `__row__1` and so on, holding the row's position within the rows it made; sorting by every part, in order, gives the table its order. @@ -25,10 +27,11 @@ class RowKeyParser: """ ROW = "__row__" - _PART = re.compile(rf"^{ROW}(\d*)$") def __init__(self) -> None: self.column = ColumnParser() + self.patterns = PythonPatterns() + self._part = re.compile(self.patterns.ROW_PART.format(row=re.escape(self.ROW))) def part(self, *, index: int = 0) -> str: """Name of a part of the row key. @@ -56,7 +59,7 @@ def part(self, *, index: int = 0) -> str: def is_part(self, column: str) -> bool: """Whether a column name is a part of the row key.""" - match = self._PART.match(column) + match = self._part.match(column) return bool(match) and match.group(1) != "0" def deconflict(self, *, columns: Iterable[str]) -> list[str]: @@ -74,20 +77,21 @@ def deconflict(self, *, columns: Iterable[str]) -> list[str]: """ return self.column.deconflict(columns=columns, reserved=self.is_part) - def first_position(self, *, header: int | None) -> int: - """Position in the file, counted from `0`, of the row numbered `0`. + def first_position(self, *, dialect: DialectModel) -> int: + """Position in the file's rows, counted from `0`, of the row numbered `0`. Parameters ---------- - header: int | None - The source's header row, as `DataSourceDefinition.derive_model` takes it. + dialect: DialectModel + How the file is read. Returns ------- int - `1` with the header at row `0`; otherwise `0`, since every row is read as data. + The rows skipped before the header, and the header's own, each of them a record's rows where a record + spans several. """ - return 1 if header == 0 else 0 + return dialect.skipRows + dialect.headerRowCount * len(dialect.rowsPerRecord or [1]) def sort_order(self, *, columns: Iterable[str]) -> list[str]: """The row key's parts among `columns`, in the order to sort a table by. @@ -102,4 +106,4 @@ def sort_order(self, *, columns: Iterable[str]) -> list[str]: `__row__` first, then each added part by its number. """ parts = [c for c in columns if self.is_part(c)] - return sorted(parts, key=lambda c: int(self._PART.match(c).group(1) or 0)) + return sorted(parts, key=lambda c: int(self._part.match(c).group(1) or 0)) diff --git a/whyqd/parsers/table.py b/whyqd/parsers/table.py new file mode 100644 index 0000000..4fed0de --- /dev/null +++ b/whyqd/parsers/table.py @@ -0,0 +1,142 @@ +# SPDX-License-Identifier: BSD-3-Clause +import polars as pl + +from whyqd.models import DialectModel +from whyqd.parsers.row_key import RowKeyParser + + +class TableParser: + """The grid as a table: the rows below the header, each column named once, each row keyed. + + In order: the rows before the header are skipped (`skipRows`); records over several rows are joined + (`rowsPerRecord`); each cell is trimmed (`trim`), one left empty becoming empty; the header rows + (`headerRowCount`) name the columns and leave the table; the row key is added. Nothing is inferred from the data: + every setting is the dialect's, with its defaults. Frames beside the grid, such as each cell's stored type, take + the same rows, so a column can be profiled as its cells are stored. + """ + + TRIM = {True: str.strip, "start": str.lstrip, "end": str.rstrip} + + def __init__(self) -> None: + self.row_key = RowKeyParser() + + def table( + self, + *, + grid: pl.DataFrame, + lines: list[int], + dialect: DialectModel, + names: list[str] | None = None, + place: str = "line", + beside: list[pl.DataFrame] | None = None, + ) -> tuple[pl.DataFrame, list[pl.DataFrame]]: + """The table a grid reads as under its dialect, its first column the row key, and the frames beside it. + + Parameters + ---------- + grid: pl.DataFrame + Each cell's text, null where empty, as a reader gives it. + lines: list[int] + Where in the file each grid row starts, counted from 1, for messages. + dialect: DialectModel + names: list[str] | None, default None + One name per column, in place of the header row's; required for a header over several rows. + place: str, default "line" + What `lines` counts, as the wrangler would look for it: a text file's `line`, a sheet's `row`. + beside: list[pl.DataFrame] | None, default None + Frames in the grid's shape, such as each cell's stored type. + + Raises + ------ + ValueError + For a declaration the file does not fit, or one that is incomplete. + + Returns + ------- + tuple[pl.DataFrame, list[pl.DataFrame]] + The table: row `0` is the first row after the header; each column's name is its only name. A grid with no + cells gives a table of no columns and no rows. Then each frame beside it, its rows and names the table's, + without the row key, and its cells untrimmed. + """ + beside = beside or [] + if grid.width == 0: + return pl.DataFrame({self.row_key.ROW: []}, schema={self.row_key.ROW: pl.UInt32}), [pl.DataFrame()] * len(beside) + frames, lines = [f.slice(dialect.skipRows) for f in [grid, *beside]], lines[dialect.skipRows :] + if dialect.rowsPerRecord: + self.fit(grid=frames[0], lines=lines, fields=dialect.rowsPerRecord, place=place) + frames = [self.join(grid=f, fields=dialect.rowsPerRecord) for f in frames] + lines = lines[:: len(dialect.rowsPerRecord)] + if trim := self.TRIM.get(dialect.trim): + frames[0] = frames[0].with_columns( + pl.col(c).map_elements(lambda v: trim(v) or None, return_dtype=pl.String) for c in frames[0].columns + ) + header = frames[0].head(dialect.headerRowCount) + frames = [f.slice(dialect.headerRowCount) for f in frames] + if header.height < dialect.headerRowCount: + raise ValueError( + f"The file ends before its header: `skipRows` {dialect.skipRows} and `headerRowCount` " + f"{dialect.headerRowCount} pass every row. Declare where the header starts. [whyqd:header-past-end]" + ) + if dialect.skipBlankRows: + kept = ~frames[0].select(pl.all_horizontal(pl.all().is_null())).to_series() + frames = [f.filter(kept) for f in frames] + columns = self.names(header=header, names=names, dialect=dialect) + for frame in frames: + frame.columns = columns + grid, *beside = frames + return grid.with_row_index(self.row_key.ROW), beside + + def names(self, *, header: pl.DataFrame, names: list[str] | None, dialect: DialectModel) -> list[str]: + width = header.width + if names is not None: + if len(names) != width: + raise ValueError( + f"`names` gives {len(names)} names for a table of {width} columns. Give one per column, in order. " + "[whyqd:names-count]" + ) + elif dialect.headerRowCount > 1: + raise ValueError( + f"`headerRowCount` is {dialect.headerRowCount}, so declare the names in `names`, one for each of its " + f"{width} columns, in order. [whyqd:header-names]" + ) + elif dialect.headerRowCount == 0: + names = [f"column_{i}" for i in range(width)] + else: + names = [f"Unnamed: {i}" if n is None else n for i, n in enumerate(header.row(0))] + return self.row_key.deconflict(columns=names) + + def fit(self, *, grid: pl.DataFrame, lines: list[int], fields: list[int], place: str = "line") -> None: + """Whether a grid's rows fit their records. + + Raises + ------ + ValueError + Naming the line where a row holds values past its fields, or where the file ends inside a record, since + every record after it would be misread. + """ + span, width = len(fields), grid.width + for i in range(grid.height): + n = fields[i % span] + if n < width and any(v is not None for v in grid.row(i)[n:]): + raise ValueError( + f"`rowsPerRecord` {fields} declares {n} field{'s' if n != 1 else ''} on {place} {lines[i]}, but " + "it holds more. Check the file there: a record may have lost a row. [whyqd:record-overflow]" + ) + if grid.height % span: + last = grid.height - grid.height % span + raise ValueError( + f"`rowsPerRecord` {fields} declares {span} rows to a record, but the last, from {place} " + f"{lines[last]}, has {grid.height - last}. [whyqd:record-short]" + ) + + def join(self, *, grid: pl.DataFrame, fields: list[int]) -> pl.DataFrame: + """Each record's rows joined into one: the first `fields[0]` cells of its first row, then the first + `fields[1]` of its next, and on.""" + span, width = len(fields), grid.width + parts = [] + for j, n in enumerate(fields): + part = grid.gather_every(span, offset=j).select(pl.nth(range(min(n, width)))) + for k in range(part.width, n): + part = part.with_columns(pl.lit(None, dtype=pl.String).alias(f"column_{k}")) + parts.append(part.rename(lambda c, j=j: f"{j}:{c}")) + return pl.concat(parts, how="horizontal") diff --git a/whyqd/parsers/v1.py b/whyqd/parsers/v1.py index 9842b26..08d671f 100644 --- a/whyqd/parsers/v1.py +++ b/whyqd/parsers/v1.py @@ -1,19 +1,23 @@ # SPDX-License-Identifier: BSD-3-Clause from __future__ import annotations -from typing import TYPE_CHECKING +from pathlib import Path +import hashlib -from whyqd.dtypes import ChecksumContent -from whyqd.parsers.datasource import DataSourceParser +import numpy as np +import pandas as pd -if TYPE_CHECKING: - from whyqd.models import DataSourceModel +from whyqd.dtypes import MimeType +from whyqd.models import DataSourceModel +from whyqd.parsers.core import CoreParser class V1ChecksumParser: """whyqd 1.x's data checksum: the table as 1.x reads it, printed by pandas, hashed with BLAKE2b-512. - Its result depends on how pandas prints a table, so it verifies 1.x checksums and makes no new ones. It reads - with the data source's 1.x settings (`header`, `sheet_name`, `attributes`), as 1.x did. + Its result depends on how pandas reads and prints a table, so this keeps 1.x's read unchanged, and with it both + ways 1.x checked a table: a data source derived again from its file (`checksum`), and a file read with its stored + settings (`read`), as a transform checks its source and destination. It verifies 1.x checksums and makes no new + ones. Example: ```python @@ -23,9 +27,166 @@ class V1ChecksumParser: ``` """ + def __init__(self) -> None: + self.core = CoreParser() + def checksum(self, *, model: DataSourceModel) -> str: + """Derive a data source again from its file, with its 1.x settings (`header`, `sheet_name`, `attributes`), + and return its checksum. + + Parameters: + model: A data source as 1.x derived it. + + Returns: + The checksum of the table, as 1.x wrote it. + """ attributes = model.attributes.terms if model.attributes else {} - derived = DataSourceParser().derive_data_model( - source=model.path, mimetype=model.mime, sheet_name=model.sheet_name, header=model.header, **attributes - ) - return derived.get_checksum(ChecksumContent.TABLE_1X) + path, header, sheet_name = model.path, model.header, model.sheet_name + if header is None: + df = self.read(source=path, mimetype=model.mime, sheet_name=sheet_name, nrows=50, **attributes) + names = [] + if not ((isinstance(df, dict) and not df) or df.empty): + names = [f"column_{i}" for i in range(len(df.columns))] + else: + nrows = header + 1 if header > 1 else 1 + df = self.read( + source=path, mimetype=model.mime, sheet_name=sheet_name, header=header, nrows=nrows, **attributes + ) + names = df.columns.tolist() + if header is None or header != 0: + df = self.read( + source=path, + mimetype=model.mime, + names=names, + preserve=names, + sheet_name=sheet_name, + header=None, + **attributes, + ) + else: + df = self.read(source=path, mimetype=model.mime, preserve=names, sheet_name=sheet_name, **attributes) + return self.table(df=df) + + def table(self, *, df: pd.DataFrame) -> str: + """Return the checksum of a table: pandas' printed table, as text, hashed. + + Parameters: + df: The table. Array cells are changed to lists in place. + + Returns: + The checksum. + """ + for column in df.columns: + if isinstance(df.iloc[0][column], np.ndarray): + df[column] = df[column].apply(lambda x: x.tolist()) + df_string = df.astype("string").astype("string").to_string(index=False).encode("utf-8") + return hashlib.blake2b(df_string).hexdigest() + + def read( + self, + *, + source: str | Path | DataSourceModel, + mimetype: str | MimeType | None = None, + header: int | None = 0, + preserve: str | list[str] | bool = False, + names: list[str] | None = None, + nrows: int | None = None, + sheet_name: str | None = None, + directory: str | Path | None = None, + **attributes, + ) -> dict[str, pd.DataFrame] | pd.DataFrame: + """Read a file as 1.x did, with pandas' keywords as 1.x's `attributes`. + + Parameters: + source: A path or URI, or a data source, whose settings then replace the others. + mimetype: CSV, XLS, XLSX, Feather, or Parquet. + header: The header row, counted from 0. + preserve: Columns read as text, or `True` for every column. CSV and Excel only. + names: Column names, for a file without a header row. CSV and Excel only. + nrows: The number of rows to read. + sheet_name: The sheet to read; by default every sheet that is not empty. + directory: Where a remote source is saved. + attributes: pandas' own keywords for the reader. + + Returns: + A dataframe, or one for each sheet. + """ + kwargs = {} | attributes + if isinstance(source, DataSourceModel): + if source.attributes: + kwargs |= source.attributes.terms + mimetype = source.mime + preserve = source.preserve + sheet_name = source.sheet_name + header = source.header + if source.names: + names = source.names + source = source.path + if self.core.check_uri(source=source): + source = self.core.download_uri_source(source=source, directory=directory) + if not self.core.check_source(source=source): + raise FileNotFoundError(f"Source at `{source}` not found.") + if not isinstance(mimetype, MimeType): + try: + mimetype = MimeType.value_of(mimetype) + except ValueError: + mimetype = MimeType(mimetype) + if mimetype in [MimeType.PRQ, MimeType.PARQUET]: + return self._read_parquet(source=source, nrows=nrows, **kwargs) + if mimetype in [MimeType.FTR, MimeType.FEATHER]: + return pd.read_feather(source, **kwargs) + if preserve: + if isinstance(preserve, bool): + kwargs["dtype"] = pd.StringDtype() + else: + if not isinstance(preserve, list): + preserve = [preserve] + kwargs["dtype"] = {k: pd.StringDtype() for k in preserve} + kwargs["header"] = header + if names: + kwargs["header"] = None + kwargs["names"] = names + if nrows: + kwargs["nrows"] = nrows + if mimetype in [MimeType.XLS, MimeType.XLSX]: + kwargs["sheet_name"] = sheet_name + # The engines 1.x read with, named so a change of pandas' default leaves the read, and its checksums, alone + kwargs.setdefault("engine", "xlrd" if mimetype == MimeType.XLS else "openpyxl") + return self._read_excel(source=source, **kwargs) + if mimetype == MimeType.CSV: + return self._read_csv(source=source, **kwargs) + raise FileExistsError(f"Source at `{source}` does not have a recognised mimetype `{mimetype}`.") + + def _read_excel(self, *, source: str | Path, **kwargs) -> dict[str, pd.DataFrame] | pd.DataFrame: + # Every sheet unless one is named; empty sheets dropped, and a single sheet left returned alone + if kwargs.get("sheet_name"): + return pd.read_excel(source, **kwargs) + df = pd.read_excel(source, **kwargs) + keys = list(df.keys()) + for k in keys: + if df[k].empty: + del df[k] + if len(df.keys()) == 1: + df = df[keys[0]] + return df + + def _read_csv(self, *, source: str | Path, **kwargs) -> pd.DataFrame: + # 1.x drops bytes that do not decode, and guesses the delimiter with the python engine + kwargs["encoding_errors"] = "ignore" + kwargs["engine"] = "python" + if not kwargs.get("nrows"): + return pd.read_csv(source, **kwargs) + kwargs["iterator"] = True + kwargs["chunksize"] = 10000 + return pd.concat(pd.read_csv(source, **kwargs), ignore_index=True) + + def _read_parquet(self, *, source: str | Path, nrows: int | None = None, **kwargs) -> pd.DataFrame: + if nrows: + import pyarrow as pa + from pyarrow.parquet import ParquetFile + + batch = next(ParquetFile(str(source)).iter_batches(batch_size=nrows)) + return pa.Table.from_batches([batch]).to_pandas(**kwargs) + if "engine" not in kwargs: + kwargs["engine"] = "pyarrow" + return pd.read_parquet(source, **kwargs)