diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7e7d70..cfc3496 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: testy +name: tests on: push: @@ -20,75 +20,75 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - - name: Instalacja pakietu z ekstrasem testowym + - name: Install the package with the test extra run: python -m pip install --upgrade pip && python -m pip install -e ".[test]" - - name: Testy + - name: Tests run: python -m pytest -q -rs - - name: Zgodnosc deklarowanej wersji + - name: Declared version is consistent shell: python run: | - # tomllib jest w bibliotece standardowej od Pythona 3.11, a macierz obejmuje 3.10, - # wiec wersje z pyproject.toml czytamy wyrazeniem regularnym zamiast parserem TOML. + # tomllib is in the standard library from Python 3.11 and the matrix includes 3.10, + # so the version in pyproject.toml is read with a regular expression, not a TOML parser. import pathlib import re import gridqueue - tekst = pathlib.Path("pyproject.toml").read_text(encoding="utf-8") - m = re.search(r'(?m)^\s*version\s*=\s*"([^"]+)"', tekst) - assert m, "nie znaleziono pola version w pyproject.toml" + text = pathlib.Path("pyproject.toml").read_text(encoding="utf-8") + m = re.search(r'(?m)^\s*version\s*=\s*"([^"]+)"', text) + assert m, "no version field found in pyproject.toml" assert m.group(1) == gridqueue.__version__, (m.group(1), gridqueue.__version__) - print("wersja zgodna:", gridqueue.__version__) + print("version consistent:", gridqueue.__version__) pandas3: - name: pandas 3 (zgodnosc w przod) + name: pandas 3 (forward compatibility) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: "3.13" - - name: Instalacja z najnowszym pandas - # Macierz wyzej testuje wersje Pythona, nie wersje bibliotek. Defekt budowy - # klucza wzdluznego ujawnil sie dopiero na pandas 3, gdzie .astype(str) - # zachowuje brak jako wartosc brakujaca zamiast zamieniac go na "nan". - # To zadanie pilnuje, zeby taka zmiana zachowania nie przeszla niezauwazona. + - name: Install with the latest pandas + # The matrix above tests Python versions, not library versions. A defect in building + # the longitudinal key appeared only on pandas 3, where .astype(str) keeps a missing + # value as missing instead of turning it into "nan". This job makes sure that such a + # change of behaviour does not pass unnoticed. run: | python -m pip install --upgrade pip python -m pip install -e ".[test]" python -m pip install --upgrade "pandas>=3.0" - - name: Zadanie faktycznie dziala na pandas 3 + - name: The job really runs on pandas 3 shell: python run: | - # Bez tej asercji zadanie przechodziloby POZORNIE, gdyby pandas 3 nie - # zainstalowal sie z jakiegokolwiek powodu, a wtedy nie pilnowaloby niczego. + # Without this assertion the job would pass VACUOUSLY if pandas 3 failed to install + # for any reason, and it would then guard nothing. import sys import pandas print("pandas", pandas.__version__, "| Python", sys.version.split()[0]) major = int(pandas.__version__.split(".")[0]) - assert major >= 3, f"zadanie ma testowac pandas 3, a zainstalowano {pandas.__version__}" - - name: Testy + assert major >= 3, f"the job must test pandas 3, but {pandas.__version__} was installed" + - name: Tests run: python -m pytest -q -rs - notatnik: - name: notatnik Colab, poprawnosc skladni i API + notebook: + name: Colab notebook, syntax and API runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Instalacja + - name: Install run: python -m pip install --upgrade pip && python -m pip install -e ".[test]" - - name: Komorki kodu sa poprawne skladniowo, a uzyte nazwy istnieja w API + - name: Code cells parse, and every name they import exists in the API shell: python run: | - # Nie uruchamiamy notatnika w CI, bo pobiera dokumenty od publikujacego, - # a te sa wymieniane kwartalnie. Sprawdzamy to, co sprawdzic mozna bez - # siegania do sieci: czy kod sie kompiluje i czy kazda nazwa importowana - # z pakietu faktycznie w nim istnieje. + # The notebook is not executed in CI, because it downloads documents from the + # publisher and those are replaced quarterly. What can be checked without the + # network is checked: that the code compiles and that every name imported from + # the package actually exists in it. import ast import json import pathlib @@ -96,18 +96,18 @@ jobs: import gridqueue nb = json.loads(pathlib.Path("notebooks/gridqueue_colab.ipynb").read_text(encoding="utf-8")) - brakujace, komorek = [], 0 + missing, cells = [], 0 for i, c in enumerate(nb["cells"]): if c["cell_type"] != "code": continue src = "".join(c["source"]) src = "\n".join(l for l in src.split("\n") if not l.lstrip().startswith(("%", "!"))) - drzewo = ast.parse(src) # rzuci SyntaxError i zatrzyma zadanie - komorek += 1 - for w in ast.walk(drzewo): + tree = ast.parse(src) # raises SyntaxError and stops the job + cells += 1 + for w in ast.walk(tree): if isinstance(w, ast.ImportFrom) and (w.module or "").startswith("gridqueue"): for n in w.names: if not hasattr(gridqueue, n.name): - brakujace.append((i, n.name)) - assert not brakujace, f"notatnik uzywa nazw nieobecnych w API: {brakujace}" - print(f"komorek kodu: {komorek}; wszystkie nazwy z gridqueue istnieja") + missing.append((i, n.name)) + assert not missing, f"the notebook uses names absent from the API: {missing}" + print(f"code cells: {cells}; every name imported from gridqueue exists") diff --git a/CHANGELOG.md b/CHANGELOG.md index a79dd8b..e8f2ee9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,36 @@ # Change history -Format based on [Keep a Changelog](https://keepachangelog.com/pl/1.1.0/); +Format based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); semantic versioning. The version of the data schema is separate from the version of the package and is reported as `SCHEMA_VERSION`. +## 1.7.0 (2026-10-24) + +### Changed +- **The whole package is in English.** Function arguments, the keys of result objects and the + columns of result tables, local names, comments, docstrings, error messages, the CI workflow, + the Colab notebook and the documentation file names (`schemat.md` is now `schema.md`, and so on). + The full list of renamed public names is in [docs/renamed.md](docs/renamed.md). +- The three renamed top-level names (`PRZEDROSTKI_STACJI`, `normalizuj_nazwe`, + `resolver_from_gazeter_json`) keep working with a `DeprecationWarning` until 2.0. Renamed keyword + arguments and result keys have no aliases. +- Legal citations follow the style of the article: `Art. 7(8l), item 1/3/4` instead of + `pkt 1/3/4`. Numbers in the documentation use English formatting (`1,838`, `98.88%`). +- Chart and report text: percentages without a space before `%`, readable labels for every code. + +### Not changed +- Schema field names, extension columns, controlled vocabularies and geolocation codes are the + data format of the panel and stay as they are; their English meaning is in the new glossary of + `docs/schema.md`. +- Every number reported in the article and in the Supplement is reproduced unchanged; the + expected-value files under `reproducibility/expected/` carry the same values under English keys. + ## 1.6.0 (2026-10-24) ### Added - **Charts** (`gridqueue.plots`, optional extra `pip install "gridqueue[plot]"`): field coverage, quality rules, column-shift audit, stage of the proceeding, requested capacity by class and - voltage, Kaplan-Meier processing-time curves with censoring and a 95 % band, node loading, + voltage, Kaplan-Meier processing-time curves with censoring and a 95% band, node loading, flows / transitions / linkage audit of two editions, location resolution, a four-panel overview, and `save_all`, which draws every chart the data supports and lists the skipped ones with the reason. A chart is a view of a result, never a second computation of it. Figures are @@ -81,22 +102,22 @@ and is reported as `SCHEMA_VERSION`. stated that the columns "with permission" and "in progress" reconstruct the occupied capacity in 395 of 767 rows. That was **a description of an incomplete identity**: both groups are COMPONENTS, not alternative breakdowns. Eleven subcolumns, nine substation items plus those - two, sum to the occupied capacity **in exactly 1 838 of 1 838 rows, in both + two, sum to the occupied capacity **in exactly 1,838 of 1,838 rows, in both editions checked**. The previous figure was true, but it measured a fragment. ### Added -- `suma_skladnikow_mocy_zajetej` and the field `tozsamosc_sumy_mocy_zajetej` in the report of the +- `total_occupied_capacity_components` and the field `identity_sum_occupied_capacity` in the report of the Spanish adapter. For this publisher **there is no reference set** read independently of the parser, so the extraction accuracy cannot be measured against a reference set. Arithmetic closure is evidence of **consistency of reading**: for the sum of eleven cells to agree with the twelfth, all twelve have to be read correctly. Across two - editions that gives 3 676 independent checks, all of them closed. This is + editions that gives 3,676 independent checks, all of them closed. This is evidence weaker than a reference set and it is described as such. -- `NODE_KEY_FIELDS`, a content key for rows of the WĘZEŁ entity. The default key is - built for the WNIOSEK entity and on node rows it **falls apart**, because the application fields - are empty there. Measured on a pair of Spanish issues: with the default key 916 of 1 838 +- `NODE_KEY_FIELDS`, a content key for rows of the NODE entity. The default key is + built for the APPLICATION entity and on node rows it **falls apart**, because the application fields + are empty there. Measured on a pair of Spanish issues: with the default key 916 of 1,838 rows fall into a collision and 660 are rejected as ambiguous; with the node - key the collisions number **zero**, all 1 838 pairs are linked unambiguously, + key the collisions number **zero**, all 1,838 pairs are linked unambiguously, at a precision and recall of **1,000**. The more general conclusion: a content key has to be matched to the entity just as the schema core is, and a mismatch **does not report an error**, it quietly produces collisions that look like events in the register. @@ -106,13 +127,13 @@ and is reported as `SCHEMA_VERSION`. ### Added - The `es_edistribucion` adapter, **the first foreign source issued as a document meant for printing**, which closes the strongest missing test of transferability. A monthly issue, - 28 pages, 1 838 rows of the WĘZEŁ entity, all declared fields populated + 28 pages, 1,838 rows of the NODE entity, all declared fields populated at 100 %, including the geographic coordinates of every substation. Capacities in MW of active power. -- The `ie_esbnetworks` adapter, the largest source in the panel, 46 527 rows, the only one - going below high voltage substations. Coordinates in 46 525 rows. -- Reference panel: **68 054 rows, nine publishers, five jurisdictions** - (PL, GB, PT, ES, IE). The WĘZEŁ entity grows from 634 to 48 999 rows, and the coverage of the - coordinate field from 6,2 % to 72,9 % of the panel. +- The `ie_esbnetworks` adapter, the largest source in the panel, 46,527 rows, the only one + going below high voltage substations. Coordinates in 46,525 rows. +- Reference panel: **68,054 rows, nine publishers, five jurisdictions** + (PL, GB, PT, ES, IE). The NODE entity grows from 634 to 48,999 rows, and the coverage of the + coordinate field from 6.2 % to 72.9 % of the panel. ### Settled - **Units, for the second and third time.** The Irish source mixes, in one file, apparent power @@ -136,13 +157,13 @@ and is reported as `SCHEMA_VERSION`. ### Added - The `pt_eredes` adapter, the reception capability of the national distribution network of Portugal, - 469 rows of the WĘZEŁ entity, published quarterly. **The third jurisdiction in the panel + 469 rows of the NODE entity, published quarterly. **The third jurisdiction in the panel and the third mode of issue**: not a document meant for printing and not a file on a page, but a dataset on an open data portal with a REST interface. -- The first foreign source of the WĘZEŁ entity. That entity grows in the panel from 165 rows +- The first foreign source of the NODE entity. That entity grows in the panel from 165 rows to 634, and both jurisdictions satisfy its core **through different members of the same - alternative**: Poland with available capacity (58 % of the node rows), Portugal with the - constraint flag (100 %). The conditional core introduced in version 1.0 for a Polish + alternative**: Poland with available capacity (58% of the node rows), Portugal with the + constraint flag (100%). The conditional core introduced in version 1.0 for a Polish category error turned out to be the condition for absorbing a foreign jurisdiction. ### Settled @@ -155,7 +176,7 @@ and is reported as `SCHEMA_VERSION`. ### Fixed - **The adapter declaration test carried an out-of-date core contract.** It required - the core of the WNIOSEK entity from every adapter, that is it repeated the category error fixed + the core of the APPLICATION entity from every adapter, that is it repeated the category error fixed in rule R7 in version 1.0: a publisher discharging only point 2 of the duty discloses nodes rather than applications, so it has neither an application identifier nor an application capacity. The defect was revealed only by the Portuguese adapter. The test now checks the core **appropriate @@ -163,7 +184,7 @@ and is reported as `SCHEMA_VERSION`. ### Changed - `reproducibility/manifest.csv` covers the Portuguese export with a checksum. -- The reference panel counts 19 689 rows from seven publishers in three jurisdictions. +- The reference panel counts 19,689 rows from seven publishers in three jurisdictions. ## 1.3.0 (2026-10-07) @@ -176,14 +197,14 @@ and is reported as `SCHEMA_VERSION`. describe as unmeasurable because of the absence of the data. - `linkage_recall`, the **recall** of the longitudinal linkage, alongside the precision measured so far. On a pair of editions of the transmission operator the recall of the key is - 0,9652: of the 29 missed pairs, 10 are broken by the notation of the location, 15 by a change of capacity, + 0.9652: of the 29 missed pairs, 10 are broken by the notation of the location, 15 by a change of capacity, 4 by a change of voltage. -- `surrogate_key(..., zero_jako_brak=True)`, an option equating zero with a missing +- `surrogate_key(..., zero_as_missing=True)`, an option equating zero with a missing value. It recovers 12 of the 51 pairs lost by the key. **Disabled by default**: zero is a disclosed value and a gap is a non-disclosure of it, so equating them is a decision of the analyst and is meant to be visible in the code. -- `schema_table()` returns the columns `encja` and `status_rdzenia`; the core is - conditional per entity, which the two-valued `rdzen` column did not convey. +- `schema_table()` returns the columns `entity` and `core_status`; the core is + conditional per entity, which the two-valued `core` column did not convey. - `reproducibility/fetch_sources.py`, downloads the source documents from the manifest and verifies the SHA-256 checksums. It supports a custom bundle of trusted issuers through `GRIDQUEUE_CA_BUNDLE`, without disabling verification. @@ -204,8 +225,8 @@ and is reported as `SCHEMA_VERSION`. - `reproducibility/manifest.csv` covers the July edition of the disclosure of the transmission operator, used in the longitudinal analysis. - `DATA_LICENSE.md` explains why the documents are not redistributed, - and records that the link of one of the publishers stopped leading to the file -, which the download script recognizes by the signature of the file and reports separately. + and records that the link of one of the publishers stopped leading to the file, + which the download script recognizes by the signature of the file and reports separately. ## 1.2.1 (2026-10-04) @@ -240,7 +261,7 @@ and is reported as `SCHEMA_VERSION`. - `linkage_audit`, an assessment of the precision of the longitudinal linkage on nine fields **excluded from the key** (entity name, object name, application date, six fields of the capacity breakdown). The key does not enforce the agreement of these fields, so their agreement is - independent evidence. On the pair of PSE editions the precision is 98,88-99,63 % + independent evidence. On the pair of PSE editions the precision is 98.88-99.63% across all 804 pairs. - `EditionDelta` now reports key collisions and the number of rows rejected from the linkage as ambiguous. @@ -263,14 +284,14 @@ and is reported as `SCHEMA_VERSION`. - The `longitudinal` module: `compare_editions`, `surrogate_key`, `EditionDelta`. The register does not publish a persistent application identifier, so linkage between editions goes by a content key. -- `examples/longitudinal_use_case.py` and `docs/wzdluzna.md`. +- `examples/longitudinal_use_case.py` and `docs/longitudinal.md`. ### Fixed - **Multiple capacity cells.** When an application concerns several connection points, the publisher enters several values into one cell, separated by a line break. Whitespace normalization turned the line break into a space, and the capacity conversion treats a space as a thousands separator, so "129⏎240" (129 MW + 240 MW) became - 129 240 MW. In the edition with semicolons the same cell did not convert at all. + 129,240 MW. In the edition with semicolons the same cell did not convert at all. The capacity is now split on the line break and on the semicolon, and deliberately not on the space; the components are summed and the split is preserved in the extensions. @@ -279,9 +300,9 @@ and is reported as `SCHEMA_VERSION`. ### Changed - **The schema core is conditional and separate for each entity.** The core completeness rule required the `moc_pobierana` field unconditionally and applied the requirements of the - WNIOSEK entity to rows of the WĘZEŁ entity as well. Both decisions turned a property of the schema - into an apparent violation of the disclosure. Violations of rule R7 number 771 (4,011 %) instead of - 5 390 (28,044 %). + APPLICATION entity to rows of the NODE entity as well. Both decisions turned a property of the schema + into an apparent violation of the disclosure. Violations of rule R7 number 771 (4.011%) instead of + 5,390 (28.044%). - `SCHEMA_VERSION` was introduced, exposed in the validation report and in the public API. ### Added diff --git a/CITATION.cff b/CITATION.cff index 004d51f..56719f5 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -7,7 +7,7 @@ abstract: >- and the corresponding registers from other jurisdictions into a single set of rows in a 21-field schema derived from the structure of the legal obligation. type: software -version: "1.6.0" +version: "1.7.0" date-released: "2026-10-24" license: Apache-2.0 repository-code: "https://github.com/s-matysik/gridqueue" diff --git a/CODE_METADATA.md b/CODE_METADATA.md index f315ff6..cc1d9d8 100644 --- a/CODE_METADATA.md +++ b/CODE_METADATA.md @@ -8,8 +8,8 @@ listed again in the final section. | Nr | Item | Value | |----|------|-------| -| C1 | Current code version | `1.6.0` (declared identically in `pyproject.toml` and in `gridqueue.__version__`; CI asserts the two agree on every push) | -| C2 | Permanent link to code / repository used for this code version | https://github.com/s-matysik/gridqueue (tag `v1.6.0`) | +| C1 | Current code version | `1.7.0` (declared identically in `pyproject.toml` and in `gridqueue.__version__`; CI asserts the two agree on every push) | +| C2 | Permanent link to code / repository used for this code version | https://github.com/s-matysik/gridqueue (tag `v1.7.0`) | | C3 | Permanent link to Reproducible Capsule | ⟦TEAM DECISION⟧, archive DOI. The repository is ready for a Zenodo deposit of tag `v1.6.0`; enabling the integration requires the team's account | | C4 | Legal Code License | Apache-2.0, `LICENSE.txt` at the distribution root. Input data licences are fixed and separate: OpenStreetMap gazetteer under ODbL 1.0; operators' statutory publications under the terms each publisher applies to its own site | | C5 | Code versioning system used | git | @@ -22,8 +22,8 @@ listed again in the final section. | Nr | Item | Value | |----|------|-------| -| S1 | Current software version | `1.6.0`, matching C1 | -| S2 | Permanent link to executables of this version | Source distribution only, `gridqueue-1.6.0.tar.gz`, pure Python, no binaries, no compiled extensions | +| S1 | Current software version | `1.7.0`, matching C1 | +| S2 | Permanent link to executables of this version | Source distribution only, `gridqueue-1.7.0.tar.gz`, pure Python, no binaries, no compiled extensions | | S3 | Legal Software License | Apache-2.0, same licence as C4 | | S4 | Computing platforms / Operating Systems | Linux, macOS, Windows × Python 3.10, 3.11, 3.12, 3.13, a twelve-cell matrix exercised by GitHub Actions (`.github/workflows/ci.yml`) on every push and pull request. The measured figures reported in the paper were produced on macOS/arm64 with Python 3.11.15 | | S5 | Installation requirements & dependencies | As C7. No GPU, no network access required at run time once the source documents and the gazetteer are local | @@ -54,7 +54,7 @@ not. NODE, point 2); mandatory core of 5 fields, 16 optional and declared per publisher; canonical units kW, ISO-8601 dates, WGS84 coordinates; controlled vocabularies of 12 resource classes and 11 process statuses. Verified by measured coverage across the six packaged row-level sources - (`gridqueue_macierz_pokrycia_6publ.csv`): ENERGA-OPERATOR 17/21, PSE 17/21, TAURON 15/21, + (`gridqueue_coverage_matrix_6publ.csv`): ENERGA-OPERATOR 17/21, PSE 17/21, TAURON 15/21, Stoen 13/21, National Grid (GB) 10/21, Boryszew 8/21. **All 21 schema fields are populated somewhere in the 19,220-row panel.** `moc_dostepna` is closed from sources by 96 ENERGA node rows (0-175,000 kW); PSE discharges the same obligation point as a **count of free connection @@ -63,7 +63,7 @@ not. populated by derivation from the place name, not by disclosure: no publisher reports coordinates. Extensions across publishers are complementary, not conflicting. **Resolved in 1.0.** The core no longer requires `moc_pobierana` unconditionally. It requires the - row's OWN entity core: for a WNIOSEK row four unconditional fields plus **at least one** of + row's OWN entity core: for a APPLICATION row four unconditional fields plus **at least one** of (`moc_pobierana`, `moc_wprowadzana`); for a WEZEL row `id_wezla` plus at least one of (`moc_dostepna`, `ograniczenie_flaga`). Rule R7 therefore fell from 5,390 violations (28.044%) to 771 (4.011%), and rows with a complete core rose from 13,830 to 18,449 of 19,220. What remains @@ -81,8 +81,8 @@ not. 1,179 resolved, 98.81%; without the intersection rule 1,095 of 1,179 = 92.88%; without both the inverted-order and the intersection rule 1,041 of 1,179 = 88.30%. Coordinate coverage after the gazetteer was rebuilt with geometry (13,149 street names in 25 - localities, 292 address points, OpenStreetMap under ODbL): **1,225 of 19,220 panel rows = 6.374%** -, Stoen 1,217 of 1,236 = 98.463% (Warsaw 1,167 of 1,179 = 98.982%; outside Warsaw 50 of 57 = + localities, 292 address points, OpenStreetMap under ODbL): **1,225 of 19,220 panel rows = 6.374%**: + Stoen 1,217 of 1,236 = 98.463% (Warsaw 1,167 of 1,179 = 98.982%; outside Warsaw 50 of 57 = 87.719%), Boryszew 8 of 8, and **zero for TAURON, PSE, ENERGA and National Grid**, each with a recorded reason. **No publisher discloses coordinates**; the field is derived from the place name, which is why the manuscript reports it separately from transcribed fields. Verification against @@ -113,7 +113,7 @@ blocker is server-side client filtering, the site returns a rejection page to th control the operator put in place deliberately, so it was not attempted. Electricity North West: rows behind the same access control as UK Power Networks; metadata only. -**Size.** Read from the built source distribution `gridqueue-1.6.0.tar.gz`: 7 core modules under `src/gridqueue`, 2,125 lines, `geoloc.py` 672, `schema.py` 480, `quality.py` 370, `layout.py` 353, `cli.py` 148, `registry.py` 64, `__init__.py` 38, and 9 adapter modules, 1,777 lines (`pl_pse.py` 303, `pl_energa.py` 282, `_ptpiree.py` 278, `pl_boryszew.py` 232, `pl_stoen.py` 194, `uk_nationalgrid.py` 166, `pl_tauron.py` 161, `base.py` 156, `__init__.py` 5); 3,902 lines in total under `src`. `_ptpiree.py` is shared by the PSE and ENERGA adapters because both publishers fill the same agreed industry template, one as a spreadsheet and one as a print document. Tests: 7 files, 1,040 lines (`test_geoloc_wspolrzedne.py` 280, `test_schema.py` 164, `test_pse_energa.py` 147, `test_adapters.py` 144, `test_quality.py` 124, `test_geoloc.py` 95, `test_layout.py` 86). Runnable workflows shipped alongside the package: `examples/research_use_case.py` 168 lines, and `reproducibility/` 382 lines (`run_figures.py` 248, `run_validation.py` 134). `python -m pytest -q`: **153 passed, 2 skipped (155 collected), 0 failed, 0 errors**, the two skips are the PSE and ENERGA integration tests, which require the source documents and are skipped when those are absent from the working tree. The twelve-cell OS × Python matrix in CI runs the same suite. +**Size.** Read from the built source distribution `gridqueue-1.7.0.tar.gz`: 7 core modules under `src/gridqueue`, 2,125 lines, `geoloc.py` 672, `schema.py` 480, `quality.py` 370, `layout.py` 353, `cli.py` 148, `registry.py` 64, `__init__.py` 38, and 9 adapter modules, 1,777 lines (`pl_pse.py` 303, `pl_energa.py` 282, `_ptpiree.py` 278, `pl_boryszew.py` 232, `pl_stoen.py` 194, `uk_nationalgrid.py` 166, `pl_tauron.py` 161, `base.py` 156, `__init__.py` 5); 3,902 lines in total under `src`. `_ptpiree.py` is shared by the PSE and ENERGA adapters because both publishers fill the same agreed industry template, one as a spreadsheet and one as a print document. Tests: 7 files, 1,040 lines (`test_geoloc_coordinates.py` 280, `test_schema.py` 164, `test_pse_energa.py` 147, `test_adapters.py` 144, `test_quality.py` 124, `test_geoloc.py` 95, `test_layout.py` 86). Runnable workflows shipped alongside the package: `examples/research_use_case.py` 168 lines, and `reproducibility/` 382 lines (`run_figures.py` 248, `run_validation.py` 134). `python -m pytest -q`: **153 passed, 2 skipped (155 collected), 0 failed, 0 errors**, the two skips are the PSE and ENERGA integration tests, which require the source documents and are skipped when those are absent from the working tree. The twelve-cell OS × Python matrix in CI runs the same suite. **Functionalities.** Document ingest with per-row provenance; layout-based extraction; unit normalisation to the canonical kW (MW→kW, with the source unit retained in an extension column); resource-class and process-status vocabulary mapping; composite-date diff --git a/README.md b/README.md index 430ae7a..b82d99c 100644 --- a/README.md +++ b/README.md @@ -17,11 +17,15 @@ public, but it is not processable. `gridqueue` turns those publications into one set of rows with **measured accuracy**. -Release 1.6.0: **nine adapters in five jurisdictions** (Poland, Spain, Ireland, Portugal, Great +Release 1.7.0: **nine adapters in five jurisdictions** (Poland, Spain, Ireland, Portugal, Great Britain), all 21 schema fields populated by at least one publisher, quality rules, a longitudinal module with a linkage audit, processing times with censoring, node loading, a location resolver and **charts for every result**. Documentation: **https://s-matysik.github.io/gridqueue/** +The code and documentation are in English. Schema field names and codes (`id_wniosku`, +`WARUNKI_WYDANE`, ...) keep the wording of the Polish statute, because they are the data format of +the panel; their English meaning is in the [glossary](https://s-matysik.github.io/gridqueue/schema.html#glossary-of-field-and-code-names). + ## Three ways to run it | | Python library | Google Colab | In the browser | @@ -75,7 +79,7 @@ resolver is shown on a synthetic gazetteer, with a one-line change to use your o One command, with no compilation and no external services: ```bash -pip install "gridqueue[plot] @ git+https://github.com/s-matysik/gridqueue@v1.6.0" +pip install "gridqueue[plot] @ git+https://github.com/s-matysik/gridqueue@v1.7.0" ``` The `plot` extra adds matplotlib for the charts; without it everything else works. @@ -98,19 +102,19 @@ from gridqueue import get_adapter, run_quality, validate_frame, audit_column_shi res = get_adapter("pl_tauron").parse("tauron_informacja.pdf") df = res.frame # frame in the 21-field schema -print(res.report["n_wierszy"]) # 10955 -print(audit_column_shift(df).udzial) # 0.0, share of rows with shifted columns -print(run_quality(df).as_dict()["naruszen_lacznie"]) +print(res.report["n_rows"]) # 10955 +print(audit_column_shift(df).share) # 0.0, share of rows with shifted columns +print(run_quality(df).as_dict()["violations_total"]) ``` A row with a resolved location: ```python from gridqueue import resolver_from_osm_json -r = resolver_from_osm_json("ulice_warszawa.json") +r = resolver_from_osm_json("streets_warsaw.json") res = r.resolve("WARSZAWA, WINCENTEGO ŚW.") -print(res.matched, res.pewnosc) # Świętego Wincentego REVERSED -print(r.resolve("SKB3").powod) # kod_stacji_bez_slownika, we do not guess +print(res.matched, res.confidence) # Świętego Wincentego REVERSED +print(r.resolve("SKB3").reason) # kod_stacji_bez_slownika, we do not guess ``` ## CLI @@ -120,7 +124,7 @@ gridqueue schema # specification of the 21 fie gridqueue adapters # publisher declarations gridqueue parse tauron.pdf --out tauron.parquet --report tauron.json gridqueue parse --adapter uk_nationalgrid --out uk.parquet # downloads from the CKAN catalogue -gridqueue validate tauron.parquet --out walidacja.json +gridqueue validate tauron.parquet --out validation.json gridqueue export tauron.parquet stoen.parquet --out panel.parquet --only-schema gridqueue compare pse_07.parquet pse_08.parquet --label-a 2026-07-31 --label-b 2026-08-31 --out delta.json gridqueue plot energa.parquet --out charts/ --format png,svg # every chart the data supports @@ -141,7 +145,7 @@ skipped node with the reason), never as zero. | `plot_column_shift` | rows in which a value stands in a field it cannot belong to | | `plot_status_distribution` | applications by stage of the proceeding | | `plot_capacity_by_class` | requested capacity of the active queue by resource class and voltage | -| `plot_processing_time` | Kaplan-Meier curves with censoring and a 95 % band | +| `plot_processing_time` | Kaplan-Meier curves with censoring and a 95% band | | `plot_node_loading` | queue over declared free capacity, per node, log scale | | `plot_edition_flows`, `plot_transitions`, `plot_linkage_audit`, `plot_edition_comparison` | two editions | | `plot_location_resolution` | where a coordinate comes from, and the reason where it does not | @@ -151,7 +155,7 @@ skipped node with the reason), never as zero. ```python from gridqueue import parse_sources, plots -panel, reports = parse_sources(["wnioski.pdf", "moc_odbiorcza.pdf", "moc_wytworcza.pdf"]) +panel, reports = parse_sources(["applications.pdf", "demand_capacity.pdf", "generation_capacity.pdf"]) plots.plot_processing_time(panel, by="klasa_zasobu") # displays in Jupyter / Colab plots.save_all(panel, "charts/", formats=("png", "svg"), dpi=300) ``` @@ -215,16 +219,16 @@ status transition matrix: ```python from gridqueue import compare_editions, linkage_audit, get_adapter -a = get_adapter("pl_pse").parse("edycja_2026_07_31.xlsx").frame -b = get_adapter("pl_pse").parse("edycja_2026_08_31.xlsx").frame +a = get_adapter("pl_pse").parse("edition_2026_07_31.xlsx").frame +b = get_adapter("pl_pse").parse("edition_2026_08_31.xlsx").frame d = compare_editions(a, b, "2026-07-31", "2026-08-31") -d.as_dict()["udzial_przejsc_w_przod"] # 0.9828 -d.as_dict()["migawka_monotoniczna"] # False +d.as_dict()["forward_share"] # 0.9828 +d.as_dict()["monotone_snapshot"] # False linkage_audit(a, b).as_dict() # linkage precision: 0.9888-0.9963 ``` -Two results on the pair of PSE editions, July to August 2026. **98,28 % of the status transitions run +Two results on the pair of PSE editions, July to August 2026. **98.28% of the status transitions run in accordance with the order of the proceeding**, which is an independent test of the key's linkage, because a key matching at random would not have produced that order. And the second: **the register is not a monotonic snapshot**, 13 rows appear already in a closed state, and 33 rows with an active status @@ -234,10 +238,10 @@ queue alone, and the tool reports this with a flag instead of letting the user f Since the register has no persistent identifier, the linkage itself also requires checking. `linkage_audit` compares every pair on nine fields **excluded from the key**; the key does not enforce their agreement, so the agreement is independent evidence. Across all 804 pairs the -precision is **98,88-99,63 %**: 795 pairs confirmed, 6 uncertain, 3 false. +precision is **98.88-99.63%**: 795 pairs confirmed, 6 uncertain, 3 false. ```bash -python examples/longitudinal_use_case.py edycja_A.xlsx edycja_B.xlsx 2026-07-31 2026-08-31 +python examples/longitudinal_use_case.py edition_A.xlsx edition_B.xlsx 2026-07-31 2026-08-31 ``` ## Reproducibility @@ -261,7 +265,7 @@ declares injected capacity, a consuming one consumed capacity, and point 2 of th discharged with available capacity and sometimes with a count of free connection slots. In version 0.3.x the core unconditionally required consumed capacity and applied the requirements of the application entity to node rows as well, which turned a property of the schema into an apparent violation of the -disclosure; violations of that rule numbered 5 390 (28,044 %) against **771 (4,011 %)** now. +disclosure; violations of that rule numbered 5,390 (28.044%) against **771 (4.011%)** now. ## Tests @@ -289,10 +293,10 @@ detected by comparing the output with the reference set read from the document. the codes, so the resolver returns missing with the reason `kod_stacji_bez_slownika`. It does not guess. * **None of the publishers reports coordinates.** The `wspolrzedne` field is populated by - derivation from the place name against a gazetteer with geometry (13 149 names in 25 localities): - 1 225 of 19 220 panel rows. The evidential value of this field is therefore different from that of + derivation from the place name against a gazetteer with geometry (13,149 names in 25 localities): + 1,225 of 19,220 panel rows. The evidential value of this field is therefore different from that of the fields transcribed from the document, and it is described as such in the result. -* A street name does not determine a point: 431 of 6 057 Warsaw names (7,12%) have disjoint components +* A street name does not determine a point: 431 of 6,057 Warsaw names (7.12%) have disjoint components more than 2 km apart. The register gives the street but not the district, so **this ambiguity cannot be removed from the published data**. Every result carries the number of components and their spread, so that the recipient can filter such rows out. diff --git a/docs/adaptery.md b/docs/adapters.md similarity index 91% rename from docs/adaptery.md rename to docs/adapters.md index 2ae6eb7..a8160c8 100644 --- a/docs/adaptery.md +++ b/docs/adapters.md @@ -28,7 +28,7 @@ practice. from gridqueue.adapters.base import Adapter, ParseResult class MojAdapter(Adapter): - klucz = "pl_moj_operator" + key = "pl_moj_operator" publikujacy = "Nazwa Operatora S.A." jurysdykcja = "PL" detekcja = [r"moj_operator", r"nazwa_operatora"] # file name patterns @@ -58,9 +58,9 @@ settles the rest. When neither settles it, `None` is returned, with no guessing. The third jurisdiction in the panel and the third mode of issue: **a dataset on an open data portal with a REST interface**, published quarterly. The unit of a row is a substation or a cut point of the high voltage network; there are 469 rows and all of them -belong to the WĘZEŁ entity. +belong to the NODE entity. -What this adapter contributes that was not there before: **the first foreign source of the WĘZEŁ +What this adapter contributes that was not there before: **the first foreign source of the NODE entity**. Until now the node rows came only from two Polish publishers, so the criterion "available connection capacity" could be computed in one jurisdiction. The Portuguese source additionally reports the **composition of the substation group**, exactly the @@ -88,13 +88,13 @@ This is the same principle that was applied to the publisher reporting a count o free connection slots instead of a capacity: entering a count of slots into a field with a different unit would be fabrication, not harmonization. -The core of the WĘZEŁ entity is nevertheless satisfied, because it is discharged by the second member of +The core of the NODE entity is nevertheless satisfied, because it is discharged by the second member of the alternative, `ograniczenie_flaga`, set on the basis of a zero reception capability. Zero is a **disclosed** value and means a constraint; an empty cell means non-disclosure and does not set the flag. This concerns 222 of 469 nodes. That settlement is at the same time an observational finding: **both jurisdictions satisfy -the core of the WĘZEŁ entity through different members of the same alternative**, Poland with available +the core of the NODE entity through different members of the same alternative**, Poland with available capacity, Portugal with the constraint flag. The conditional core, introduced in version 1.0 for a Polish category error, turned out to be the condition for absorbing a foreign jurisdiction. @@ -102,7 +102,7 @@ a Polish category error, turned out to be the condition for absorbing a foreign from gridqueue import get_adapter r = get_adapter("pt_eredes").parse("pt_eredes_capacidade_rececao_rnd_2T2026.csv") r.frame[["id_wezla", "nazwa_wezla", "lokalizacja_tekst", "ograniczenie_flaga"]] -r.report["uwaga_jednostki"] +r.report["note_units"] ``` ## e-Distribución (Spain), `es_edistribucion` @@ -115,7 +115,7 @@ Article 5(4) of Decree 1183/2020 and Article 12 of Circular 1/2021 of the regula in a legal construction identical to the Polish one: a statutory duty, a fixed frequency, the format left to the publisher. -The issue of 1 August 2025: 28 pages, **1 838 rows of the WĘZEŁ entity**, all declared fields +The issue of 1 August 2025: 28 pages, **1,838 rows of the NODE entity**, all declared fields populated at 100 %, including **the geographic coordinates of every substation**, which no Polish publisher does. The capacities are in **MW of active power**, so conversion to the canonical kW is an ordinary change of order of magnitude. @@ -125,7 +125,7 @@ to the canonical kW is an ordinary change of order of magnitude. The header has cells merged at two levels and suggests that the columns "Con permiso de AyC" and "En trámite con capacidad" belong to the group of **accepted and unsettled** capacity. The arithmetic of the document settles it. Eleven subcolumns, nine substation items plus those two, -sum to the **occupied** capacity, in exactly **1 838 of 1 838 rows, in both editions +sum to the **occupied** capacity, in exactly **1,838 of 1,838 rows, in both editions checked**; the technology breakdown sums to the accepted and unsettled capacity in all rows with a non-zero value. Both groups are therefore **components** rather than alternative breakdowns. Taking the header on trust would have given a silent transposition of two @@ -136,16 +136,16 @@ capacity fields. For this publisher there is **no reference set read independently of the parser**, so the extraction accuracy cannot be measured the way it was measured for the Polish publishers. Arithmetic closure is a substitute piece of evidence: for the sum of eleven cells to agree with the twelfth, all -twelve have to be read correctly. Across two editions that gives **3 676 independent +twelve have to be read correctly. Across two editions that gives **3,676 independent checks, all of them closed**. This is evidence of **consistency of reading, not a measurement of accuracy**, weaker than a reference set, but checkable on every new edition, without an annotator's -work. The adapter report gives it in the field `tozsamosc_sumy_mocy_zajetej`. +work. The adapter report gives it in the field `identity_sum_occupied_capacity`. ## ESB Networks (Ireland), `ie_esbnetworks` -The largest source in the panel: **46 527 rows**, the only one going below the level of high +The largest source in the panel: **46,527 rows**, the only one going below the level of high voltage substations all the way down to MV/LV substations. The unit of a row is a group of -transformers at a substation. Coordinates are present in 46 525 rows. +transformers at a substation. Coordinates are present in 46,525 rows. ### Two units in one file diff --git a/docs/analiza.md b/docs/analysis.md similarity index 85% rename from docs/analiza.md rename to docs/analysis.md index db3e5c8..b0fcb12 100644 --- a/docs/analiza.md +++ b/docs/analysis.md @@ -19,10 +19,10 @@ of the substations belonging to the node. In the panel, one publisher does so. ```python from gridqueue import station_node_map, assign_to_nodes, node_loading -mapa = station_node_map(panel) # station -> group of stations -panel = assign_to_nodes(panel, mapa) # assignment of applications to groups -L = node_loading(panel, pole_mocy="moc_wprowadzana") -L[["nazwa_wezla", "wnioskow", "moc_wnioskowana", "moc_dostepna", "obciazenie"]] +mapping = station_node_map(panel) # station -> group of stations +panel = assign_to_nodes(panel, mapping) # assignment of applications to groups +L = node_loading(panel, capacity_field="moc_wprowadzana") +L[["nazwa_wezla", "applications", "capacity_requested", "moc_dostepna", "loading"]] ``` **The key is the group of stations, not the node row.** The publisher lists the same @@ -64,9 +64,9 @@ time is, that is most strongly exactly where the result matters to us. ```python from gridqueue import processing_time, kaplan_meier -d = processing_time(panel) # czas_dni + zdarzenie (1/0) -km = kaplan_meier(d["czas_dni"], d["zdarzenie"]) -km.as_dict()["mediana"], km.as_dict()["udzial_cenzurowanych"] +d = processing_time(panel) # time_days + event (1/0) +km = kaplan_meier(d["time_days"], d["event"]) +km.as_dict()["median"], km.as_dict()["censored_share"] ``` The Kaplan-Meier estimator was implemented directly, with Greenwood variance, so as @@ -85,13 +85,13 @@ linkage. ```python from gridqueue import linkage_recall -r = linkage_recall(edycja_a, edycja_b) -r.as_dict()["czulosc_klucza"] -r.odzyskane["pola_klucza_rozne"].value_counts() +r = linkage_recall(edition_a, edition_b) +r.as_dict()["key_sensitivity"] +r.recovered_keys["different_key_fields"].value_counts() ``` The identity fields used for recovery have to be **disjoint from the key**. A field shared with the key will not detect a pair broken by a change in exactly that field, because the identity diverges together with the key. Measured on a pair of editions of the transmission operator: adding the location to the set of identity fields raises the apparent -recall from 0,9652 to 0,9757, and in reality hides ten pairs. +recall from 0.9652 to 0,9757, and in reality hides ten pairs. diff --git a/docs/app/gridqueue-1.6.0-py3-none-any.whl b/docs/app/gridqueue-1.6.0-py3-none-any.whl deleted file mode 100644 index c678d41..0000000 Binary files a/docs/app/gridqueue-1.6.0-py3-none-any.whl and /dev/null differ diff --git a/docs/app/gridqueue-1.7.0-py3-none-any.whl b/docs/app/gridqueue-1.7.0-py3-none-any.whl new file mode 100644 index 0000000..3276751 Binary files /dev/null and b/docs/app/gridqueue-1.7.0-py3-none-any.whl differ diff --git a/docs/app/index.html b/docs/app/index.html index fa882cb..6162987 100644 --- a/docs/app/index.html +++ b/docs/app/index.html @@ -372,7 +372,7 @@

Give it documents

aria-label="Drop disclosure files, or click to choose them">
+

Preparing, one moment

PDF or XLSX as published: one disclosure, all documents of one edition, or two editions

@@ -393,7 +393,7 @@

Disclosure documents

aria-label="Drop disclosure files, or click to choose them">
+

Preparing, one moment

PDF, XLSX or CSV, as published, several at once

@@ -505,7 +505,7 @@

Analyses

Time from application to connection conditions, per resource class - (Kaplan-Meier, open applications censored at the as-at date, 95 % Greenwood interval of the median).

+ (Kaplan-Meier, open applications censored at the as-at date, 95% Greenwood interval of the median).

@@ -569,7 +569,7 @@

What runs where

3 The Parquet engine has no WebAssembly build; use the CSV or XLSX export, or the notebook.

For everything at once, open the Colab notebook, - or install the package with pip install "gridqueue[plot] @ git+https://github.com/s-matysik/gridqueue@v1.6.0".

+ or install the package with pip install "gridqueue[plot] @ git+https://github.com/s-matysik/gridqueue@v1.7.0".