diff --git a/.env b/.env index 999fa0f..a9d7b27 100644 --- a/.env +++ b/.env @@ -32,4 +32,18 @@ DEFAULT_URI=http://localhost ###> app ### APP_BROKER_BASE_URI=http://scorpio.local:9090/ + +# JSON-LD contexts attached to every entity, outermost last so the ETSI core +# context resolves the NGSI-LD terms and the domain context resolves the +# Smart Data Models ones. +ENTER_NGSI_CONTEXT_URLS='https://raw.githubusercontent.com/smart-data-models/dataModel.Parking/master/context.jsonld,https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld' + +# A single domain context, for the Link header that read requests need. Only +# used by `task broker:entities`, and it must be the context defining the type +# being read — Parking while that is the only model published. +ENTER_NGSI_DOMAIN_CONTEXT=https://raw.githubusercontent.com/smart-data-models/dataModel.Parking/master/context.jsonld + +# Where each feed is read from is not configured here. It belongs to the data +# set rather than to the environment, and lives in config/sources.yaml — see +# docs/adr/007-source-manifest.md. ###< app ### diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..e39b258 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,19 @@ +- + ddev exec sh -c + "APP_BROKER_BASE_URI='$APP_BROKER_BASE_URI' + ENTER_NGSI_DOMAIN_CONTEXT='$ENTER_NGSI_DOMAIN_CONTEXT' + sh task/scripts/broker-entities {{.CLI_ARGS}}" + coding-standards:apply: desc: 'Apply coding standards' cmds: diff --git a/config/sources.yaml b/config/sources.yaml new file mode 100644 index 0000000..8081a03 --- /dev/null +++ b/config/sources.yaml @@ -0,0 +1,46 @@ +# The data sets this application publishes, one entry per source key. +# +# The facts the import needs — where the feed is read from, the coordinate +# reference system it publishes, the Smart Data Model it is published as — are +# read from here, so they exist in exactly one place. +# +# The remaining fields record what no code can state: who owns the data, on +# what terms, how often it changes, and which source fields are deliberately +# not published. The rule is to record only what the code cannot tell you, so +# the field mapping itself stays in the source class, which is the only place +# that knows the feed's shape. +# +# Field names follow DCAT-AP, the metadata profile European data portals +# harvest, so registering a data set is a translation of its entry rather than +# a new survey. See docs/adr/007-source-manifest.md. + +sources: + mtm_spatialmaps-handicap-parking: + title: 'Handicapparkering, Aarhus Kommune' + description: >- + Disabled parking bays in Aarhus Municipality, with the number of + reserved bays per location. + publisher: 'Aarhus Kommune' + contact: ppg@aarhus.dk + landing_page: 'https://www.opendata.dk/city-of-aarhus/parkering-i-aarhus-kommune' + access_url: 'https://webkort.aarhuskommune.dk/spatialmap?page=get_geojson_opendata&datasource=invap' + media_type: application/geo+json + crs: 'EPSG:25832' + model: OnStreetParking + update_frequency: continuous + + # The portal states no licence for this data set. DCAT-AP requires + # one, so it has to be settled with the data owner before the + # catalogue can be registered anywhere. + licence: ~ + + # Fields the feed carries that are not published. Recorded here + # because the source class shows what is mapped but cannot show what + # was left out, or why. + omitted_fields: + ident: 'Single-letter code; its meaning is not documented and not confirmed by the data owner.' + oprettet_af: 'Directory username of the municipal employee who created the record — personal data.' + rettet_af: 'Directory username of the municipal employee who last edited the record — personal data.' + oprettet_dato: 'Describes the register record rather than the parking bay.' + rettet_dato: 'Describes the register record rather than the parking bay. A candidate for observedAt, not yet mapped.' + mi_style: 'MapInfo rendering style, empty throughout the export.' diff --git a/docs/adr/001-architecture-symfony-docker.md b/docs/adr/001-architecture-symfony-docker.md new file mode 100644 index 0000000..c0f8708 --- /dev/null +++ b/docs/adr/001-architecture-symfony-docker.md @@ -0,0 +1,87 @@ +# 001: Architecture — Symfony 8 on the ITK Dev Docker template + +| Field | Value | +|--------------------|----------------------------------------| +| **Created By** | Jeppe Krogh | +| **Date** | 2026-08-24 | +| **Decision Maker** | ITK Dev team | +| **Stakeholders** | ITK Dev developers, future maintainers | +| **Status** | Draft | + +## Context + +The adapter reads open data sets, converts them to a standard smart-city +representation, and publishes them to a context broker. It needs a runtime, an +HTTP client, a console for running imports, and a local development environment +including a broker to import into. It has no web UI and no domain data of its +own. + +ITK Dev maintains a fleet of PHP services with an established Docker-based +development convention, expressed as versioned project templates with shared CI +and coding-standards configuration. A new application either adopts that or +diverges from it. + +This ADR serves to decide the runtime, framework and development +environment the application is built on. + +### Drivers + +- **Functional:** scheduled console commands; outbound HTTP; a local broker. + No database and no HTTP surface of its own. +- **Non-functional:** minimal onboarding cost; shared tooling rather than + reimplemented tooling; reproducible across developers and CI; long-term + vendor support. + +### Options Considered + +1. **PHP 8.4 / Symfony 8 on the ITK Dev `symfony-8` template.** Matches the + organisation's existing stack, so CI, coding standards and task runner come + for free; the console component suits scheduled imports. Provisions a web + server, database and mail catcher this application never uses, and its PHP + version runs ahead of developer hosts, making containers mandatory. +2. **Minimal framework project without the template, run on the host.** No + unused services, no container requirement for the application — but shared + CI and coding-standards config would be reimplemented by hand, and a local + broker needs containers anyway, so the dependency is moved rather than + removed. +3. **A second entry point in an existing internal application.** One + deployment to operate, but couples a batch importer's release cycle to a + user-facing application and inherits dependencies it has no use for. +4. **A different language ecosystem on a bespoke setup.** Richer geospatial + libraries in some ecosystems, but no internal expertise and no shared + tooling. The transformations needed are available as mature libraries in the + established stack too. + +## Decision + +**PHP 8.4 + Symfony 8** on the ITK Dev `symfony-8` template, as its **own +deployable service**, with a containerised broker overlay for local development. + +- Standardising costs less over the application's lifetime than trimming unused + services. A second toolchain must be learned and patched; idle containers + cost only disk. +- A batch importer and a user-facing application have different lifecycles and + failure modes, so they stay separate services. +- No domain persistence is needed — the broker is the system of record — so the + template's database service is left unused rather than removed, keeping + template updates a clean diff. +- Local development includes a real broker, so imports are verified end to end + rather than only as serialised output. + +## Consequences + +### Positive + +- Onboarding cost close to zero; CI and coding standards work from the first + commit. +- No schema, no migrations, no state to keep consistent with the broker. + +### Negative / Trade-offs + +- **Containers are mandatory.** The template's PHP runs ahead of developer + hosts, so dependency management, console commands and tests cannot run + natively. Most likely source of first-run confusion. +- A web server, database and mail catcher are provisioned and never used. +- Broker images are not published for every CPU architecture, so local start-up + may be slow under emulation. +- The application follows the template's choices; deviating later has a cost. diff --git a/docs/adr/002-publish-to-a-context-broker.md b/docs/adr/002-publish-to-a-context-broker.md new file mode 100644 index 0000000..7372479 --- /dev/null +++ b/docs/adr/002-publish-to-a-context-broker.md @@ -0,0 +1,104 @@ +# 002: Publication mechanism — publish to a context broker + +| Field | Value | +|--------------------|--------------------------------------------------------| +| **Created By** | Jeppe Krogh | +| **Date** | 2026-08-24 | +| **Decision Maker** | ITK Dev team | +| **Stakeholders** | ITK Dev developers, data consumers, future maintainers | +| **Status** | Draft | + +## Context + +The adapter makes data available to consumers the organisation does +not control and cannot brief. The data already exists in operational systems, +with heterogeneous formats, coordinate systems and access methods. What has to +be decided is the mechanism by which it is published. The source systems remain +authoritative; the published copy is not a system of record. + +This ADR serves to decide the mechanism by which data is published to +consumers. + +### Drivers + +- **Functional:** query by location and attribute, not bulk download only; + several data sets reaching one consumer-facing surface; change notification; + adding a data set without changing consumer integrations. +- **Non-functional:** interpretable by consumers we have never spoken to; + operational cost proportionate to the data and the number of consumers; + existing client tooling rather than clients we supply; an interface that + outlives individual data sets. + +### Options Considered + +1. **An NGSI-LD context broker.** Provides geospatial and attribute queries, + pagination, subscriptions and a temporal interface without implementing + them; payloads carry a vocabulary reference, so they are self-describing; + many producers converge on one consumer surface. Substantial operational + weight — typically a database and message bus alongside the broker — and the + strictness of a particular implementation is inherited. +2. **Static file export** on a web server or object store. Near-zero + operational cost, trivially cacheable, readable by anything. No query, so + consumers download everything and filter client-side; no change + notification; conventions must be documented in prose because nothing in the + file declares its own meaning. +3. **A bespoke REST API** over our own datastore. Exact fit, full control of + the query surface and semantics. Every capability is ours to build and + maintain — geo-queries, filtering, pagination, notifications, documentation, + clients, versioning — and consumers must learn an interface that exists + nowhere else. +4. **Direct database access or a read replica.** No API layer, powerful ad-hoc + querying. Exposes internal schema as a public contract, requires per-consumer + credentials and network access, and is unusable by browser-based consumers. + +## Decision + +Publish to an **NGSI-LD context broker**. + +- Consumers of geographic data need "everything within this area" and + "everything of this kind" more often than the whole data set. A broker + provides that as a standard interface rather than a per-data-set feature. +- Publishing structure without a vocabulary reference requires every consumer + to have our documentation. A broker payload carries the reference. +- For a single small data set a static export would be cheaper and better. Once + several heterogeneous data sets must be published, the fixed operational cost + is paid once while the per-data-set cost approaches zero, and consumers + integrate once rather than once per source. +- New consumers require no change to the adapter, and new data sets require no + change to consumers. +- Existing viewers, dashboards and connectors speak this interface; a bespoke + API would mean supplying clients indefinitely. + +The broker's value is interoperability and query, not storage. If no consumer +reads the data through its interface, a static export would have been the better +decision. Revisit once data sets have been published long enough for consumers +to appear. + +## Consequences + +### Positive + +- Geospatial and attribute queries, pagination, subscriptions and a temporal + interface, none of which are implemented here. +- Payloads reference a shared vocabulary, so they need no bespoke + documentation. +- Additional data sets reach every existing consumer with no integration work. +- The adapter has no database and no read surface of its own. + +### Negative / Trade-offs + +- **Operational weight out of proportion to a small data set.** A broker + deployment is several services to run, patch, monitor and back up. For a + static data set a file on a web server would serve the same need. +- **Broker implementations impose constraints beyond the standard.** Those + encountered include accepting only one spelling of a UTC timestamp while + rejecting an equivalent one, requiring the vocabulary reference on read + requests — with omission returning an empty success rather than an error — + and collapsing single-element lists to scalars. +- **Vocabulary documents may be fetched over the network during writes**, so + third-party availability becomes part of the import path. +- **No delete semantics.** Upsert creates and updates but never removes, so + records that disappear upstream persist until reconciliation is built. +- JSON-LD is a learning curve for maintainers and consumers. +- Fitting data to a shared vocabulary costs effort that publishing as-is + would not. diff --git a/docs/adr/003-ngsi-ld-representation.md b/docs/adr/003-ngsi-ld-representation.md new file mode 100644 index 0000000..8adcb7e --- /dev/null +++ b/docs/adr/003-ngsi-ld-representation.md @@ -0,0 +1,70 @@ +# 003: NGSI-LD representation — normalized form and batch upsert + +| Field | Value | +|--------------------|--------------------------------------------------------| +| **Created By** | Jeppe Krogh | +| **Date** | 2026-08-24 | +| **Decision Maker** | ITK Dev team | +| **Stakeholders** | ITK Dev developers, data consumers, future maintainers | +| **Status** | Draft | + +## Context + +ADR 002 chooses an NGSI-LD context broker. That leaves two representation +choices open: how attributes are shaped, and which write operation is used. + +NGSI v2, the older FIWARE API generation, was not considered viable: it has no +`@context`, so a shared vocabulary cannot be expressed, and the +organisation operates no v2 broker. + +This ADR serves to decide how attributes are shaped and which write +operation is used. + +### Drivers + +- **Functional:** the broker must accept the payload on write; attribute-level + metadata must be expressible; repeated imports must not duplicate entities. +- **Non-functional:** idempotency; payload size; readability for consumers. + +### Options Considered + +#### Attribute form + +1. **Normalized** — every attribute an object naming its kind. Verbose, but + the form brokers accept on write and the only one carrying metadata such as + `observedAt`. +2. **Key-values** — flat `name: value`. Much smaller and easier to read, but + read-only and cannot carry attribute metadata. + +#### Write operation + +1. **Batch upsert** — creates or updates. Idempotent when identifiers are + derived from source keys. +2. **Create** — fails with `409` for identifiers that already exist, so a + re-import errors rather than refreshing. +3. **Batch replace** — silently drops attributes absent from the payload, + making partial payloads destructive. + +## Decision + +Publish **normalized** NGSI-LD with `Content-Type: application/ld+json`, via +**batch upsert**. + +- Normalized is the only form accepted on write, so key-values is not a real + option for a producer. +- Upsert makes imports idempotent: identifiers derive from each source's + primary key, so re-running updates in place. + +## Consequences + +### Positive + +- Re-imports produce no duplicates and need no prior state. +- Attribute metadata remains available if a source ever supplies it. + +### Negative / Trade-offs + +- Payloads are considerably larger than key-values. +- Upsert never deletes, so records removed upstream persist until + reconciliation is built (see ADR 002). +- Consumers unfamiliar with JSON-LD face a learning curve. diff --git a/docs/adr/004-coordinate-reference-system.md b/docs/adr/004-coordinate-reference-system.md new file mode 100644 index 0000000..177d6b7 --- /dev/null +++ b/docs/adr/004-coordinate-reference-system.md @@ -0,0 +1,112 @@ +# 004: Coordinate reference system — publish WGS84 (EPSG:4326) + +| Field | Value | +|--------------------|----------------------------------------------------------| +| **Created By** | Jeppe Krogh | +| **Date** | 2026-08-27 | +| **Decision Maker** | ITK Dev team | +| **Stakeholders** | ITK Dev developers, broker consumers, future maintainers | +| **Status** | Draft | + +## Context + +Every entity the adapter publishes carries a `location` GeoProperty, so the +coordinate reference system is a cross-cutting concern rather than a per-source +detail. + +Input data arrives in whatever CRS its publisher uses. Danish municipal data is +commonly projected — typically EPSG:25832 (ETRS89 / UTM zone 32N), as eastings +and northings in metres. Other inputs may already be geographic, or use a +different projection. The adapter cannot assume one input CRS. + +Projected coordinates are sometimes delivered inside a GeoJSON envelope, which +states a geometry type but not units. The data models specify only that +`location` is GeoJSON and make no reference to a coordinate system; the +constraint comes from GeoJSON itself. + +This ADR serves to decide which coordinate reference system is published, +and at what precision. + +### Drivers + +- **Functional:** consumers must be able to interpret `location` unbriefed; + geo-queries must return correct results; clients must render without + preprocessing; one rule must hold for every input. +- **Non-functional:** self-description; conformance; uniformity across inputs; + precision no worse than the input. + +### Options Considered + +1. **Normalise everything to WGS84, reprojecting in the adapter.** Conforms to + RFC 7946; self-describing; geo-queries work; one rule however many input + CRSs accumulate. Requires a reprojection dependency, and each input must + declare its CRS. +2. **Pass each input's native CRS through unchanged.** No transformation, no + dependency — but produces invalid GeoJSON with nowhere to declare the CRS, + makes entities from different inputs mutually incomparable, and breaks + geo-queries because distances are read as degrees. Every failure is silent. +3. **Publish WGS84 and also retain original coordinates in an extra + attribute.** Avoids a round trip for consumers wanting native coordinates, + but the attribute cannot have a stable shape: each input brings its own CRS + and geometry type, and it would be absent for inputs already in WGS84. A + consumer cannot code against that, so it would go unused. +4. **Pass native CRSs through under RFC 7946's "prior arrangement" clause, + documenting each out of band.** Permitted by the RFC, but the clause + requires all parties to have agreed — incompatible with a broker whose + consumers are unknown by design. + +## Decision + +Every `location` the adapter emits is **EPSG:4326 (WGS84) +longitude/latitude**, at **full precision — coordinates are not rounded**. Each +input declares its own CRS; reprojection happens at the boundary between reading +an input and building an entity, and nowhere else. + +- **The specification is unambiguous.** RFC 7946 §4: "The coordinate reference + system for all GeoJSON coordinates is a geographic coordinate reference + system, using the World Geodetic System 1984 (WGS 84) datum, with longitude + and latitude units of decimal degrees." NGSI-LD GeoProperty values are + GeoJSON, so the requirement is inherited. +- **There is no way to declare otherwise.** RFC 7946 Appendix B.1: + "Specification of coordinate reference systems has been removed, i.e., the + 'crs' member of [GJ2008] is no longer used." Publishing projected coordinates + means publishing an undeclarable assumption. Inputs may still carry that + deprecated member; it can be read, but not passed on. +- Entities from different inputs are queried together, so a query spanning two + inputs published in different CRSs returns meaningless results. +- A broker given projected coordinates accepts them, answers geo-queries + incorrectly, and renders points in the wrong location. No error is raised at + any stage. +- **The conversion is lossless at full precision.** A projected-to-geographic + round trip returns the input exactly when no rounding is applied. Rounding + trades accuracy for a marginal reduction in payload size. +- `source` and `seeAlso` can reference the originating export, which states its + own CRS. A coordinate copied into an extra attribute states nothing. + +## Consequences + +### Positive + +- Payloads are valid GeoJSON and NGSI-LD; geo-queries work and are comparable + across inputs; any client renders them unmodified. +- One rule for every present and future input. +- Reprojection is isolated in one component with its own tests, verified + against independently known reference coordinates, so a regression fails + loudly instead of silently relocating data. + +### Negative / Trade-offs + +- Adds a reprojection dependency. National grid definitions are not always + shipped and may need registering explicitly, making them load-bearing + project code. +- **Datum shifts are approximated.** ETRS89-based grids are treated as + equivalent to WGS84 via a null datum transformation. The two were coincident + in 1989 and have diverged by roughly 0.5–1 m since, at about 2.5 cm per year. + What is published is therefore ETRS89 labelled WGS84. This is standard + practice in web GIS, but it is the largest error in the pipeline — greater + than the source's own positional accuracy — so a consumer using a rigorous + transformation with an explicit epoch will land about a metre away. +- Consumers with natively projected stacks must convert. +- Every new input must declare its CRS, and unsupported ones need adding. +- Only point geometries were implemented initially; line and area geometries + were added later. diff --git a/docs/adr/005-smart-data-models-as-vocabulary.md b/docs/adr/005-smart-data-models-as-vocabulary.md new file mode 100644 index 0000000..42f11ec --- /dev/null +++ b/docs/adr/005-smart-data-models-as-vocabulary.md @@ -0,0 +1,126 @@ +# 005: Vocabulary — adopt Smart Data Models + +| Field | Value | +|--------------------|--------------------------------------------------------| +| **Created By** | Jeppe Krogh | +| **Date** | 2026-08-31 | +| **Decision Maker** | ITK Dev team | +| **Stakeholders** | ITK Dev developers, data consumers, future maintainers | +| **Status** | Draft | + +## Context + +NGSI-LD defines how attributes are carried and how to reference a vocabulary. +It does not define entity types or attribute names. Without a vocabulary the +JSON-LD context resolves to nothing, entity types are local strings, and +consumers still need our documentation to interpret anything. + +The choice also determines the cost of onboarding each data set, since mapping +a source onto an existing model takes more effort than exposing its fields +verbatim. + +This ADR serves to decide which vocabulary supplies entity types and +attribute names, and the rules for mapping sources onto it. + +### Drivers + +- **Functional:** types and attributes interpretable without our + documentation; expressible as a JSON-LD context a broker can resolve; + coverage across the domains in scope. +- **Non-functional:** a vocabulary consumers plausibly already know; governed + and maintained by someone else; mapping cost that does not dominate + onboarding. + +### Options Considered + +1. **Smart Data Models.** Purpose-built for NGSI-LD and the reference + vocabulary of that ecosystem; publishes JSON-LD context documents per + domain; broad coverage; each model ships a JSON schema and examples, giving + an objective conformance target; open governance. Model depth varies; + many models assume real-time sensing, so static inventory leaves attributes + unset; required attributes occasionally presuppose a hierarchy the source + lacks; enum spellings sometimes disagree between a model's schema and its + examples; versioning is loose. +2. **A vocabulary of our own, with self-hosted context documents.** Exact fit, + no required attributes we cannot satisfy, full control of naming and + versioning. Nobody else speaks it, so consumers return to reading our + documentation; governance, documentation and versioning become ours + indefinitely; no existing tooling recognises the types. +3. **A general-purpose web vocabulary.** Widely recognised, stable governance, + adequate for names, addresses and descriptions. No NGSI-LD conventions for + geometry or relationships, and no domain-specific terms, so the domains in + scope would remain unmodelled. + +## Decision + +Adopt **Smart Data Models**, referencing the relevant domain context documents +alongside the NGSI-LD core context. + +Two rules follow, and they matter more than the choice itself: + +1. **Use an existing model; do not invent a type.** An imperfect standard type + is more useful to a consumer than a perfect private one. +2. **Never fabricate a value to satisfy a model.** An absent attribute is + honest; a fabricated one is indistinguishable from a measured one. Applied: + + - Attributes the source cannot fill are left unset, not approximated, + defaulted or inferred. + - Where a model *requires* an attribute the source cannot supply, choose a + different model rather than inventing the value. Required relationships + are the common case: inventing a related entity yields something + schema-valid and factually wrong that must then be maintained + indefinitely. A sibling model without the requirement is the better + choice even if its terms are less precise. + - In a hierarchy — a root site, subdivisions beneath it, individual units + beneath those, each lower level requiring a relationship upward — publish + at the highest level the source can populate. Static inventory typically + describes a location and a count of units without describing what the + location is part of, so the site level is usually correct. + +The concrete model chosen for a given data set is recorded in its own ADR; this +one states policy only. + +Rationale: + +- The context must resolve to terms a consumer recognises, or publishing gains + nothing over a file. +- Smart Data Models is the vocabulary the surrounding ecosystem uses and ships + the context documents needed to reference it, so adoption is a URL rather + than a project. +- Shipped schemas and examples make modelling disagreements checkable against a + specification instead of settled by preference. +- Mandatory relationships propagate downward: choosing a subdivision level + schedules the need for a parent rather than avoiding it, because the + individual-unit level requires a site as well. +- The costs are asymmetric. Publishing at site level and later finding real + sites exist means a one-off migration. Publishing at subdivision level and + never acquiring real sites means maintaining an invented entity + indefinitely, with every consumer that follows the relationship receiving + something meaningless. + +## Consequences + +### Positive + +- Types and attributes resolve to shared global identifiers. +- Consumers may already have code for the types published. +- Modelling decisions have an external reference point. +- Later data sets are likely already covered, so onboarding does not start with + vocabulary design. +- Published entities are self-contained, with nothing invented to keep in sync. +- Finer granularity can be added later beneath what is already published. + +### Negative / Trade-offs + +- **Many attributes will always be empty.** Models built around real-time + sensing carry availability, occupancy and detection attributes that static + inventory cannot fill. +- **Model choice is embedded in entity identifiers.** Changing model later + means deleting and re-publishing rather than updating in place, so selection + deserves attention before a data set is first published. +- **Enum values must be read from the schema, not the examples.** Where the two + disagree the schema is authoritative, and equivalent-looking values differ + between sibling models, so they must not be copied across. +- **Loose versioning.** A model can change without an obvious signal. +- Mapping a source to a model takes longer than exposing its fields verbatim, + and occasionally the fit is poor. diff --git a/docs/adr/006-onstreetparking-over-parkinggroup.md b/docs/adr/006-onstreetparking-over-parkinggroup.md new file mode 100644 index 0000000..8b04272 --- /dev/null +++ b/docs/adr/006-onstreetparking-over-parkinggroup.md @@ -0,0 +1,104 @@ +# 006: Model selection — OnStreetParking over ParkingGroup + +| Field | Value | +|--------------------|--------------------------------------------------------| +| **Created By** | Jeppe Krogh | +| **Date** | 2026-08-31 | +| **Decision Maker** | ITK Dev team | +| **Stakeholders** | ITK Dev developers, data consumers, future maintainers | +| **Status** | Draft | + +## Context + +The parking domain is organised as a hierarchy: + +```text +OnStreetParking / OffStreetParking site — no parent, requires id, type, location + └── ParkingGroup subdivision — requires refParkingSite + └── ParkingSpot individual unit — requires refParkingSite, status, category +``` + +This decision applies where a data set provides a count of units per location +with a point geometry, no reference to a containing site, and neither per-unit +geometry nor occupancy. + +Entity identifiers embed the type by convention, so the choice must be made +before first publication: changing it afterwards means deleting and +re-publishing. + +This ADR serves to decide which model in the parking hierarchy is published +under those conditions. + +### Drivers + +- **Functional:** every mandatory relationship must point at an entity that + exists; the restriction on who may park must be expressible unambiguously; + finer granularity addable later without restructuring what is published. +- **Non-functional:** nothing invented purely to satisfy a schema; a choice + that is cheap to reverse in preference to one that is not. + +### Options Considered + +1. **`ParkingGroup`, creating the missing parent site.** `category` offers + `onlyDisabled`, which by name states exclusivity, and the model's reference + example for disabled parking sits at this level. But `refParkingSite` is + mandatory and no value is available for it, so a parent must be invented; + one spanning the whole administrative area asserts a false containment, and + its own mandatory geometry would carry no meaning. + `ParkingSpot` also requires a site, so adding per-unit data later would force + the invented entity into existence after entities had been published against + it. +2. **`ParkingGroup`, omitting `refParkingSite`.** Nothing invented, smallest + change — but knowingly non-conformant, and a schema validator flags every + entity. +3. **`OnStreetParking`.** Requires only `id`, `type` and `location`, all of + which are available. It is the entity both `ParkingGroup` and + `ParkingSpot` are required to reference, so finer granularity can be + attached beneath it, and migration down to `ParkingGroup` stays possible if + real site data appears. `category` offers only `forDisabled`, which does not + state exclusivity as plainly. +4. **`ParkingSpot`.** Models an individual unit, but only a count per location + is available, `status` is mandatory with no occupancy data, and a parent site + is required. Rejected outright. + +## Decision + +Publish each record as an **`OnStreetParking`** entity, with +`category: ["forDisabled"]` and no `refParkingSite`. + +- It is the only option that invents nothing; everything the model requires is + available. +- Mandatory relationships propagate downward, so choosing `ParkingGroup` would + schedule the invented parent rather than avoid it — `ParkingSpot` requires a + site too. +- The costs are asymmetric. Site level, then finding real sites exist, is a + one-off migration. Subdivision level, then never acquiring real sites, means + maintaining an invented entity indefinitely. + +The model's reference example does use `ParkingGroup` for disabled parking, but +points at a real street-address site. It shows what to do when a site exists, +not when none does. + +## Consequences + +### Positive + +- No dangling relationship; every entity is self-contained. +- Nothing invented to create, document or keep in sync. +- Conformant to the model's schema without exceptions. +- `ParkingGroup` or `ParkingSpot` entities can be attached beneath these later + without changing them. + +### Negative / Trade-offs + +- **`forDisabled` does not state exclusivity.** The schema documents `category` + only as "Street parking category" with an enum list and defines no individual + value. The two models' enums use inconsistent prefixes for what appear to be + the same concepts — `forDisabled` / `forResidents` against `onlyDisabled` / + `onlyResidents` — while both carry `onlyWithPermit`, so the spelling cannot + be relied on to carry exclusivity. +- Values must not be copied between the two models' `category` enums. +- Entity type is embedded in identifiers, so any later change means deleting + and re-publishing. +- Domain experts may find a single address described as a "site" + counter-intuitive. diff --git a/docs/adr/007-source-manifest.md b/docs/adr/007-source-manifest.md new file mode 100644 index 0000000..241eb8f --- /dev/null +++ b/docs/adr/007-source-manifest.md @@ -0,0 +1,122 @@ +# 007: Data set metadata — a committed source manifest + +| Field | Value | +|--------------------|--------------------------------------------------------| +| **Created By** | Jeppe Krogh | +| **Date** | 2026-09-02 | +| **Decision Maker** | ITK Dev team | +| **Stakeholders** | ITK Dev developers, data consumers, future maintainers | +| **Status** | Draft | + +## Context + +Each published data set carries facts the import needs — where the feed is read +from, the coordinate reference system its coordinates are in, and the model it +is published as — and facts only people need: who owns the data, on what terms +it may be republished, how often it changes, and which of its fields are +deliberately not published, with the reason for each. + +The first group must be readable by code. The second is what a public data +portal requires at registration, and what a data owner asks for when +establishing what happened to their data. Both grow with the number of data +sets. + +Recording the two groups separately produces a machine-readable value and a +written description of the same value, which can then disagree. Recording only +the first leaves the rest unwritten, and the questions it answers are then +answered from memory. + +This ADR serves to decide where a data set's own facts are recorded, and which +of them the code reads. + +### Drivers + +- **Functional:** the code reads the facts it needs from the same record a + person reads; a data set is registerable on a public portal without a fresh + survey; an incomplete record fails the import that needs it rather than + producing an incomplete publication. +- **Non-functional:** adding a data set requires no deployment change; the + record is reviewable as a diff; no fact exists in two places. + +### Options Considered + +1. **One environment variable per data set.** Follows the convention that + configuration belongs in the environment, and lets a value differ per + environment. But variable names grow with the catalogue, so each new data + set becomes a deployment change; the environment carries strings only, + leaving metadata beyond an address nowhere to live; and values are invisible + in review, so a wrong one is found by running the import. +2. **Every fact in the class that maps the data set.** Nothing can diverge, + there being one copy, and the language enforces its presence. But metadata + is then readable only by opening code, extracting it for portal registration + requires writing an extractor, and correcting a licence or a contact becomes + a code change reviewed as one. +3. **A committed manifest the code reads.** One record per data set, keyed by + the identifier the import selects it with, holding the facts the code needs + beside those it does not, in a shape a catalogue profile can be generated + from. Validating the record becomes work of our own, and the values are + identical in every environment. +4. **An external catalogue or registry service.** The eventual home of + published metadata, with search and harvesting already built. But it has to + be running for an import to work, it is a second system to operate, and it + must be populated before anything can be published from it — from records + that would have to live somewhere else in the meantime. + +## Decision + +Record each data set in a **committed manifest**, keyed by the identifier the +import selects it with, and read from it every fact the code needs. + +Four rules follow: + +1. **Record only what the code cannot state.** How a feed's fields map onto the + model, and every quirk of its shape, stay in the class that performs the + mapping. Restating them in the manifest recreates the divergence the + manifest exists to prevent. +2. **A fact both the code and a reader need is read from the manifest.** It is + not also written in code, in a comment, or in the README. +3. **An incomplete or malformed record is an error.** Fields an import cannot + run without are required, and their absence raises rather than defaulting. A + fact that is unknown is recorded as unknown, so the gap stays visible. +4. **Name the fields after the catalogue profile the data will be registered + under** — DCAT-AP — so that publication is a translation rather than a + redesign. + +Rationale: + +- What has to be prevented is a value diverging from its description, so the + two belong to the same record. +- Portal registration and answering a data owner need the same fields, and + neither can be derived from mapping code. +- A record is reviewed alongside the class that consumes it, so a reviewer sees + the address, the reference system and the model together with the mapping + that assumes them. +- Where a feed is read from is a fact about the data set, not about the machine + running the import, so the environment is the wrong place for it. + +## Consequences + +### Positive + +- One record per data set, reviewable as a diff and versioned with the code. +- Adding a data set is one class and one record, with no deployment change. +- Registration on a public portal is a translation of records that exist. +- Licence, ownership and what was withheld have a single answer, and an + unanswered question shows as an empty value rather than as nothing at all. +- The same records can generate catalogue entities later without introducing a + second source of truth. + +### Negative / Trade-offs + +- **Values are identical in every environment.** Pointing a data set at a copy + for testing means editing a committed file. That is consistent with reading + feeds where they live, but it removes an escape hatch the environment offered. +- **A wrong reference system or model in the manifest is as damaging as a wrong + one in code, while looking less like code.** A wrong reference system yields + coordinates that are well-formed and in the wrong place. +- **Changing the published model changes entity identifiers.** Editing one + value can therefore orphan everything already published; see ADR 005 and ADR + 006. +- **Fields no code reads have nothing keeping them current.** Until catalogue + entities are generated from them, only review does. +- Validating the record is work the environment did not require. diff --git a/docs/adr/CLAUDE.md b/docs/adr/CLAUDE.md new file mode 100644 index 0000000..679a5cd --- /dev/null +++ b/docs/adr/CLAUDE.md @@ -0,0 +1,78 @@ +# CLAUDE.md — writing ADRs in this project + +Guidance for Claude Code when creating or editing files in `docs/adr/`. + +An ADR here **decides a specific choice on general grounds**. The choice is +concrete; the reasoning must hold beyond any one data set, developer or machine. + +## Never include + +- **Local project or repository names.** These are organisation-level documents; + someone's working copies have no place in them. +- **Specific data sources, their fields, or their quirks.** Not field names, not + envelope shapes, not "the source provides…". State the *condition* the + decision applies under instead: "This decision applies where a data set + provides a count of units per location…". +- **Measurements or counts taken from a data set.** No record counts, no + "measured across N records", no byte or centimetre figures derived from one + export. Standards identifiers (EPSG codes, RFC numbers, framework versions) + and general domain facts are fine. +- **Educational or self-referential framing.** No "two things make this easy to + get wrong", no "note that", no "the real lesson is", no bolded aphorism + followed by the actual fact, and no commentary on the ADR's own importance + ("this is the hardest to reverse"). State the fact and stop. + +## Structure + +```text +# NNN: Area — the choice + +| Field | Value | Created By, Date, Decision Maker, Stakeholders, Status + +## Context the situation; what is undecided + ends with: "This ADR serves to decide …" +### Drivers Functional / Non-functional, brief +### Options Considered one paragraph each, upsides then downsides +## Decision the choice, then terse rationale bullets +## Consequences +### Positive +### Negative / Trade-offs the honest costs, including ones found by hitting them +``` + +The Context section **must end with a single "This ADR serves to …" sentence** +so the purpose is unmissable. + +Status is `Draft` while it needs review, then `Accepted`. Other values: +`Rejected`, `Deprecated by NNN`, `Supersedes NNN`. + +## Numbering + +Numbering follows **dependency order**: every ADR may reference only +lower-numbered ones. Check for forward references after adding or renumbering. +Dates therefore need not run in the same order as numbers. + +Renumbering is only acceptable while nothing external cites the numbers. Once +cited, supersede instead. + +## Before writing, ask whether it earns its place + +- **Is there a real alternative?** An ADR comparing one viable candidate against + a dead end documents a decision that made itself, and dilutes the ones + recording genuine trade-offs. Reframe it around the decisions that did have + alternatives, or fold it into a neighbour. +- **Would folding it into an existing ADR be better?** Prefer fewer, shorter + documents. Consolidation has been chosen over adding a document more than + once. +- **Are the rules distinct?** One principle restated against several targets is + one rule with sub-points, not several rules. + +## Before finishing + +- Grep for source field names, model names in the general ADRs, project names, + record counts and measured figures. +- Grep for `Note that`, `worth noting`, `is the lesson`, `easy to get wrong`, + `Two things`, `earns its`, `is the point`. +- Check no ADR cites a higher number than its own. +- `task coding-standards:markdown:check` — recurring failures are line length + over 120 (MD013), misaligned table pipes (MD060), and bold used as a heading + (MD036); use `####` for sub-headings inside a section. diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 0000000..0560124 --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,22 @@ +# Architecture Decision Records + +This directory contains Architecture Decision Records (ADRs) for this project. + +See [adr.github.io](https://adr.github.io/) for background on the format. + +| Number | Title | Status | Date | +| ----------------------------------------------- | ------------------------------------------------------- | ------ | ---------- | +| [001](001-architecture-symfony-docker.md) | Architecture — Symfony 8 on the ITK Dev Docker template | Draft | 2026-08-24 | +| [002](002-publish-to-a-context-broker.md) | Publication mechanism — publish to a context broker | Draft | 2026-08-24 | +| [003](003-ngsi-ld-representation.md) | NGSI-LD representation — normalized form, batch upsert | Draft | 2026-08-24 | +| [004](004-coordinate-reference-system.md) | Coordinate reference system — publish WGS84 | Draft | 2026-08-27 | +| [005](005-smart-data-models-as-vocabulary.md) | Vocabulary — adopt Smart Data Models | Draft | 2026-08-31 | +| [006](006-onstreetparking-over-parkinggroup.md) | Model selection — OnStreetParking over ParkingGroup | Draft | 2026-08-31 | +| [007](007-source-manifest.md) | Data set metadata — a committed source manifest | Draft | 2026-09-02 | + +Numbering follows dependency order: each ADR relies only on lower-numbered +ones. Dates therefore do not run in the same order as numbers. + +All ADRs state general policy and name no data set. Concrete per-data-set +facts are recorded in `config/sources.yaml`, and the mappings in the source +classes themselves. diff --git a/src/Broker/NgsiLdBroker.php b/src/Broker/NgsiLdBroker.php new file mode 100644 index 0000000..806a99f --- /dev/null +++ b/src/Broker/NgsiLdBroker.php @@ -0,0 +1,69 @@ +> $entities + * + * @return int the broker's HTTP status code + */ + public function upsert(array $entities): int + { + if ([] === $entities) { + return 204; + } + + $response = $this->client->request( + 'POST', + rtrim($this->brokerUrl, '/').self::UPSERT_PATH, + [ + 'headers' => ['Content-Type' => self::CONTENT_TYPE], + 'json' => $entities, + ] + ); + + $status = $response->getStatusCode(); + + if ($status >= 400) { + throw new \RuntimeException(\sprintf('Broker rejected the upsert with HTTP %d: %s', $status, $response->getContent(false))); + } + + return $status; + } + + public function brokerUrl(): string + { + return $this->brokerUrl; + } +} diff --git a/src/Command/ImportCommand.php b/src/Command/ImportCommand.php new file mode 100644 index 0000000..b0c6a46 --- /dev/null +++ b/src/Command/ImportCommand.php @@ -0,0 +1,157 @@ + $sources + */ + public function __construct( + #[AutowireIterator('app.source')] + private readonly iterable $sources, + private readonly NgsiLdBroker $broker, + #[Autowire(env: 'ENTER_NGSI_CONTEXT_URLS')] + private readonly string $contextUrls, + ) { + parent::__construct(); + } + + protected function configure(): void + { + $this + ->addArgument('source', InputArgument::OPTIONAL, 'Source to import. Omit to list the available sources.') + ->addOption('dry-run', null, InputOption::VALUE_NONE, 'Print the NGSI-LD payload instead of sending it.') + ->addOption('limit', 'l', InputOption::VALUE_REQUIRED, 'Import at most this many entities.'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $io = new SymfonyStyle($input, $output); + + $sources = []; + foreach ($this->sources as $source) { + $sources[$source->key()] = $source; + } + + if ([] === $sources) { + $io->error('No data sources are registered.'); + $io->listing([ + 'A source must implement App\Source\SourceInterface.', + 'Implementations are picked up automatically — check the class exists and is not excluded from the container.', + ]); + + return Command::FAILURE; + } + + $key = $input->getArgument('source'); + + if (null === $key) { + // Options only make sense together with a source. Listing the + // sources and exiting successfully would look like an import ran. + if ($input->getOption('dry-run') || null !== $input->getOption('limit')) { + $io->error(\sprintf( + 'No source given. Available: %s.', + implode(', ', array_keys($sources)) + )); + + return Command::INVALID; + } + + $io->section('Available sources'); + $io->listing(array_keys($sources)); + + return Command::SUCCESS; + } + + if (!isset($sources[$key])) { + $io->error(\sprintf('Unknown source "%s". Available: %s.', $key, implode(', ', array_keys($sources)))); + + return Command::INVALID; + } + + $limit = null !== $input->getOption('limit') ? max(1, (int) $input->getOption('limit')) : null; + $contexts = $this->contexts(); + + $payload = []; + foreach ($sources[$key]->entities() as $entity) { + $payload[] = $entity->toArray($contexts); + + if (null !== $limit && \count($payload) >= $limit) { + break; + } + } + + // A source that yields nothing is almost always misconfigured rather + // than genuinely empty, and it fails silently by construction: a + // record skipped for a missing field looks exactly like a feed with no + // records. Fail loudly so it cannot be mistaken for a successful run. + if ([] === $payload) { + $io->error(\sprintf('Source "%s" produced no entities.', $key)); + $io->text( + 'The source ran to completion without raising an exception, so every record was ' + .'discarded by the source\'s own guards rather than failing. Verbosity flags will ' + .'not reveal more: there is no exception to show.' + ); + $io->listing([ + 'Does the configured path or URL point at the intended document?', + 'Does the document match the shape the source expects — envelope, nesting, field names?', + 'Which guard returns early — a missing identifier, or a missing geometry?', + ]); + + return Command::FAILURE; + } + + if ($input->getOption('dry-run')) { + $output->writeln(json_encode($payload, self::JSON_FLAGS)); + $io->note(\sprintf('Dry run: %d entities were not sent.', \count($payload))); + + return Command::SUCCESS; + } + + try { + $status = $this->broker->upsert($payload); + } catch (\Throwable $exception) { + $io->error($exception->getMessage()); + + return Command::FAILURE; + } + + $io->success(\sprintf( + 'Upserted %d entities into %s (HTTP %d).', + \count($payload), + $this->broker->brokerUrl(), + $status + )); + + return Command::SUCCESS; + } + + /** + * @return list + */ + private function contexts(): array + { + return array_values(array_filter(array_map(trim(...), explode(',', $this->contextUrls)))); + } +} diff --git a/src/Geo/Wgs84Transformer.php b/src/Geo/Wgs84Transformer.php new file mode 100644 index 0000000..4894555 --- /dev/null +++ b/src/Geo/Wgs84Transformer.php @@ -0,0 +1,160 @@ + + */ + private const array DEFINITIONS = [ + // ETRS89 / UTM zone 32N — most of Denmark. + 'EPSG:25832' => '+proj=utm +zone=32 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs', + // ETRS89 / UTM zone 33N — Bornholm and eastwards. + 'EPSG:25833' => '+proj=utm +zone=33 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs', + // Web Mercator, in case an input arrives as tile coordinates. + 'EPSG:3857' => '+proj=merc +a=6378137 +b=6378137 +lat_ts=0 +lon_0=0 +x_0=0 +y_0=0 +k=1 +units=m +nadgrids=@null +no_defs', + self::TARGET_SRID => '+proj=longlat +datum=WGS84 +no_defs', + ]; + + private readonly Proj4php $proj4; + + /** @var array */ + private readonly array $definitions; + + /** @var array */ + private array $projections = []; + + /** + * @param array $definitions additional PROJ definitions, keyed by SRID + */ + public function __construct(array $definitions = []) + { + $this->definitions = [...self::DEFINITIONS, ...$definitions]; + + $this->proj4 = new Proj4php(); + foreach ($this->definitions as $srid => $definition) { + $this->proj4->addDef($srid, $definition); + } + } + + /** + * Coordinates are not rounded. Downstream use is unknown and may include + * planning work, so the transformed value is published as computed. + * + * @param string $srid source CRS, e.g. "EPSG:25832" + * + * @return array{float, float} GeoJSON coordinate order: [longitude, latitude] + */ + public function toWgs84(string $srid, float $x, float $y): array + { + if (self::TARGET_SRID === $srid) { + return [$x, $y]; + } + + $transformed = $this->proj4->transform( + $this->projection(self::TARGET_SRID), + new Point($x, $y, $this->projection($srid)) + ); + + return [(float) $transformed->x, (float) $transformed->y]; + } + + /** + * @param string $srid source CRS, e.g. "EPSG:25832" + * + * @return array{type: string, coordinates: array{float, float}} GeoJSON Point + */ + public function point(string $srid, float $x, float $y): array + { + return [ + 'type' => 'Point', + 'coordinates' => $this->toWgs84($srid, $x, $y), + ]; + } + + /** + * Reprojects a whole GeoJSON geometry, whatever its type. + * + * Handles Point, LineString, Polygon, MultiPoint, MultiLineString and + * MultiPolygon — every type an NGSI-LD GeoProperty accepts. GeometryCollection + * is not supported, because it carries `geometries` rather than `coordinates`. + * + * A third ordinate (elevation) is dropped: the inputs are two-dimensional, + * and vertical datums are a separate concern this class does not model. + * + * @param string $srid source CRS, e.g. "EPSG:25832" + * @param array $geometry GeoJSON geometry object + * + * @return array{type: string, coordinates: mixed} + */ + public function geometry(string $srid, array $geometry): array + { + $type = $geometry['type'] ?? null; + + if (!\is_string($type) || !\array_key_exists('coordinates', $geometry)) { + throw new \InvalidArgumentException('Not a GeoJSON geometry: "type" and "coordinates" are both required.'); + } + + return [ + 'type' => $type, + 'coordinates' => $this->transformCoordinates($srid, $geometry['coordinates']), + ]; + } + + /** + * GeoJSON nests coordinates to a depth that depends on the geometry type: + * a bare position for Point, an array of positions for LineString, an array + * of those for Polygon, and so on. Recursing until the first element is + * numeric handles every depth without enumerating the types. + * + * @return array + */ + private function transformCoordinates(string $srid, mixed $coordinates): array + { + if (!\is_array($coordinates) || [] === $coordinates) { + throw new \InvalidArgumentException('GeoJSON coordinates must be a non-empty array.'); + } + + if (is_numeric($coordinates[0] ?? null)) { + if (!is_numeric($coordinates[1] ?? null)) { + throw new \InvalidArgumentException('A GeoJSON position needs at least two ordinates.'); + } + + return $this->toWgs84($srid, (float) $coordinates[0], (float) $coordinates[1]); + } + + return array_map( + fn (mixed $nested): array => $this->transformCoordinates($srid, $nested), + array_values($coordinates) + ); + } + + private function projection(string $srid): Proj + { + if (!isset($this->definitions[$srid])) { + throw new \InvalidArgumentException(\sprintf('Unknown CRS "%s". Register a PROJ definition for it before use. Known: %s.', $srid, implode(', ', array_keys($this->definitions)))); + } + + return $this->projections[$srid] ??= new Proj($srid, $this->proj4); + } +} diff --git a/src/Ngsi/NgsiEntity.php b/src/Ngsi/NgsiEntity.php new file mode 100644 index 0000000..6f38391 --- /dev/null +++ b/src/Ngsi/NgsiEntity.php @@ -0,0 +1,81 @@ +> */ + private array $attributes = []; + + public function __construct( + private readonly string $id, + private readonly string $type, + ) { + } + + public function id(): string + { + return $this->id; + } + + /** + * Null and empty-string values are dropped rather than emitted as null, + * because the source data uses "" for "not filled in" and a broker would + * otherwise store the emptiness as a fact. + */ + public function property(string $name, mixed $value, ?string $observedAt = null): self + { + if (null === $value || '' === $value || [] === $value) { + return $this; + } + + $attribute = ['type' => 'Property', 'value' => $value]; + + if (null !== $observedAt) { + $attribute['observedAt'] = $observedAt; + } + + $this->attributes[$name] = $attribute; + + return $this; + } + + /** + * @param array{type: string, coordinates: mixed} $geoJson + */ + public function geoProperty(string $name, array $geoJson): self + { + $this->attributes[$name] = ['type' => 'GeoProperty', 'value' => $geoJson]; + + return $this; + } + + public function relationship(string $name, string $object): self + { + $this->attributes[$name] = ['type' => 'Relationship', 'object' => $object]; + + return $this; + } + + /** + * @param list $contextUrls + * + * @return array + */ + public function toArray(array $contextUrls): array + { + return [ + 'id' => $this->id, + 'type' => $this->type, + ...$this->attributes, + '@context' => $contextUrls, + ]; + } +} diff --git a/src/Source/FeedReader.php b/src/Source/FeedReader.php new file mode 100644 index 0000000..6a9c280 --- /dev/null +++ b/src/Source/FeedReader.php @@ -0,0 +1,58 @@ + the decoded document + * + * @throws \RuntimeException when the location is not an http(s) URL, cannot + * be fetched, does not contain valid JSON, or + * does not decode to an array + */ + public function read(string $location): array + { + if (!str_starts_with($location, 'http://') && !str_starts_with($location, 'https://')) { + throw new \RuntimeException(\sprintf('Feed location must be an http(s) URL, got "%s".', $location)); + } + + $json = $this->fetch($location); + + try { + $decoded = json_decode($json, true, 512, \JSON_THROW_ON_ERROR); + } catch (\JsonException $exception) { + throw new \RuntimeException(\sprintf('Invalid JSON in "%s": %s', $location, $exception->getMessage()), previous: $exception); + } + + // A JSON document may legally be a scalar. Every feed we consume is a + // list or an object, and a scalar here means the location is wrong + // rather than that the feed is empty. + if (!\is_array($decoded)) { + throw new \RuntimeException(\sprintf('Expected a JSON array or object in "%s", got %s.', $location, get_debug_type($decoded))); + } + + return $decoded; + } + + private function fetch(string $url): string + { + try { + return $this->client->request('GET', $url)->getContent(); + } catch (\Throwable $exception) { + throw new \RuntimeException(\sprintf('Could not fetch "%s": %s', $url, $exception->getMessage()), previous: $exception); + } + } +} diff --git a/src/Source/MtmSpatialMaps/HandicapParking.php b/src/Source/MtmSpatialMaps/HandicapParking.php new file mode 100644 index 0000000..72e0653 --- /dev/null +++ b/src/Source/MtmSpatialMaps/HandicapParking.php @@ -0,0 +1,109 @@ +catalog->get(self::KEY); + + // The export is a GeoJSON FeatureCollection, so the records live under + // `features`. Iterating the document itself would walk its two + // top-level keys instead. + foreach ($this->reader->read($source->accessUrl)['features'] ?? [] as $feature) { + if (\is_array($feature) && null !== $entity = $this->toEntity($feature, $source)) { + yield $entity; + } + } + } + + /** + * @param array $feature GeoJSON Feature + */ + private function toEntity(array $feature, SourceDescriptor $source): ?NgsiEntity + { + // A Feature keeps its attributes under `properties` and its geometry + // beside them, so neither is at the feature's top level. + $row = $feature['properties'] ?? null; + $geometry = $feature['geometry'] ?? null; + + if (!\is_array($row) || !\is_array($geometry)) { + return null; + } + + // mi_prinx is the feed's stable primary key. Without it there is no + // way to address the same bay again on the next import, and an upsert + // would create duplicates instead of updating. + $key = $row['mi_prinx'] ?? null; + if (null === $key || '' === $key) { + return null; + } + + $entity = new NgsiEntity( + \sprintf('urn:ngsi-ld:%s:aarhus-handicap-%s', $source->model, $key), + $source->model + ); + + // `forDisabled` rather than ParkingGroup's `onlyDisabled`: the two + // models have separate category enums, so values are not interchangeable. + // `onStreet` is dropped because the entity type already states it. + return $entity + ->property('name', $this->address($row)) + ->property('description', trim((string) ($row['bemrk'] ?? ''))) + ->property('category', ['forDisabled']) + ->property('totalSpotNumber', (int) ($row['invalidepladser'] ?? 0)) + ->property('source', $source->accessUrl) + ->geoProperty('location', $this->transformer->geometry($source->crs, $geometry)); + } + + /** + * @param array $row + */ + private function address(array $row): string + { + return trim(\sprintf( + '%s %s', + trim((string) ($row['vejnavn'] ?? '')), + trim((string) ($row['husnnr'] ?? '')) + )); + } +} diff --git a/src/Source/SourceCatalog.php b/src/Source/SourceCatalog.php new file mode 100644 index 0000000..cdcd9f9 --- /dev/null +++ b/src/Source/SourceCatalog.php @@ -0,0 +1,169 @@ +|null */ + private ?array $descriptors = null; + + public function __construct( + #[Autowire('%kernel.project_dir%/config/sources.yaml')] + private readonly string $manifest, + ) { + } + + /** + * @throws \RuntimeException when the manifest cannot be read, or carries no entry for the key + */ + public function get(string $key): SourceDescriptor + { + $descriptors = $this->all(); + + if (!isset($descriptors[$key])) { + throw new \RuntimeException(\sprintf('No entry for source "%s" in %s. Entries: %s.', $key, $this->manifest, [] === $descriptors ? 'none' : implode(', ', array_keys($descriptors)))); + } + + return $descriptors[$key]; + } + + /** + * @return array keyed by source key + * + * @throws \RuntimeException when the manifest cannot be read + */ + public function all(): array + { + return $this->descriptors ??= $this->load(); + } + + /** + * @return array + */ + private function load(): array + { + if (!is_file($this->manifest)) { + throw new \RuntimeException(\sprintf('Source manifest "%s" does not exist.', $this->manifest)); + } + + try { + $parsed = Yaml::parseFile($this->manifest); + } catch (ParseException $exception) { + throw new \RuntimeException(\sprintf('Source manifest "%s" is not valid YAML: %s', $this->manifest, $exception->getMessage()), previous: $exception); + } + + // Without this the wrong shape yields an empty catalogue, which reads + // as "no data sets are registered" rather than as a broken file. + $sources = \is_array($parsed) ? $parsed['sources'] ?? null : null; + if (!\is_array($sources)) { + throw new \RuntimeException(\sprintf('Source manifest "%s" must contain a "sources" mapping at the top level.', $this->manifest)); + } + + $descriptors = []; + foreach ($sources as $key => $entry) { + $key = (string) $key; + + if (!\is_array($entry)) { + throw new \RuntimeException(\sprintf('Entry "%s" in %s must be a mapping, got %s.', $key, $this->manifest, get_debug_type($entry))); + } + + $descriptors[$key] = $this->descriptor($key, $entry); + } + + return $descriptors; + } + + /** + * @param array $entry + */ + private function descriptor(string $key, array $entry): SourceDescriptor + { + return new SourceDescriptor( + key: $key, + title: $this->required($key, $entry, 'title'), + accessUrl: $this->required($key, $entry, 'access_url'), + crs: $this->required($key, $entry, 'crs'), + model: $this->required($key, $entry, 'model'), + description: $this->optional($key, $entry, 'description'), + publisher: $this->optional($key, $entry, 'publisher'), + contact: $this->optional($key, $entry, 'contact'), + landingPage: $this->optional($key, $entry, 'landing_page'), + mediaType: $this->optional($key, $entry, 'media_type'), + updateFrequency: $this->optional($key, $entry, 'update_frequency'), + licence: $this->optional($key, $entry, 'licence'), + omittedFields: $this->omittedFields($key, $entry), + ); + } + + /** + * @param array $entry + */ + private function required(string $key, array $entry, string $field): string + { + return $this->optional($key, $entry, $field) + ?? throw new \RuntimeException(\sprintf('Entry "%s" in %s is missing the required field "%s"; an import cannot run without it.', $key, $this->manifest, $field)); + } + + /** + * An empty value means "not filled in", the same as an absent key, so both + * become null rather than an empty string. + * + * @param array $entry + */ + private function optional(string $key, array $entry, string $field): ?string + { + $value = $entry[$field] ?? null; + + if (null === $value || '' === $value) { + return null; + } + + if (!is_scalar($value)) { + throw new \RuntimeException(\sprintf('Field "%s" of entry "%s" in %s must be a single value, got %s.', $field, $key, $this->manifest, get_debug_type($value))); + } + + return trim((string) $value); + } + + /** + * @param array $entry + * + * @return array + */ + private function omittedFields(string $key, array $entry): array + { + $omitted = $entry['omitted_fields'] ?? []; + + if (!\is_array($omitted)) { + throw new \RuntimeException(\sprintf('Field "omitted_fields" of entry "%s" in %s must map each field to the reason it is not published.', $key, $this->manifest)); + } + + $reasons = []; + foreach ($omitted as $field => $reason) { + // The reason is the half of the record that cannot be recovered + // from the code, so a bare list of names is not accepted. + if (!\is_string($reason) || '' === trim($reason)) { + throw new \RuntimeException(\sprintf('Omitted field "%s" of entry "%s" in %s needs a reason.', $field, $key, $this->manifest)); + } + + $reasons[(string) $field] = trim($reason); + } + + return $reasons; + } +} diff --git a/src/Source/SourceDescriptor.php b/src/Source/SourceDescriptor.php new file mode 100644 index 0000000..83b2b14 --- /dev/null +++ b/src/Source/SourceDescriptor.php @@ -0,0 +1,43 @@ + $omittedFields source field name => why it is not published + */ + public function __construct( + public string $key, + public string $title, + public string $accessUrl, + public string $crs, + public string $model, + public ?string $description = null, + public ?string $publisher = null, + public ?string $contact = null, + public ?string $landingPage = null, + public ?string $mediaType = null, + public ?string $updateFrequency = null, + public ?string $licence = null, + public array $omittedFields = [], + ) { + } +} diff --git a/src/Source/SourceInterface.php b/src/Source/SourceInterface.php new file mode 100644 index 0000000..978c5ce --- /dev/null +++ b/src/Source/SourceInterface.php @@ -0,0 +1,30 @@ + + */ + public function entities(): iterable; +} diff --git a/task/scripts/broker-entities b/task/scripts/broker-entities new file mode 100755 index 0000000..280bb6c --- /dev/null +++ b/task/scripts/broker-entities @@ -0,0 +1,34 @@ +#!/bin/sh +# List entities of a given type from the NGSI-LD broker. +# +# Read requests need a domain context supplied in a Link header; without it the +# broker cannot expand a short type name into the full term the entity was +# written under. The context comes from ENTER_NGSI_DOMAIN_CONTEXT, which must +# be the one defining the type being asked for. +# +# Usage: broker-entities [limit] + +set -eu + +if [ $# -lt 1 ]; then + echo "Usage: broker-entities [limit]" >&2 + echo " NGSI-LD entity type, e.g. OnStreetParking" >&2 + echo " [limit] maximum entities to return (default 100)" >&2 + exit 64 +fi + +TYPE="$1" +LIMIT="${2:-100}" + +RESPONSE=$(curl -sS \ + -H "Link: <${ENTER_NGSI_DOMAIN_CONTEXT}>; rel=\"http://www.w3.org/ns/json-ld#context\"; type=\"application/ld+json\"" \ + "${APP_BROKER_BASE_URI%/}/ngsi-ld/v1/entities?type=${TYPE}&limit=${LIMIT}") + +printf '%s\n' "$RESPONSE" + +# A misspelled type, or one whose context is not the one configured, returns an +# empty list with HTTP 200 — indistinguishable from an empty broker or a failed +# import. Say what was asked for so the difference is visible. +if [ "$(printf '%s' "$RESPONSE" | tr -d '[:space:]')" = "[]" ]; then + printf '\nNo entities of type "%s". A type that does not match, or one defined in a context other than the configured ENTER_NGSI_DOMAIN_CONTEXT, returns an empty list rather than an error — check both before concluding the import failed.\n' "$TYPE" >&2 +fi diff --git a/tests/Command/ImportCommandTest.php b/tests/Command/ImportCommandTest.php new file mode 100644 index 0000000..ae858a3 --- /dev/null +++ b/tests/Command/ImportCommandTest.php @@ -0,0 +1,148 @@ + $sources + */ + private function tester(iterable $sources): CommandTester + { + return new CommandTester(new ImportCommand( + $sources, + new NgsiLdBroker(new MockHttpClient(), 'http://broker.invalid'), + 'https://example.com/context.jsonld', + )); + } + + private function source(string $key, NgsiEntity ...$entities): SourceInterface + { + return new readonly class($key, $entities) implements SourceInterface { + /** @param list $entities */ + public function __construct( + private string $key, + private array $entities, + ) { + } + + public function key(): string + { + return $this->key; + } + + public function entities(): iterable + { + yield from $this->entities; + } + }; + } + + /** + * The important one: a source yielding nothing used to exit successfully + * with a warning, which is indistinguishable from a working import. + */ + public function testItFailsWhenASourceProducesNothing(): void + { + $tester = $this->tester([$this->source('empty-source')]); + + $status = $tester->execute(['source' => 'empty-source', '--dry-run' => true]); + + $this->assertSame(Command::FAILURE, $status); + $this->assertStringContainsString('produced no entities', $tester->getDisplay()); + } + + public function testItSuggestsCausesWhenASourceProducesNothing(): void + { + $tester = $this->tester([$this->source('empty-source')]); + $tester->execute(['source' => 'empty-source', '--dry-run' => true]); + + $display = $tester->getDisplay(); + + $this->assertStringContainsString('path or URL', $display); + $this->assertStringContainsString('envelope, nesting, field names', $display); + } + + public function testItFailsWhenNoSourcesAreRegistered(): void + { + $tester = $this->tester([]); + + $status = $tester->execute([]); + + $this->assertSame(Command::FAILURE, $status); + $this->assertStringContainsString('No data sources are registered', $tester->getDisplay()); + } + + /** + * Passing --dry-run with no source used to print the source listing and + * exit successfully, silently ignoring the flag. + */ + public function testItRejectsOptionsWithoutASource(): void + { + $tester = $this->tester([$this->source('some-source')]); + + $status = $tester->execute(['--dry-run' => true]); + + $this->assertSame(Command::INVALID, $status); + $this->assertStringContainsString('No source given', $tester->getDisplay()); + } + + public function testItStillListsSourcesWhenCalledBare(): void + { + $tester = $this->tester([$this->source('some-source')]); + + $status = $tester->execute([]); + + $this->assertSame(Command::SUCCESS, $status); + $this->assertStringContainsString('some-source', $tester->getDisplay()); + } + + public function testItRejectsAnUnknownSource(): void + { + $tester = $this->tester([$this->source('some-source')]); + + $status = $tester->execute(['source' => 'nope']); + + $this->assertSame(Command::INVALID, $status); + $this->assertStringContainsString('Unknown source "nope"', $tester->getDisplay()); + } + + public function testDryRunPrintsThePayloadAndSendsNothing(): void + { + $entity = new NgsiEntity('urn:ngsi-ld:Example:1', 'Example') + ->property('name', 'Example'); + + $tester = $this->tester([$this->source('one-entity', $entity)]); + + $status = $tester->execute(['source' => 'one-entity', '--dry-run' => true]); + $display = $tester->getDisplay(); + + $this->assertSame(Command::SUCCESS, $status); + $this->assertStringContainsString('"urn:ngsi-ld:Example:1"', $display); + $this->assertStringContainsString('1 entities were not sent', $display); + } + + public function testLimitCapsThePayload(): void + { + $entities = []; + foreach (range(1, 5) as $i) { + $entities[] = new NgsiEntity(\sprintf('urn:ngsi-ld:Example:%d', $i), 'Example'); + } + + $tester = $this->tester([$this->source('many', ...$entities)]); + $tester->execute(['source' => 'many', '--dry-run' => true, '--limit' => 2]); + + $this->assertStringContainsString('2 entities were not sent', $tester->getDisplay()); + } +} diff --git a/tests/Geo/Wgs84TransformerTest.php b/tests/Geo/Wgs84TransformerTest.php new file mode 100644 index 0000000..aacf039 --- /dev/null +++ b/tests/Geo/Wgs84TransformerTest.php @@ -0,0 +1,182 @@ +transformer = new Wgs84Transformer(); + } + + public function testItReprojectsFromUtm32ToWgs84(): void + { + [$longitude, $latitude] = $this->transformer->toWgs84( + self::UTM32, + self::REFERENCE_EASTING, + self::REFERENCE_NORTHING + ); + + $this->assertEqualsWithDelta(10.2152, $longitude, 0.001); + $this->assertEqualsWithDelta(56.1592, $latitude, 0.001); + } + + public function testItReturnsLongitudeLatitudeOrder(): void + { + $point = $this->transformer->point(self::UTM32, 574108.2557507273, 6222343.6199512165); + + $this->assertSame('Point', $point['type']); + + // Longitude first, per GeoJSON. A swapped pair is easy to spot here: + // latitude cannot be 10.19. + $this->assertGreaterThan(9.0, $point['coordinates'][0]); + $this->assertLessThan(11.0, $point['coordinates'][0]); + $this->assertGreaterThan(55.0, $point['coordinates'][1]); + $this->assertLessThan(57.0, $point['coordinates'][1]); + } + + public function testItPassesThroughCoordinatesAlreadyInWgs84(): void + { + $this->assertSame( + [10.199512, 56.149753], + $this->transformer->toWgs84(Wgs84Transformer::TARGET_SRID, 10.199512, 56.149753) + ); + } + + public function testItSupportsAnotherRegisteredCrs(): void + { + // UTM zone 33N: same northing, easting near the zone's central + // meridian, so the result must land east of zone 32's output. + [$longitude] = $this->transformer->toWgs84('EPSG:25833', 500000.0, 6224460.0); + + $this->assertEqualsWithDelta(15.0, $longitude, 0.001); + } + + public function testItRejectsAnUnregisteredCrs(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessageMatches('/Unknown CRS "EPSG:31700"/'); + + $this->transformer->toWgs84('EPSG:31700', 500000.0, 6224460.0); + } + + public function testItAcceptsAdditionalDefinitions(): void + { + $transformer = new Wgs84Transformer([ + 'EPSG:23032' => '+proj=utm +zone=32 +ellps=intl +units=m +no_defs', + ]); + + [$longitude, $latitude] = $transformer->toWgs84('EPSG:23032', self::REFERENCE_EASTING, self::REFERENCE_NORTHING); + + // Different datum, so not identical to EPSG:25832 — but the same + // corner of the world, which proves the definition was registered. + $this->assertEqualsWithDelta(10.2152, $longitude, 0.01); + $this->assertEqualsWithDelta(56.1592, $latitude, 0.01); + } + + public function testItReprojectsAPointGeometry(): void + { + $geometry = $this->transformer->geometry(self::UTM32, [ + 'type' => 'Point', + 'coordinates' => [self::REFERENCE_EASTING, self::REFERENCE_NORTHING], + ]); + + $this->assertSame('Point', $geometry['type']); + $this->assertEqualsWithDelta(10.2152, $geometry['coordinates'][0], 0.001); + } + + public function testItReprojectsALineString(): void + { + $geometry = $this->transformer->geometry(self::UTM32, [ + 'type' => 'LineString', + 'coordinates' => [ + [574108.2557507273, 6222343.6199512165], + [self::REFERENCE_EASTING, self::REFERENCE_NORTHING], + ], + ]); + + $this->assertSame('LineString', $geometry['type']); + $this->assertCount(2, $geometry['coordinates']); + $this->assertEqualsWithDelta(10.1926, $geometry['coordinates'][0][0], 0.001); + $this->assertEqualsWithDelta(10.2152, $geometry['coordinates'][1][0], 0.001); + } + + public function testItReprojectsAPolygonPreservingNesting(): void + { + $geometry = $this->transformer->geometry(self::UTM32, [ + 'type' => 'Polygon', + 'coordinates' => [ + [ + [574108.0, 6222343.0], + [574208.0, 6222343.0], + [574208.0, 6222443.0], + [574108.0, 6222343.0], + ], + ], + ]); + + $this->assertSame('Polygon', $geometry['type']); + $this->assertCount(1, $geometry['coordinates']); + $this->assertCount(4, $geometry['coordinates'][0]); + $this->assertIsFloat($geometry['coordinates'][0][0][0]); + } + + public function testItReprojectsAMultiPolygon(): void + { + $ring = [ + [574108.0, 6222343.0], + [574208.0, 6222343.0], + [574208.0, 6222443.0], + [574108.0, 6222343.0], + ]; + + $geometry = $this->transformer->geometry(self::UTM32, [ + 'type' => 'MultiPolygon', + 'coordinates' => [[$ring], [$ring]], + ]); + + $this->assertCount(2, $geometry['coordinates']); + $this->assertCount(4, $geometry['coordinates'][1][0]); + } + + public function testItRejectsAGeometryWithoutCoordinates(): void + { + $this->expectException(\InvalidArgumentException::class); + + $this->transformer->geometry(self::UTM32, ['type' => 'Point']); + } + + public function testItRejectsAGeometryCollection(): void + { + $this->expectException(\InvalidArgumentException::class); + + $this->transformer->geometry(self::UTM32, [ + 'type' => 'GeometryCollection', + 'geometries' => [], + ]); + } + + public function testItRejectsAPositionWithASingleOrdinate(): void + { + $this->expectException(\InvalidArgumentException::class); + + $this->transformer->geometry(self::UTM32, ['type' => 'Point', 'coordinates' => [574108.0]]); + } +} diff --git a/tests/Source/FeedReaderTest.php b/tests/Source/FeedReaderTest.php new file mode 100644 index 0000000..8d925b8 --- /dev/null +++ b/tests/Source/FeedReaderTest.php @@ -0,0 +1,114 @@ +reader($client)->read('https://example.com/feed.json'); + + $this->assertSame([['example' => 1]], $document); + } + + public function testItAcceptsABareArrayDocument(): void + { + $client = new MockHttpClient(new MockResponse('[1, 2, 3]')); + + $this->assertSame([1, 2, 3], $this->reader($client)->read('http://example.com/feed.json')); + } + + /** + * The envelope is feed-specific knowledge, so it must survive intact for + * the caller to interpret. Returning the features list directly would push + * that knowledge into the wrong class. + */ + public function testItDoesNotUnwrapTheEnvelope(): void + { + $client = new MockHttpClient(new MockResponse(self::FEATURE_COLLECTION)); + + $document = $this->reader($client)->read('https://example.com/feed.json'); + + $this->assertArrayHasKey('features', $document); + $this->assertArrayNotHasKey(0, $document); + $this->assertCount(2, $document['features']); + } + + public function testItRejectsALocationThatIsNotAUrl(): void + { + $client = new MockHttpClient(); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessageMatches('/must be an http\(s\) URL/'); + + try { + $this->reader($client)->read('data/feed.json'); + } finally { + // Rejection has to happen before the request, or a mistyped + // location becomes an opaque transport error instead. + $this->assertSame(0, $client->getRequestsCount()); + } + } + + /** + * A string that merely begins with "http" is not a URL. + */ + public function testItRejectsAPathThatMerelyStartsWithHttp(): void + { + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessageMatches('/must be an http\(s\) URL/'); + + $this->reader()->read('https-export.json'); + } + + public function testItFailsOnInvalidJson(): void + { + $client = new MockHttpClient(new MockResponse('{ not json')); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessageMatches('/Invalid JSON/'); + + $this->reader($client)->read('https://example.com/feed.json'); + } + + public function testItFailsWhenTheDocumentIsAScalar(): void + { + $client = new MockHttpClient(new MockResponse('"just a string"')); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessageMatches('/Expected a JSON array or object/'); + + $this->reader($client)->read('https://example.com/feed.json'); + } + + public function testItWrapsTransportFailures(): void + { + $client = new MockHttpClient(new MockResponse('', ['http_code' => 500])); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessageMatches('/Could not fetch/'); + + $this->reader($client)->read('https://example.com/feed.json'); + } +} diff --git a/tests/Source/MtmSpatialMaps/HandicapParkingTest.php b/tests/Source/MtmSpatialMaps/HandicapParkingTest.php new file mode 100644 index 0000000..e0065a7 --- /dev/null +++ b/tests/Source/MtmSpatialMaps/HandicapParkingTest.php @@ -0,0 +1,190 @@ +> */ + private array $entities; + + protected function setUp(): void + { + $catalog = new SourceCatalog(\dirname(__DIR__, 3).'/config/sources.yaml'); + $this->source = $catalog->get(self::KEY); + + $client = new MockHttpClient(function (string $method, string $url): MockResponse { + $this->requestedUrl = $url; + + return new MockResponse(json_encode($this->feed(), \JSON_THROW_ON_ERROR)); + }); + + $source = new HandicapParking(new FeedReader($client), new Wgs84Transformer(), $catalog); + + $this->entities = array_map( + static fn (NgsiEntity $entity): array => $entity->toArray(['https://example.com/context.jsonld']), + iterator_to_array($source->entities(), false) + ); + } + + public function testItReadsTheFeedTheManifestPointsAt(): void + { + $this->assertSame($this->source->accessUrl, $this->requestedUrl); + } + + public function testItSkipsRecordsWithoutAnIdentifierOrGeometry(): void + { + // Four features, of which one has no mi_prinx and one no geometry. + $this->assertCount(2, $this->entities); + } + + public function testItTakesIdentifierAndTypeFromTheManifestModel(): void + { + $first = $this->entities[0]; + + // The id is derived from mi_prinx so that re-importing upserts the + // same bay instead of creating a duplicate. + $this->assertSame( + \sprintf('urn:ngsi-ld:%s:aarhus-handicap-172', $this->source->model), + $first['id'] + ); + $this->assertSame($this->source->model, $first['type']); + } + + public function testItJoinsStreetAndHouseNumberIntoName(): void + { + $this->assertSame('Domkirkeplads/Bispegade 1', $this->entities[0]['name']['value']); + } + + public function testItOmitsTheHouseNumberWhenBlank(): void + { + // husnnr is "" for this row, so the name must not end in a space. + $this->assertSame('Brammersgade', $this->entities[1]['name']['value']); + } + + public function testItMarksEveryEntityAsDisabledParking(): void + { + foreach ($this->entities as $entity) { + $this->assertSame(['forDisabled'], $entity['category']['value']); + } + } + + public function testItCarriesTheBayCountAsTotalSpotNumber(): void + { + $this->assertSame(6, $this->entities[0]['totalSpotNumber']['value']); + $this->assertSame(1, $this->entities[1]['totalSpotNumber']['value']); + } + + public function testItDropsEmptyDescriptions(): void + { + // bemrk is null for the first row and filled in for the second. + $this->assertArrayNotHasKey('description', $this->entities[0]); + $this->assertSame('Ved indgangen', $this->entities[1]['description']['value']); + } + + public function testItReprojectsLocationIntoWgs84(): void + { + $location = $this->entities[0]['location']; + + $this->assertSame('GeoProperty', $location['type']); + $this->assertSame('Point', $location['value']['type']); + + // The feed publishes metres in the manifest's CRS, so degrees within + // Denmark are the evidence that the reprojection ran. + [$longitude, $latitude] = $location['value']['coordinates']; + $this->assertGreaterThan(8.0, $longitude); + $this->assertLessThan(13.0, $longitude); + $this->assertGreaterThan(54.5, $latitude); + $this->assertLessThan(57.8, $latitude); + } + + public function testItRecordsTheManifestUrlAsTheEntitySource(): void + { + $this->assertSame($this->source->accessUrl, $this->entities[0]['source']['value']); + } + + /** + * The first feature is a record from the live export, kept verbatim. The + * rest are constructed to exercise a blank house number and the two guards + * that discard a record. + * + * @return array + */ + private function feed(): array + { + return [ + 'type' => 'FeatureCollection', + 'crs' => ['type' => 'name', 'properties' => ['name' => 'EPSG:25832']], + 'features' => [ + [ + 'type' => 'Feature', + 'geometry' => ['type' => 'Point', 'coordinates' => [575153.9524951308, 6224260.609753487]], + 'properties' => [ + 'vejnavn' => 'Domkirkeplads/Bispegade', + 'husnnr' => '1', + 'invalidepladser' => 6, + 'bemrk' => null, + 'ident' => 'P', + 'oprettet_af' => 'ADM\\aztnbnd', + 'oprettet_dato' => '2019-03-19 14:51:23.91', + 'rettet_af' => 'ADM\\aztnbnd', + 'rettet_dato' => '2019-03-19 14:51:23.91', + 'mi_style' => null, + 'mi_prinx' => 172, + ], + ], + [ + 'type' => 'Feature', + 'geometry' => ['type' => 'Point', 'coordinates' => [574000.0, 6223000.0]], + 'properties' => [ + 'vejnavn' => 'Brammersgade', + 'husnnr' => '', + 'invalidepladser' => 1, + 'bemrk' => 'Ved indgangen', + 'mi_prinx' => 378, + ], + ], + [ + 'type' => 'Feature', + 'geometry' => ['type' => 'Point', 'coordinates' => [574100.0, 6223100.0]], + 'properties' => [ + 'vejnavn' => 'Uden nøgle', + 'husnnr' => '3', + 'invalidepladser' => 2, + ], + ], + [ + 'type' => 'Feature', + 'geometry' => null, + 'properties' => [ + 'vejnavn' => 'Uden geometri', + 'husnnr' => '5', + 'invalidepladser' => 2, + 'mi_prinx' => 999, + ], + ], + ], + ]; + } +} diff --git a/tests/Source/SourceCatalogTest.php b/tests/Source/SourceCatalogTest.php new file mode 100644 index 0000000..e5f1dbd --- /dev/null +++ b/tests/Source/SourceCatalogTest.php @@ -0,0 +1,144 @@ + */ + private array $written = []; + + protected function tearDown(): void + { + foreach ($this->written as $path) { + if (is_file($path)) { + unlink($path); + } + } + + $this->written = []; + } + + public function testTheShippedManifestIsUsable(): void + { + $catalog = new SourceCatalog(\dirname(__DIR__, 2).'/config/sources.yaml'); + + $this->assertNotSame([], $catalog->all(), 'The manifest registers no data sets.'); + } + + /** + * A wrong URL scheme or a CRS the transformer does not know only surfaces + * mid-import otherwise, after the feed has been fetched. + */ + public function testEveryShippedEntryCanBeImportedFrom(): void + { + $catalog = new SourceCatalog(\dirname(__DIR__, 2).'/config/sources.yaml'); + + foreach ($catalog->all() as $key => $descriptor) { + $this->assertSame($key, $descriptor->key); + $this->assertMatchesRegularExpression('#^https?://#', $descriptor->accessUrl, $key); + $this->assertMatchesRegularExpression('/^EPSG:\d+$/', $descriptor->crs, $key); + $this->assertNotSame('', $descriptor->model, $key); + } + } + + public function testItNamesTheKnownEntriesWhenAskedForAnUnknownOne(): void + { + $catalog = new SourceCatalog($this->manifest(<<<'YAML' + sources: + a-source: + title: A source + access_url: https://example.com/feed.json + crs: EPSG:25832 + model: Example + YAML)); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('Entries: a-source.'); + + $catalog->get('no-such-source'); + } + + public function testItRejectsAManifestWithoutASourcesMapping(): void + { + $catalog = new SourceCatalog($this->manifest("data_sets:\n a-source: {}\n")); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('must contain a "sources" mapping'); + + $catalog->all(); + } + + public function testItRejectsAnEntryMissingAFieldTheImportNeeds(): void + { + $catalog = new SourceCatalog($this->manifest(<<<'YAML' + sources: + a-source: + title: A source + access_url: https://example.com/feed.json + model: Example + YAML)); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('missing the required field "crs"'); + + $catalog->all(); + } + + public function testItRejectsAnOmittedFieldWithoutAReason(): void + { + $catalog = new SourceCatalog($this->manifest(<<<'YAML' + sources: + a-source: + title: A source + access_url: https://example.com/feed.json + crs: EPSG:25832 + model: Example + omitted_fields: + some_field: ~ + YAML)); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('needs a reason'); + + $catalog->all(); + } + + public function testItReportsAManifestThatIsNotThere(): void + { + $catalog = new SourceCatalog('/no/such/sources.yaml'); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('does not exist'); + + $catalog->all(); + } + + public function testItReportsUnparsableYaml(): void + { + $catalog = new SourceCatalog($this->manifest("sources:\n - [unbalanced\n")); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('is not valid YAML'); + + $catalog->all(); + } + + private function manifest(string $yaml): string + { + $path = tempnam(sys_get_temp_dir(), 'sources-'); + + if (false === $path) { + $this->fail('Could not create a temporary manifest.'); + } + + file_put_contents($path, $yaml); + $this->written[] = $path; + + return $path; + } +}