From d7f8404b6296ef34c6478d0bbece6fa320a4a8c5 Mon Sep 17 00:00:00 2001 From: anierbeck Date: Sun, 11 Oct 2026 18:30:07 +0200 Subject: [PATCH 1/2] Add optional contentCreatedEnd for content created over a period contentCreated could only hold one point in time and told authors to use the start of a period, which dropped the end. contentCreatedEnd is an optional companion: contentCreated stays the start of the period, so existing records remain valid. The SHACL shapes reject an end that lies before the start. The same shapes were checked with pyshacl and with Jena SHACL, which the registry uses. Apache-ai: Yes Generated-by: pi (claude-haiku-5-5) Reviewed-by: anierbeck --- CHANGELOG.md | 3 ++ context/0.0.1/sourcelume.jsonld | 4 +++ schema/0.0.1/sourcelume.schema.json | 7 ++++- schema/0.0.1/sourcelume.shacl.ttl | 6 ++++ spec/0.0.1/index.md | 3 +- tests/test_validate.py | 49 +++++++++++++++++++++++++++++ 6 files changed, 70 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1eb5288..db0f83b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,9 @@ described in `VERSIONING.md`. ### Added +- Optional `contentCreatedEnd`: end of the period in which a dataset's content was created. + `contentCreated` stays the start of the period, so existing records remain valid. The SHACL + shapes reject an end earlier than the start. - Added intended usage for custodyChain in spec. - Unit tests for the validation tooling (`tests/test_validate.py`), covering successful validation of minimal valid ProvenanceRecords and proper rejection diff --git a/context/0.0.1/sourcelume.jsonld b/context/0.0.1/sourcelume.jsonld index 5196b85..f9880e6 100644 --- a/context/0.0.1/sourcelume.jsonld +++ b/context/0.0.1/sourcelume.jsonld @@ -37,6 +37,10 @@ "@id": "schema:dateCreated", "@type": "xsd:dateTime" }, + "contentCreatedEnd": { + "@id": "sl:contentCreatedEnd", + "@type": "xsd:dateTime" + }, "origin": "sl:origin", "custodyChain": "sl:custodyChain", "agent": { diff --git a/schema/0.0.1/sourcelume.schema.json b/schema/0.0.1/sourcelume.schema.json index 3935b63..2436313 100644 --- a/schema/0.0.1/sourcelume.schema.json +++ b/schema/0.0.1/sourcelume.schema.json @@ -124,7 +124,12 @@ "contentCreated": { "type": "string", "format": "date-time", - "description": "When the dataset's underlying documents/content were originally created (e.g. the historical date range of the texts). Mirrors the `created` field commonly found in dataset datasheets. Use the start of the range if a range applies." + "description": "When the dataset's underlying documents/content were originally created (e.g. the historical date range of the texts). Mirrors the `created` field commonly found in dataset datasheets. If the content was created over a period, this is the start of the period and `contentCreatedEnd` is its end." + }, + "contentCreatedEnd": { + "type": "string", + "format": "date-time", + "description": "Optional. End of the period in which the dataset's underlying content was created, for content created over a period rather than at one point in time. `contentCreated` is the start of that period. Omit for content created at a single point in time. Must not be earlier than `contentCreated` (checked by the SHACL shapes)." }, "origin": { "type": "string", diff --git a/schema/0.0.1/sourcelume.shacl.ttl b/schema/0.0.1/sourcelume.shacl.ttl index 2f62836..1f3f801 100644 --- a/schema/0.0.1/sourcelume.shacl.ttl +++ b/schema/0.0.1/sourcelume.shacl.ttl @@ -75,6 +75,12 @@ sl:ProvenanceRecordShape sh:minCount 1 ; sh:maxCount 1 ; sh:datatype xsd:dateTime ; + sh:lessThanOrEquals sl:contentCreatedEnd ; + ] ; + sh:property [ + sh:path sl:contentCreatedEnd ; + sh:maxCount 1 ; + sh:datatype xsd:dateTime ; ] ; sh:property [ sh:path sl:origin ; diff --git a/spec/0.0.1/index.md b/spec/0.0.1/index.md index 4141bd4..19a2a06 100644 --- a/spec/0.0.1/index.md +++ b/spec/0.0.1/index.md @@ -58,7 +58,8 @@ stable `https://sourcelume.apache.org/ns#`, so term IRIs stay the same across ve | `creator` | yes | array (min 1) | One or more parties involved in producing this dataset — each a `schema:Organization` or `schema:Person` with a `name`, and an optional `role` (`originator`, `curator`, or `distributor`) to distinguish who made the underlying data from who aggregated or redistributed it. Omit `role` for a single, undifferentiated creator. | | `created` | yes | xsd:dateTime | When this provenance record was authored (record metadata, not content provenance). Distinct from `added` and `contentCreated` below. | | `added` | yes | xsd:dateTime | When the dataset was added to the collection being described (e.g. when a source was incorporated into an aggregator corpus). Mirrors the `added` field commonly found in dataset datasheets. | -| `contentCreated` | yes | xsd:dateTime | When the dataset's underlying documents/content were originally created (e.g. the historical date of the texts). Mirrors the `created` field commonly found in dataset datasheets; use the start of the range if a range applies. | +| `contentCreated` | yes | xsd:dateTime | When the dataset's underlying documents/content were originally created (e.g. the historical date of the texts). Mirrors the `created` field commonly found in dataset datasheets. For content created over a period this is the start of the period; see `contentCreatedEnd`. | +| `contentCreatedEnd` | no | xsd:dateTime | End of the period in which the underlying content was created. Omit for content created at a single point in time. Must not be earlier than `contentCreated`. | | `origin` | yes | string | Free-text description of where the dataset's underlying data came from. | | `custodyChain` | yes | array (min 1) | Ordered chain of custody events, earliest first. Each entry: `agent` (IRI), `action` (string, e.g. `"collected"`, `"ingested"`, `"transformed"`), `startTime` (xsd:dateTime). A single-hop dataset has exactly one entry. | diff --git a/tests/test_validate.py b/tests/test_validate.py index ff87921..f6d1efc 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -102,6 +102,55 @@ def test_foreign_context_iri_is_rejected(self) -> None: errors = [e for e in validator.iter_errors(record) if list(e.path) == ["@context"]] self.assertEqual(len(errors), 1) + def _record_with_content_period(self, start: str, end: str | None) -> dict: + record = { + "@context": validate.CANONICAL_CONTEXT_IRI, + "id": "https://example.org/records/test", + "type": "ProvenanceRecord", + "identifier": "https://example.org/datasets/test", + "name": "Test Dataset", + "license": "https://creativecommons.org/publicdomain/zero/1.0/", + "creator": [{"type": "schema:Organization", "name": "Test Org", "role": "originator"}], + "created": "2024-01-01T00:00:00Z", + "added": "2024-01-01T00:00:00Z", + "contentCreated": start, + "origin": "https://example.org/source", + "custodyChain": [ + { + "agent": "https://example.org/agents/test-org", + "action": "created", + "startTime": "2024-01-01T00:00:00Z", + } + ], + } + if end is not None: + record["contentCreatedEnd"] = end + return record + + def _validate_record(self, record: dict) -> bool: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "record.jsonld" + path.write_text(json.dumps(record), encoding="utf-8") + return validate.validate_example( + path, validate.load_validator(), validate.SCHEMA_DIR / "sourcelume.shacl.ttl" + ) + + def test_content_created_period_is_accepted(self) -> None: + record = self._record_with_content_period("1850-01-01T00:00:00Z", "1900-12-31T00:00:00Z") + self.assertTrue(self._validate_record(record)) + + def test_content_created_end_is_optional(self) -> None: + record = self._record_with_content_period("1850-01-01T00:00:00Z", None) + self.assertTrue(self._validate_record(record)) + + def test_content_created_end_before_start_is_rejected(self) -> None: + record = self._record_with_content_period("1900-12-31T00:00:00Z", "1850-01-01T00:00:00Z") + self.assertFalse(self._validate_record(record)) + + def test_content_created_end_must_be_a_date_time(self) -> None: + record = self._record_with_content_period("1850-01-01T00:00:00Z", "not a date") + self.assertFalse(self._validate_record(record)) + def test_validate_example_with_invalid_json_schema(self) -> None: """Test that an invalid record fails JSON Schema validation.""" with tempfile.TemporaryDirectory() as tmp: From 0855fe702cf3d3fe943617274b1b1bcfcca92bcb Mon Sep 17 00:00:00 2001 From: anierbeck Date: Sun, 11 Oct 2026 18:38:17 +0200 Subject: [PATCH 2/2] Require @context and assert the record type survives expansion Review findings on the canonical context change: - @context is now a required property. The const only applies when the key is present, and tools/validate.py injects the local context, so a record without @context used to pass the command line check although other JSON-LD processors cannot expand it. - The valid-record test asserts that the expanded graph contains rdf:type sl:ProvenanceRecord. Without a focus node the SHACL shapes pass vacuously; the assertion fails if the namespace and the shapes drift apart. - The context section no longer claims the file is already served at the canonical address; the release process publishes it. Apache-ai: Yes Generated-by: pi (claude-sonnet-5-5) Reviewed-by: anierbeck --- schema/0.0.1/sourcelume.schema.json | 1 + spec/0.0.1/index.md | 2 +- tests/test_validate.py | 17 +++++++++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/schema/0.0.1/sourcelume.schema.json b/schema/0.0.1/sourcelume.schema.json index 2436313..14031ac 100644 --- a/schema/0.0.1/sourcelume.schema.json +++ b/schema/0.0.1/sourcelume.schema.json @@ -7,6 +7,7 @@ "required": [ "id", "type", + "@context", "identifier", "name", "license", diff --git a/spec/0.0.1/index.md b/spec/0.0.1/index.md index 19a2a06..c4ecbc2 100644 --- a/spec/0.0.1/index.md +++ b/spec/0.0.1/index.md @@ -37,7 +37,7 @@ claim's accuracy — it gives the claim a consistent shape so it *can* be verifi Records reference the context with the canonical IRI `https://sourcelume.apache.org/context/0.0.1/sourcelume.jsonld` and no other `@context` value. -The context is published at that address and is not modified after release. Its namespace is the +The context is published at that address by the release process and is not modified after release. Its namespace is the stable `https://sourcelume.apache.org/ns#`, so term IRIs stay the same across versions. Inline `@context` objects and other context IRIs are rejected. diff --git a/tests/test_validate.py b/tests/test_validate.py index f6d1efc..9279943 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -23,6 +23,8 @@ import unittest from pathlib import Path +from rdflib import RDF, Graph, URIRef + # Add tools directory to path so we can import validate REPO_ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(REPO_ROOT / "tools")) @@ -74,10 +76,25 @@ def test_validate_example_with_valid_record(self) -> None: validator = validate.load_validator() shapes_path = validate.SCHEMA_DIR / "sourcelume.shacl.ttl" + # The expanded graph must carry the ProvenanceRecord type, otherwise the + # SHACL shapes have no focus node and the run below would pass vacuously. + context = json.loads((REPO_ROOT / "context" / "0.0.1" / "sourcelume.jsonld").read_text())["@context"] + graph = Graph() + graph.parse(data=json.dumps({**valid_record, "@context": context}), format="json-ld") + expected = URIRef("https://sourcelume.apache.org/ns#ProvenanceRecord") + self.assertIn((URIRef(valid_record["id"]), RDF.type, expected), graph) + # This should return True for a valid record result = validate.validate_example(example_path, validator, shapes_path) self.assertTrue(result) + def test_missing_context_is_required(self) -> None: + """A record without @context fails: other processors cannot expand it.""" + record = self._record_with_content_period("2024-01-01T00:00:00Z", None) + del record["@context"] + errors = list(validate.load_validator().iter_errors(record)) + self.assertEqual([e.message for e in errors], ["'@context' is a required property"]) + def test_schema_pins_the_canonical_context_iri(self) -> None: """The schema's @context const and the validator's constant must agree.""" schema = validate.load_validator().schema