diff --git a/CHANGELOG.md b/CHANGELOG.md index 1eb5288..db0f83b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,9 @@ described in `VERSIONING.md`. ### Added +- Optional `contentCreatedEnd`: end of the period in which a dataset's content was created. + `contentCreated` stays the start of the period, so existing records remain valid. The SHACL + shapes reject an end earlier than the start. - Added intended usage for custodyChain in spec. - Unit tests for the validation tooling (`tests/test_validate.py`), covering successful validation of minimal valid ProvenanceRecords and proper rejection diff --git a/context/0.0.1/sourcelume.jsonld b/context/0.0.1/sourcelume.jsonld index 5196b85..f9880e6 100644 --- a/context/0.0.1/sourcelume.jsonld +++ b/context/0.0.1/sourcelume.jsonld @@ -37,6 +37,10 @@ "@id": "schema:dateCreated", "@type": "xsd:dateTime" }, + "contentCreatedEnd": { + "@id": "sl:contentCreatedEnd", + "@type": "xsd:dateTime" + }, "origin": "sl:origin", "custodyChain": "sl:custodyChain", "agent": { diff --git a/schema/0.0.1/sourcelume.schema.json b/schema/0.0.1/sourcelume.schema.json index 3935b63..14031ac 100644 --- a/schema/0.0.1/sourcelume.schema.json +++ b/schema/0.0.1/sourcelume.schema.json @@ -7,6 +7,7 @@ "required": [ "id", "type", + "@context", "identifier", "name", "license", @@ -124,7 +125,12 @@ "contentCreated": { "type": "string", "format": "date-time", - "description": "When the dataset's underlying documents/content were originally created (e.g. the historical date range of the texts). Mirrors the `created` field commonly found in dataset datasheets. Use the start of the range if a range applies." + "description": "When the dataset's underlying documents/content were originally created (e.g. the historical date range of the texts). Mirrors the `created` field commonly found in dataset datasheets. If the content was created over a period, this is the start of the period and `contentCreatedEnd` is its end." + }, + "contentCreatedEnd": { + "type": "string", + "format": "date-time", + "description": "Optional. End of the period in which the dataset's underlying content was created, for content created over a period rather than at one point in time. `contentCreated` is the start of that period. Omit for content created at a single point in time. Must not be earlier than `contentCreated` (checked by the SHACL shapes)." }, "origin": { "type": "string", diff --git a/schema/0.0.1/sourcelume.shacl.ttl b/schema/0.0.1/sourcelume.shacl.ttl index 2f62836..1f3f801 100644 --- a/schema/0.0.1/sourcelume.shacl.ttl +++ b/schema/0.0.1/sourcelume.shacl.ttl @@ -75,6 +75,12 @@ sl:ProvenanceRecordShape sh:minCount 1 ; sh:maxCount 1 ; sh:datatype xsd:dateTime ; + sh:lessThanOrEquals sl:contentCreatedEnd ; + ] ; + sh:property [ + sh:path sl:contentCreatedEnd ; + sh:maxCount 1 ; + sh:datatype xsd:dateTime ; ] ; sh:property [ sh:path sl:origin ; diff --git a/spec/0.0.1/index.md b/spec/0.0.1/index.md index 4141bd4..c4ecbc2 100644 --- a/spec/0.0.1/index.md +++ b/spec/0.0.1/index.md @@ -37,7 +37,7 @@ claim's accuracy — it gives the claim a consistent shape so it *can* be verifi Records reference the context with the canonical IRI `https://sourcelume.apache.org/context/0.0.1/sourcelume.jsonld` and no other `@context` value. -The context is published at that address and is not modified after release. Its namespace is the +The context is published at that address by the release process and is not modified after release. Its namespace is the stable `https://sourcelume.apache.org/ns#`, so term IRIs stay the same across versions. Inline `@context` objects and other context IRIs are rejected. @@ -58,7 +58,8 @@ stable `https://sourcelume.apache.org/ns#`, so term IRIs stay the same across ve | `creator` | yes | array (min 1) | One or more parties involved in producing this dataset — each a `schema:Organization` or `schema:Person` with a `name`, and an optional `role` (`originator`, `curator`, or `distributor`) to distinguish who made the underlying data from who aggregated or redistributed it. Omit `role` for a single, undifferentiated creator. | | `created` | yes | xsd:dateTime | When this provenance record was authored (record metadata, not content provenance). Distinct from `added` and `contentCreated` below. | | `added` | yes | xsd:dateTime | When the dataset was added to the collection being described (e.g. when a source was incorporated into an aggregator corpus). Mirrors the `added` field commonly found in dataset datasheets. | -| `contentCreated` | yes | xsd:dateTime | When the dataset's underlying documents/content were originally created (e.g. the historical date of the texts). Mirrors the `created` field commonly found in dataset datasheets; use the start of the range if a range applies. | +| `contentCreated` | yes | xsd:dateTime | When the dataset's underlying documents/content were originally created (e.g. the historical date of the texts). Mirrors the `created` field commonly found in dataset datasheets. For content created over a period this is the start of the period; see `contentCreatedEnd`. | +| `contentCreatedEnd` | no | xsd:dateTime | End of the period in which the underlying content was created. Omit for content created at a single point in time. Must not be earlier than `contentCreated`. | | `origin` | yes | string | Free-text description of where the dataset's underlying data came from. | | `custodyChain` | yes | array (min 1) | Ordered chain of custody events, earliest first. Each entry: `agent` (IRI), `action` (string, e.g. `"collected"`, `"ingested"`, `"transformed"`), `startTime` (xsd:dateTime). A single-hop dataset has exactly one entry. | diff --git a/tests/test_validate.py b/tests/test_validate.py index ff87921..9279943 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -23,6 +23,8 @@ import unittest from pathlib import Path +from rdflib import RDF, Graph, URIRef + # Add tools directory to path so we can import validate REPO_ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(REPO_ROOT / "tools")) @@ -74,10 +76,25 @@ def test_validate_example_with_valid_record(self) -> None: validator = validate.load_validator() shapes_path = validate.SCHEMA_DIR / "sourcelume.shacl.ttl" + # The expanded graph must carry the ProvenanceRecord type, otherwise the + # SHACL shapes have no focus node and the run below would pass vacuously. + context = json.loads((REPO_ROOT / "context" / "0.0.1" / "sourcelume.jsonld").read_text())["@context"] + graph = Graph() + graph.parse(data=json.dumps({**valid_record, "@context": context}), format="json-ld") + expected = URIRef("https://sourcelume.apache.org/ns#ProvenanceRecord") + self.assertIn((URIRef(valid_record["id"]), RDF.type, expected), graph) + # This should return True for a valid record result = validate.validate_example(example_path, validator, shapes_path) self.assertTrue(result) + def test_missing_context_is_required(self) -> None: + """A record without @context fails: other processors cannot expand it.""" + record = self._record_with_content_period("2024-01-01T00:00:00Z", None) + del record["@context"] + errors = list(validate.load_validator().iter_errors(record)) + self.assertEqual([e.message for e in errors], ["'@context' is a required property"]) + def test_schema_pins_the_canonical_context_iri(self) -> None: """The schema's @context const and the validator's constant must agree.""" schema = validate.load_validator().schema @@ -102,6 +119,55 @@ def test_foreign_context_iri_is_rejected(self) -> None: errors = [e for e in validator.iter_errors(record) if list(e.path) == ["@context"]] self.assertEqual(len(errors), 1) + def _record_with_content_period(self, start: str, end: str | None) -> dict: + record = { + "@context": validate.CANONICAL_CONTEXT_IRI, + "id": "https://example.org/records/test", + "type": "ProvenanceRecord", + "identifier": "https://example.org/datasets/test", + "name": "Test Dataset", + "license": "https://creativecommons.org/publicdomain/zero/1.0/", + "creator": [{"type": "schema:Organization", "name": "Test Org", "role": "originator"}], + "created": "2024-01-01T00:00:00Z", + "added": "2024-01-01T00:00:00Z", + "contentCreated": start, + "origin": "https://example.org/source", + "custodyChain": [ + { + "agent": "https://example.org/agents/test-org", + "action": "created", + "startTime": "2024-01-01T00:00:00Z", + } + ], + } + if end is not None: + record["contentCreatedEnd"] = end + return record + + def _validate_record(self, record: dict) -> bool: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "record.jsonld" + path.write_text(json.dumps(record), encoding="utf-8") + return validate.validate_example( + path, validate.load_validator(), validate.SCHEMA_DIR / "sourcelume.shacl.ttl" + ) + + def test_content_created_period_is_accepted(self) -> None: + record = self._record_with_content_period("1850-01-01T00:00:00Z", "1900-12-31T00:00:00Z") + self.assertTrue(self._validate_record(record)) + + def test_content_created_end_is_optional(self) -> None: + record = self._record_with_content_period("1850-01-01T00:00:00Z", None) + self.assertTrue(self._validate_record(record)) + + def test_content_created_end_before_start_is_rejected(self) -> None: + record = self._record_with_content_period("1900-12-31T00:00:00Z", "1850-01-01T00:00:00Z") + self.assertFalse(self._validate_record(record)) + + def test_content_created_end_must_be_a_date_time(self) -> None: + record = self._record_with_content_period("1850-01-01T00:00:00Z", "not a date") + self.assertFalse(self._validate_record(record)) + def test_validate_example_with_invalid_json_schema(self) -> None: """Test that an invalid record fails JSON Schema validation.""" with tempfile.TemporaryDirectory() as tmp: