Add contentCreatedEnd and require @context (follow-up to #8)

CHANGELOG.md

@@ -19,6 +19,9 @@ described in `VERSIONING.md`.
 
 ### Added
 
+- Optional `contentCreatedEnd`: end of the period in which a dataset's content was created.
+  `contentCreated` stays the start of the period, so existing records remain valid. The SHACL
+  shapes reject an end earlier than the start.
 - Added intended usage for custodyChain in spec.
 - Unit tests for the validation tooling (`tests/test_validate.py`), covering
   successful validation of minimal valid ProvenanceRecords and proper rejection

context/0.0.1/sourcelume.jsonld

@@ -37,6 +37,10 @@
       "@id": "schema:dateCreated",
       "@type": "xsd:dateTime"
     },
+    "contentCreatedEnd": {
+      "@id": "sl:contentCreatedEnd",
+      "@type": "xsd:dateTime"
+    },
     "origin": "sl:origin",
     "custodyChain": "sl:custodyChain",
     "agent": {

schema/0.0.1/sourcelume.schema.json

@@ -7,6 +7,7 @@
   "required": [
     "id",
     "type",
+    "@context",
     "identifier",
     "name",
     "license",
@@ -124,7 +125,12 @@
     "contentCreated": {
       "type": "string",
       "format": "date-time",
-      "description": "When the dataset's underlying documents/content were originally created (e.g. the historical date range of the texts). Mirrors the `created` field commonly found in dataset datasheets. Use the start of the range if a range applies."
+      "description": "When the dataset's underlying documents/content were originally created (e.g. the historical date range of the texts). Mirrors the `created` field commonly found in dataset datasheets. If the content was created over a period, this is the start of the period and `contentCreatedEnd` is its end."
+    },
+    "contentCreatedEnd": {
+      "type": "string",
+      "format": "date-time",
+      "description": "Optional. End of the period in which the dataset's underlying content was created, for content created over a period rather than at one point in time. `contentCreated` is the start of that period. Omit for content created at a single point in time. Must not be earlier than `contentCreated` (checked by the SHACL shapes)."
     },
     "origin": {
       "type": "string",

schema/0.0.1/sourcelume.shacl.ttl

@@ -75,6 +75,12 @@ sl:ProvenanceRecordShape
         sh:minCount 1 ;
         sh:maxCount 1 ;
         sh:datatype xsd:dateTime ;
+        sh:lessThanOrEquals sl:contentCreatedEnd ;
+    ] ;
+    sh:property [
+        sh:path sl:contentCreatedEnd ;
+        sh:maxCount 1 ;
+        sh:datatype xsd:dateTime ;
     ] ;
     sh:property [
         sh:path sl:origin ;

spec/0.0.1/index.md

@@ -37,7 +37,7 @@ claim's accuracy — it gives the claim a consistent shape so it *can* be verifi
 
 Records reference the context with the canonical IRI
 `https://sourcelume.apache.org/context/0.0.1/sourcelume.jsonld` and no other `@context` value.
-The context is published at that address and is not modified after release. Its namespace is the
+The context is published at that address by the release process and is not modified after release. Its namespace is the
 stable `https://sourcelume.apache.org/ns#`, so term IRIs stay the same across versions. Inline
 `@context` objects and other context IRIs are rejected.
 
@@ -58,7 +58,8 @@ stable `https://sourcelume.apache.org/ns#`, so term IRIs stay the same across ve
 | `creator` | yes | array (min 1) | One or more parties involved in producing this dataset — each a `schema:Organization` or `schema:Person` with a `name`, and an optional `role` (`originator`, `curator`, or `distributor`) to distinguish who made the underlying data from who aggregated or redistributed it. Omit `role` for a single, undifferentiated creator. |
 | `created` | yes | xsd:dateTime | When this provenance record was authored (record metadata, not content provenance). Distinct from `added` and `contentCreated` below. |
 | `added` | yes | xsd:dateTime | When the dataset was added to the collection being described (e.g. when a source was incorporated into an aggregator corpus). Mirrors the `added` field commonly found in dataset datasheets. |
-| `contentCreated` | yes | xsd:dateTime | When the dataset's underlying documents/content were originally created (e.g. the historical date of the texts). Mirrors the `created` field commonly found in dataset datasheets; use the start of the range if a range applies. |
+| `contentCreated` | yes | xsd:dateTime | When the dataset's underlying documents/content were originally created (e.g. the historical date of the texts). Mirrors the `created` field commonly found in dataset datasheets. For content created over a period this is the start of the period; see `contentCreatedEnd`. |
+| `contentCreatedEnd` | no | xsd:dateTime | End of the period in which the underlying content was created. Omit for content created at a single point in time. Must not be earlier than `contentCreated`. |
 | `origin` | yes | string | Free-text description of where the dataset's underlying data came from. |
 | `custodyChain` | yes | array (min 1) | Ordered chain of custody events, earliest first. Each entry: `agent` (IRI), `action` (string, e.g. `"collected"`, `"ingested"`, `"transformed"`), `startTime` (xsd:dateTime). A single-hop dataset has exactly one entry. |
 

tests/test_validate.py

@@ -23,6 +23,8 @@
 import unittest
 from pathlib import Path
 
+from rdflib import RDF, Graph, URIRef
+
 # Add tools directory to path so we can import validate
 REPO_ROOT = Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(REPO_ROOT / "tools"))
@@ -74,10 +76,25 @@ def test_validate_example_with_valid_record(self) -> None:
             validator = validate.load_validator()
             shapes_path = validate.SCHEMA_DIR / "sourcelume.shacl.ttl"
 
+            # The expanded graph must carry the ProvenanceRecord type, otherwise the
+            # SHACL shapes have no focus node and the run below would pass vacuously.
+            context = json.loads((REPO_ROOT / "context" / "0.0.1" / "sourcelume.jsonld").read_text())["@context"]
+            graph = Graph()
+            graph.parse(data=json.dumps({**valid_record, "@context": context}), format="json-ld")
+            expected = URIRef("https://sourcelume.apache.org/ns#ProvenanceRecord")
+            self.assertIn((URIRef(valid_record["id"]), RDF.type, expected), graph)
+
             # This should return True for a valid record
             result = validate.validate_example(example_path, validator, shapes_path)
             self.assertTrue(result)
 
+    def test_missing_context_is_required(self) -> None:
+        """A record without @context fails: other processors cannot expand it."""
+        record = self._record_with_content_period("2024-01-01T00:00:00Z", None)
+        del record["@context"]
+        errors = list(validate.load_validator().iter_errors(record))
+        self.assertEqual([e.message for e in errors], ["'@context' is a required property"])
+
     def test_schema_pins_the_canonical_context_iri(self) -> None:
         """The schema's @context const and the validator's constant must agree."""
         schema = validate.load_validator().schema
@@ -102,6 +119,55 @@ def test_foreign_context_iri_is_rejected(self) -> None:
         errors = [e for e in validator.iter_errors(record) if list(e.path) == ["@context"]]
         self.assertEqual(len(errors), 1)
 
+    def _record_with_content_period(self, start: str, end: str | None) -> dict:
+        record = {
+            "@context": validate.CANONICAL_CONTEXT_IRI,
+            "id": "https://example.org/records/test",
+            "type": "ProvenanceRecord",
+            "identifier": "https://example.org/datasets/test",
+            "name": "Test Dataset",
+            "license": "https://creativecommons.org/publicdomain/zero/1.0/",
+            "creator": [{"type": "schema:Organization", "name": "Test Org", "role": "originator"}],
+            "created": "2024-01-01T00:00:00Z",
+            "added": "2024-01-01T00:00:00Z",
+            "contentCreated": start,
+            "origin": "https://example.org/source",
+            "custodyChain": [
+                {
+                    "agent": "https://example.org/agents/test-org",
+                    "action": "created",
+                    "startTime": "2024-01-01T00:00:00Z",
+                }
+            ],
+        }
+        if end is not None:
+            record["contentCreatedEnd"] = end
+        return record
+
+    def _validate_record(self, record: dict) -> bool:
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp) / "record.jsonld"
+            path.write_text(json.dumps(record), encoding="utf-8")
+            return validate.validate_example(
+                path, validate.load_validator(), validate.SCHEMA_DIR / "sourcelume.shacl.ttl"
+            )
+
+    def test_content_created_period_is_accepted(self) -> None:
+        record = self._record_with_content_period("1850-01-01T00:00:00Z", "1900-12-31T00:00:00Z")
+        self.assertTrue(self._validate_record(record))
+
+    def test_content_created_end_is_optional(self) -> None:
+        record = self._record_with_content_period("1850-01-01T00:00:00Z", None)
+        self.assertTrue(self._validate_record(record))
+
+    def test_content_created_end_before_start_is_rejected(self) -> None:
+        record = self._record_with_content_period("1900-12-31T00:00:00Z", "1850-01-01T00:00:00Z")
+        self.assertFalse(self._validate_record(record))
+
+    def test_content_created_end_must_be_a_date_time(self) -> None:
+        record = self._record_with_content_period("1850-01-01T00:00:00Z", "not a date")
+        self.assertFalse(self._validate_record(record))
+
     def test_validate_example_with_invalid_json_schema(self) -> None:
         """Test that an invalid record fails JSON Schema validation."""
         with tempfile.TemporaryDirectory() as tmp: