diff --git a/.castor/earl.php b/.castor/earl.php index 2bea991e..35b6be83 100644 --- a/.castor/earl.php +++ b/.castor/earl.php @@ -258,28 +258,29 @@ function earlTurtle(array $assertions, string $release, \DateTimeImmutable $date '@prefix foaf: .', '@prefix xsd: .', '', + // Within each node, the rdf:type ("a") comes first, then the properties in + // alphabetical order. \sprintf('<%s> a doap:Project, earl:TestSubject, earl:Software ;', EARL_SUBJECT_IRI), - ' doap:name "jolicode/structured-data" ;', + \sprintf(' dc:creator <%s> ;', EARL_ASSERTOR_IRI), ' dc:title "jolicode/structured-data" ;', ' doap:description "A PHP toolkit to validate and convert structured data: Schema.org and Google Rich Results validation, microdata/RDFa/JSON-LD extraction, and the W3C JSON-LD 1.1 algorithms."@en ;', + \sprintf(' doap:developer <%s> ;', EARL_ASSERTOR_IRI), \sprintf(' doap:homepage <%s> ;', EARL_SUBJECT_IRI), + ' doap:implements , , ;', \sprintf(' doap:license <%s/blob/main/LICENSE> ;', EARL_SUBJECT_IRI), + ' doap:name "jolicode/structured-data" ;', ' doap:programming-language "PHP" ;', - \sprintf(' doap:developer <%s>, ;', EARL_ASSERTOR_IRI), - \sprintf(' dc:creator <%s> ;', EARL_ASSERTOR_IRI), ' doap:release [', + \sprintf(' doap:created "%s"^^xsd:date ;', $date->format('Y-m-d')), \sprintf(' doap:name "jolicode/structured-data %s" ;', $release), - \sprintf(' doap:revision "%s" ;', $release), - \sprintf(' doap:created "%s"^^xsd:date', $date->format('Y-m-d')), - ' ] .', + \sprintf(' doap:revision "%s"', $release), + ' ] ;', + ' doap:shortdesc "JSON-LD 1.1 processor and structured data validator for PHP"@en .', '', \sprintf('<%s> a foaf:Person, earl:Assertor ;', EARL_ASSERTOR_IRI), - ' foaf:name "Xavier Lacot" ;', - \sprintf(' foaf:homepage <%s> .', EARL_ASSERTOR_IRI), - '', - ' a foaf:Person ;', - ' foaf:name "Hédic Guibert" ;', - ' foaf:homepage .', + ' foaf:homepage ;', + \sprintf(' foaf:made <%s> ;', EARL_ASSERTOR_IRI), + ' foaf:name "Xavier Lacot" .', '', ]; @@ -288,10 +289,10 @@ function earlTurtle(array $assertions, string $release, \DateTimeImmutable $date foreach ($assertions as $assertion) { $lines[] = '[ a earl:Assertion ;'; $lines[] = \sprintf(' earl:assertedBy <%s> ;', EARL_ASSERTOR_IRI); + $lines[] = ' earl:mode earl:automatic ;'; + $lines[] = \sprintf(' earl:result [ a earl:TestResult ; dc:date "%s"^^xsd:dateTime ; earl:outcome earl:%s ] ;', $timestamp, $assertion['outcome']); $lines[] = \sprintf(' earl:subject <%s> ;', EARL_SUBJECT_IRI); - $lines[] = \sprintf(' earl:test <%s> ;', $assertion['test']); - $lines[] = \sprintf(' earl:result [ a earl:TestResult ; earl:outcome earl:%s ; dc:date "%s"^^xsd:dateTime ] ;', $assertion['outcome'], $timestamp); - $lines[] = ' earl:mode earl:automatic ] .'; + $lines[] = \sprintf(' earl:test <%s> ] .', $assertion['test']); $lines[] = ''; } diff --git a/.castor/qa.php b/.castor/qa.php index 2b439908..04a60575 100644 --- a/.castor/qa.php +++ b/.castor/qa.php @@ -54,7 +54,7 @@ // upstream change cannot break an unrelated PR. The weekly scheduled CI run is what // surfaces upstream drift: bump this SHA (and rerun `castor qa:phpunit:prepare --force`) // to pick up new W3C tests. -const W3C_TEST_SUITE_REF = '92f07705a0c0ac27aa9bc6fe1322dcc9fad0114d'; +const W3C_TEST_SUITE_REF = '61305e9c7b7d3d79178baefdb34bc1d47464c783'; // Commit of https://github.com/w3c/json-ld-framing the framing test suite is pinned to. const W3C_FRAMING_TEST_SUITE_REF = '3bf782ba9a40dd1b143435abe386d38df64f2b47'; diff --git a/.github/SECURITY.md b/.github/SECURITY.md index a5c0d1a6..d961590a 100644 --- a/.github/SECURITY.md +++ b/.github/SECURITY.md @@ -37,7 +37,7 @@ job of a validator is to inspect input you did not write). The following are in we take them seriously: - Server-Side Request Forgery through remote `@context` resolution. Remote loading is - **off by default**; see the "Loading remote contexts" section of the [README](../README.md). + **off by default**; see [Loading remote contexts](../docs/remote-contexts.md). - Denial of service through hostile documents (deeply nested input, oversized responses, context recursion). - Local file or stream-wrapper access through crafted URLs. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 43d69b7b..d0267c10 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,8 +11,10 @@ Everybody should be able to help. Here's how you can do it: Here's some tips to make you the best contributor ever: * [Rules](#rules) +* [Setting up](#setting-up) * [Green tests](#green-tests) * [Standard code](#standard-code) +* [Commands reference](#commands-reference) * [Keeping your fork up-to-date](#keeping-your-fork-up-to-date) ## Rules @@ -47,6 +49,21 @@ which gives the context and/or explains why you are creating it. Your work will then be reviewed as soon as possible (suggestions about some changes, improvements or alternatives may be given). +## Setting up + +The tooling runs with the PHP task runner [Castor](https://github.com/jolicode/castor/). +Downloading the W3C test suites also needs the +[ZipArchive PHP extension](https://www.php.net/manual/en/class.ziparchive.php). + +Clone the repository, then install the library's dependencies and the QA tooling: + +```shell +composer install # the library's own dependencies (Composer, not Castor) +castor qa:install # the QA tooling: php-cs-fixer, phpstan, phpunit, phpbench, infection +``` + +All the tasks are defined in the `.castor` directory, one file per namespace. + ## Green tests Run the tests using the following command: @@ -68,10 +85,12 @@ This project includes two main validators: the schema.org validator and the Goog ### Updating validators -When adding or updating validator support (new types, schema versions, special validation rules): +The validation classes shipped in `src/Vocabularies/Generated` are generated from the +vocabulary definitions (see [Vocabulary generation](#vocabulary-generation)). When adding or +updating validator support (new types, schema versions, special validation rules): -- **Google validator** updates are based on the [Google structured-data documentation](https://developers.google.com/search/docs/advanced/structured-data/intro-structured-data). For special rules or complex validation logic, see the existing rule implementations in `src/Vocabularies/Validators/Google/SpecialRules/`. -- **Schema.org validator** updates follow [official schema.org releases](https://schema.org/docs/releases.html). +- **Google validator** updates track the [Google structured-data documentation](https://developers.google.com/search/docs/appearance/structured-data/intro-structured-data), which evolves continuously. For special rules or complex validation logic, see the existing rule implementations in `src/Vocabularies/Validators/Google/SpecialRules/`. +- **Schema.org validator** updates are driven by the [official schema.org releases](https://schema.org/docs/releases.html). When upgrading, always review the release notes and check that the changes to the `schema-org-baseline.json` test fixture reflect real schema.org changes. For comprehensive upgrade procedures for either validator, refer to: - [Google validator upgrade guidelines](resources/google/UPGRADE_GUIDELINES.md) @@ -86,6 +105,71 @@ our coding standards: castor qa:cs ``` +## Commands reference + +### QA + +Command | Description +---- | ----- +`castor qa:install` | Installs the QA tooling +`castor qa:update` | Updates the QA tooling +`castor qa:cs` | Fixes coding standards +`castor qa:phpstan` | Runs PHPStan +`castor qa:all` | Runs all QA tasks +`castor qa:earl` | Runs the W3C test suites and writes the EARL conformance report (see [Conformance](docs/json-ld.md#conformance)) + +### Tests + +Command | Description +---- | ----- +`castor qa:phpunit:prepare` | Downloads the W3C tests suite +`castor qa:phpunit:run` | Runs PHPUnit +`castor qa:phpunit:coverage` | Runs PHPUnit with code coverage (requires the pcov or xdebug extension) +`castor qa:infection` | Runs Infection mutation testing on the validator and mapper layers (requires the pcov or xdebug extension) + +The W3C test suite is pinned to a known-good upstream commit (see `W3C_TEST_SUITE_REF` in `.castor/qa.php`). +To re-download it, or to test against the upstream main branch: + +```shell +castor qa:phpunit:prepare --force +castor qa:phpunit:prepare --force --ref main +``` + +### Benchmarks + +Command | Description +---- | ----- +`castor qa:bench:all` | Runs all the benchmarks +`castor qa:bench:algorithms` | Runs the JSON-LD manipulation algorithms benchmark +`castor qa:bench:validators` | Runs the validators benchmark +`castor qa:bench:validators -d` | Runs the detailed and slow validators benchmark + +The latest figures, and what each of them measures, are in [BENCHMARKS.md](BENCHMARKS.md). + +### Demo website + +These commands build the [demo website](website/README.md), that runs the validator in the browser: + +Command | Description +---- | ----- +`castor website:build` | Builds the demo website: its assets, and PHP as WebAssembly with the validator +`castor website:check` | Checks, in node, that what is built boots, and audits a document +`castor website:serve` | Serves the demo website + +### Vocabulary generation + +These commands refresh the validation classes shipped in `src/Vocabularies/Generated`: + +Command | Description +---- | ----- +`castor schema-org:update-version` | Bump the schema.org version to the latest release published on GitHub +`castor schema-org:download` | Download the schema.org types definition file and JSON-LD context +`castor schema-org:generate` | Generate the schema.org validation classes +`castor schema-org:download-examples` | Refresh the schema.org examples used by the tests +`castor google:download` | Crawl the Google structured-data documentation +`castor google:generate` | Generate the Google validation classes +`castor google:check` | Check the Google documentation coverage against the curated manifest + ## Keeping your fork up-to-date To keep your fork up-to-date, you should track the upstream (original) one diff --git a/README.md b/README.md index ce61189c..9d076642 100644 --- a/README.md +++ b/README.md @@ -9,591 +9,115 @@
[![CI](https://github.com/jolicode/structured-data/actions/workflows/ci.yml/badge.svg)](https://github.com/jolicode/structured-data/actions/workflows/ci.yml) -[![W3C JSON-LD 1.1 test suites](https://img.shields.io/badge/W3C_JSON--LD_1.1-test_suites_passing-brightgreen)](#conformance) +[![W3C JSON-LD 1.1 test suites](https://img.shields.io/badge/W3C_JSON--LD_1.1-test_suites_passing-brightgreen)](docs/json-ld.md#conformance) [![PHPStan level 9](https://img.shields.io/badge/PHPStan-level_9-brightgreen)](phpstan.neon) [![PHP Version Require](http://poser.pugx.org/jolicode/structured-data/require/php)](https://packagist.org/packages/jolicode/structured-data) [![License](http://poser.pugx.org/jolicode/structured-data/license)](LICENSE)
-This library provides several tools to work with [JSON-LD](https://json-ld.org/) and [schema.org](https://schema.org/) in PHP. -It includes: -- an implementation of the W3C JSON-LD algorithms described in the [JSON-LD 1.1 Processing Algorithms and API Recommendation](https://www.w3.org/TR/json-ld-api), published on July 16, 2020; -- a schema.org validator; -- a Google validator, which tells you whether your structured data is eligible for [Google Rich Results](https://developers.google.com/search/docs/appearance/structured-data/intro-structured-data); -- an extractor for JSON-LD, microdata and RDFa embedded in HTML documents. +Validate and process structured data in PHP: check that your [schema.org](https://schema.org/) +markup is valid and eligible for +[Google Rich Results](https://developers.google.com/search/docs/appearance/structured-data/intro-structured-data), +from JSON-LD, microdata or RDFa, without leaving your process. It also ships an +implementation of the W3C JSON-LD 1.1 algorithms that passes the official test suites. +The [W3C implementation report](https://w3c.github.io/json-ld-api/reports/) +lists no other PHP processor. -It is used in production by [redirection.io](https://redirection.io/), which audits the -structured data of the pages it crawls with it. +**[Try it in your browser](https://structured-data-validator.jolicode.com/)**, or see it at +work in [redirection.io](https://redirection.io/), which audits the structured data of +every page it crawls with it. ## Why this library -Structured data is usually validated in a browser, with Google's -[Rich Results Test](https://search.google.com/test/rich-results) or -[validator.schema.org](https://validator.schema.org/), or not at all. Neither tool has an -API, so nothing checks the template you just changed, the pages you deploy, or the -millions of pages a crawler visits. This library brings that check into PHP: - -- **Google Rich Results eligibility, offline.** The Google validator reproduces the - required and recommended properties of every type the Google documentation describes, - including the rules the documentation only states in prose (a `price` needs a - `priceCurrency`, a finite return window needs `merchantReturnDays`...), and applies - them the way Google does: to the schema.org subtypes of the documented types, and to - every entity of the page, however deeply it is nested. No request leaves your process. -- **Exact JSON-LD 1.1.** The four W3C algorithms pass the official test suites (see - [Conformance](#conformance)), so a document is validated on what it means rather than - on how it was written: compact IRIs, nested and scoped contexts, `@graph`, aliases and - `@reverse` are all resolved before validation. -- **One pipeline for JSON-LD, microdata and RDFa.** Everything found in the HTML is - extracted, expanded and validated the same way, and a page mixing valid and broken - blocks still yields what can be used, with a warning for the rest. -- **Safe on pages you do not control.** No network request and no file read unless you - opt in with an allow-list (see [Loading remote contexts](#loading-remote-contexts)). - Hostile nesting depths and malformed documents are refused instead of crashing the - process. -- **Built for workers.** One `Validator` audits a stream of documents with bounded - memory and results that do not depend on what was audited before. A full audit of a - heavy JSON-LD document takes about 13 ms (see [BENCHMARKS.md](BENCHMARKS.md)). -- **Diagnostics you can act on.** Every message carries its severity, its path in the - document, its line range, and a link to the Google documentation it comes from. -- **Vocabularies you can audit.** The schema.org classes are generated from the official - release file, the Google rules from a crawled and curated copy of the documentation, and - CI checks that regeneration is reproducible. The W3C suites, the Google documentation - crawl and the mutation tests are re-run every week to catch drift. - -How it compares with what you would use otherwise to *validate* structured data: - -| | [Rich Results Test](https://search.google.com/test/rich-results) | [Schema Markup Validator](https://validator.schema.org/) | jolicode/structured-data | -| --- | --- | --- | --- | -| Runs | in a browser, no API | in a browser, no API | in PHP | -| Input | a public URL, or pasted code | a public URL, or pasted code | an HTML page (JSON-LD, microdata, RDFa) or a JSON-LD document | -| JSON-LD | fully processed | fully processed | expanded with the W3C JSON-LD 1.1 algorithms before validation | -| Google rules | the reference | none: checks the schema.org vocabulary only | 36 types generated from a crawl of the documentation, drift-checked weekly | -| Validation issues location | highlighted in the tool | highlighted in the tool | a path in the document, and the position of the block in the HTML | -| Remote `@context` | fetched by Google | fetched by Google | can be loaded. By default, denied unless allow-listed | -| License | - | - | MIT | - -The other PHP libraries in this space do a different job, and combine well with this one. -[spatie/schema-org](https://github.com/spatie/schema-org) writes JSON-LD; this library -checks what it wrote. [brick/schema](https://github.com/brick/schema) and -[crwlrsoft/schema-org](https://github.com/crwlrsoft/schema-org) read structured data from -HTML into PHP objects and accept whatever they find; this library tells you what is wrong with -it. [ml/json-ld](https://github.com/lanthaler/JsonLD) and -[digitalbazaar/php-json-ld](https://github.com/digitalbazaar/php-json-ld) are JSON-LD -processors of the 1.0 era, which load remote contexts by default; neither appears in the -[W3C JSON-LD 1.1 implementation report](https://w3c.github.io/json-ld-api/reports/), -which lists no PHP processor at all so far. +- **Google Rich Results checks, offline**: 36 types, including the rules Google only states in prose, applied to subtypes and nested entities. +- **Exact JSON-LD 1.1**: the W3C test suites pass, so documents are validated on what they mean, not on how they are written. +- **One pipeline** for JSON-LD, microdata and RDFa found in an HTML page. +- **Safe on untrusted pages**: no network request or file read unless you allow-list it, hostile input refused instead of crashing. +- **Built for workers**: bounded memory, about 13 ms for a heavy document ([benchmarks](BENCHMARKS.md)). +- **Actionable diagnostics**: severity, path in the document, line range, and a link to the Google documentation. +- **Auditable rules**: generated from the official schema.org release and the Google documentation, drift-checked weekly. -## Installation - -Install the library with [Composer](https://getcomposer.org/): +## How it compares -```bash -composer require jolicode/structured-data -``` +| Tool | What it does | Difference | +| --- | --- | --- | +| [Rich Results Test](https://search.google.com/test/rich-results) | Google eligibility check | Browser only, no API | +| [Schema Markup Validator](https://validator.schema.org/) | schema.org vocabulary check | Browser only, no Google rules | +| [spatie/schema-org](https://github.com/spatie/schema-org) | Writes JSON-LD | No validation: use this library to check its output | +| [brick/schema](https://github.com/brick/schema), [crwlrsoft/schema-org](https://github.com/crwlrsoft/schema-org) | Read structured data into PHP objects | Accept whatever they find, no validation | +| [ml/json-ld](https://github.com/lanthaler/JsonLD), [digitalbazaar/php-json-ld](https://github.com/digitalbazaar/php-json-ld) | JSON-LD 1.0 processors | 1.0 only, load remote contexts by default | -Remote `@context` resolution is **off by default**, and the default setup needs no extra -dependency. If you opt in (see [Loading remote contexts](#loading-remote-contexts)), -install an HTTP client as well: +## Installation ```bash -composer require symfony/http-client -``` - -## Dependencies - -This library requires: - -- PHP >= 8.4, with the `dom`, `json` and `libxml` extensions -- (optional, only for remote `@context` resolution) an implementation of `symfony/http-client-contracts`, such as [`symfony/http-client`](https://symfony.com/doc/current/http_client.html) -- (optional) the PHP task runner [Castor](https://github.com/jolicode/castor/), used for the tooling and the CLI interface. The development tooling additionally needs the [ZipArchive PHP extension](https://www.php.net/manual/en/class.ziparchive.php) to download the W3C test suites. - -## Validating a JSON-LD document - -To validate a JSON-LD document, use the `JoliCode\StructuredData\Validator` class. - -### Accepted inputs - -`audit()` takes **the document itself**, as a string - never a URL, never a file path. - -This library deliberately does not guess what a string is, and never fetches anything on -your behalf. Guessing is a security hazard: an application that forwards user input to a -validator would silently offer an attacker a way to reach its internal network -(`http://127.0.0.1:9200/`, cloud metadata endpoints), or to read local files through a -path or a stream wrapper (`/var/www/.env`, `file://`, `phar://`). - -Whether a document may be fetched, from where, and under which restrictions is a decision -only your application can make. So your application is the one that fetches: - -```php -// From a local file - the path comes from you, not from a user -$document = file_get_contents('/path/to/document.html'); - -// From the network - your HTTP client, your allow-list, your timeouts -$document = $httpClient->request('GET', $trustedUrl)->getContent(); - -$audit = $validator->audit($document); -``` - -The same rule applies to the `@context` URLs found *inside* a document: see -[Loading remote contexts](#loading-remote-contexts). - -The validator accepts the following data formats: -- JSON-LD -- microdata -- RDFa (schema.org-style RDFa only) - -### Blocks describing the same node - -Google reads a page as one graph: the blocks that share an identifier (a JSON-LD `@id`, -a microdata `itemid`) are one node holding the union of their properties, whatever their -format, and a property one block omits is not missing when another block provides it. - -By default, the validator does the same. The audit is made of items, one per node, listed -by `Audit::getItems()`: the blocks sharing an identifier make one item, the others one -item each. The diagnostics of the audit (`isValid()`, `getDiagnostic()`) are the ones of -the items, so an issue of a merged node is reported once, with the positions of every -block it comes from. - -```php -foreach ($audit->getItems() as $item) { - $item->getType(); // the node: merged from several blocks, or the block itself - $item->isMerged(); // true when the node comes from several blocks - $item->getIdentifier(); // its absolute "@id" or "itemid", if any - $item->getSources(); // the blocks it comes from -} +composer require jolicode/structured-data ``` -Each source block keeps its own audit, as if it were alone in the document: a block may -report a property another block provides, while the merged item, which makes the verdict, -does not. `Audit::getTypes()` still lists the blocks. +Requires PHP 8.4 or later, with the `dom`, `json` and `libxml` extensions. -When the blocks disagree on a value, Google keeps every value and reports nothing; the -validator raises a warning on the property of the merged node, with the `Merge` validator -name (`AuditOptions::VALIDATOR_MERGE`), outside the Google diagnostics. Blank node -identifiers are local to their block and never merge, and identifiers are compared as -written. Only the top-level node of each block is merged: nested nodes sharing an -identifier are not. +## Usage -Google is not the only consumer of structured data, though. To audit every block on its -own, with no merge, disable the option: - -```php -$audit = $validator->audit($document, mergeNodesSharingAnIdentifier: false); -``` - -Each block is then an item of its own. - -### Using the validator - -#### Basic usage - -The validator exposes a single validation method: `audit()`. -It returns a `JoliCode\StructuredData\Audit\Audit` object holding the validation result. - -To quickly check the result, use `isValid()` or `isFullyValid()`: -- `isValid()` returns true if no errors are detected -- `isFullyValid()` returns true if no errors, no warnings, and no malformed (hence unusable) data structures were detected - -**Keep in mind that a schema.org type is considered valid even with warnings!** - -To access the messages themselves, use the `getDiagnostic()` method, which will return an array of diagnostic messages (errors and warnings). - -A typical usage example looks like this: +### Validate a page or a JSON-LD document ```php use JoliCode\StructuredData\Validator; $validator = new Validator(); +$audit = $validator->audit($html); // the document itself, never a URL or a file path -$document = file_get_contents('/path/to/a-page.html'); -$audit = $validator->audit($document); - -if ($audit->isValid()) { - echo 'The document contains valid structured data!'; -} else { - echo 'The provided document contains invalid schema.org data!'; +$audit->isValid(); // no errors +$audit->isFullyValid(); // no errors, no warnings - // Returns an array of string diagnostic messages - $diagnostic = $audit->getDiagnostic(); - - foreach ($diagnostic as $message) { - // Messages look like this: - // [Google warning] DataFeed.dataFeedElement.workExample: Missing recommended property: "sameAs" for the type "Book" - // [Google error] DataFeed.dataFeedElement.workExample.potentialAction.expectsAcceptanceOf: Missing required property: "price" for the type "Offer" when "category" is "purchase" or "rental". - echo $message; - } +foreach ($audit->getDiagnostic() as $message) { + echo $message; + // [Google warning] DataFeed.dataFeedElement.workExample: Missing recommended property: "sameAs" for the type "Book" } ``` -If you are only interested in the results of a single validator, call `setValidator()` before auditing: -```php -use JoliCode\StructuredData\Validator; -use JoliCode\StructuredData\Vocabularies\Validators\Google\GoogleValidator; - -$validator = new Validator(); -$validator->setValidator(GoogleValidator::class); - -$validator->audit($document); -``` - -Terms hosted under [pending.schema.org](https://pending.schema.org) are still under -development and may change or be removed. Using them is legitimate, so the validator -accepts them silently by default; opt in to `$reportPendingVocabularyUsage` to get a -warning for each of these usages instead: +A few options: ```php -$audit = $validator->audit($document, reportPendingVocabularyUsage: true); -``` - -#### Advanced usage - -The `getDiagnostic()` method accepts an optional parameter: a `JoliCode\StructuredData\Audit\AuditOptions` object, which lets you filter or group the results, or change the return format. See the PHPDoc on `JoliCode\StructuredData\Audit\AuditOptions` for more details. - -Finally, if you want to access the full parsed PHP tree, use `getTypes()` and inspect the underlying `MappedType` objects directly. These are low-level objects, but they are the most detailed information you can get, and they preserve the type hierarchy of the document. -For an idea of what you can do with these objects, look at the output of the `castor validate` command. - -### Command line interface - -A command is available to quickly validate a JSON-LD document from the command line or in CI: `castor check`. -Use `castor validate` to get a nicely parsed and colored full audit (it can be pretty verbose!). -Both return a meaningful exit code, for use in scripts and CI. - -```bash -castor check -castor validate -``` - -Unlike the `Validator::audit()` API, these CLI commands **do** accept a file path or a -URL, and will read or fetch it for you. That is safe here precisely because the argument -comes from the operator running the command, not from a document being processed - exactly -the distinction drawn in [Accepted inputs](#accepted-inputs). - -You can validate using a specific validator: - -```bash -castor validate google -``` - -```bash -castor validate schema-org -``` - -The blocks sharing an identifier are merged, as Google does (see -[Blocks describing the same node](#blocks-describing-the-same-node)); pass `--no-merge` -to audit every block on its own: +use JoliCode\StructuredData\Vocabularies\Validators\Google\GoogleValidator; -```bash -castor validate --no-merge +$validator->setValidator(GoogleValidator::class); // run a single validator +$validator->audit($html, mergeNodesSharingAnIdentifier: false); // audit blocks sharing an @id separately +$validator->audit($html, reportPendingVocabularyUsage: true); // warn on pending.schema.org terms ``` -Sample result of the validate command: -Output of the castor validate command - - -## Using the JSON-LD algorithms - -The currently available algorithms are: +More in [the validator documentation](docs/validator.md): filtering diagnostics, merged +nodes, the parsed tree, and the CLI. -- [x] [Compaction](https://www.w3.org/TR/json-ld11-api/#compaction-algorithm) -- [x] [Expansion](https://www.w3.org/TR/json-ld11-api/#expansion-algorithm) -- [x] [Flattening](https://www.w3.org/TR/json-ld11-api/#flattening-algorithm) -- [x] [Framing](https://www.w3.org/TR/json-ld11-framing/#framing-algorithm) +### Use the JSON-LD algorithms -### Conformance - -Each algorithm is validated against the official W3C test suites ([json-ld-api](https://github.com/w3c/json-ld-api) and [json-ld-framing](https://github.com/w3c/json-ld-framing)), pinned to a known-good upstream commit and re-run weekly against `main` to surface drift. - -The full suites pass, covering expansion, compaction, flattening and framing. The only skipped fixtures are a handful that target JSON-LD 1.0-specific behavior (declared `specVersion: json-ld-1.0` in the upstream manifest), which this library does not implement; each skip is documented in the corresponding test. Serialization to and from RDF (`toRdf` / `fromRdf`) is out of scope and not implemented. - -As of the pinned upstream commits, the manifests list 781 tests, and every test that applies to a JSON-LD 1.1 processor passes: - -| Suite | Passed | JSON-LD 1.0 only | -| --- | --- | --- | -| expand | 385 | 0 | -| compact | 244 | 2 | -| flatten | 58 | 0 | -| frame | 91 | 1 | - -`castor qa:earl` runs these suites and writes an [EARL](https://www.w3.org/TR/EARL10-Schema/) report in Turtle, one assertion per manifest entry, in the format the [W3C JSON-LD implementation report](https://w3c.github.io/json-ld-api/reports/) consolidates. - -To use them, create an instance of `JoliCode\StructuredData\JsonLd\Algorithms\Expand\Expander`, `JoliCode\StructuredData\JsonLd\Algorithms\Flatten\Flattener`, `JoliCode\StructuredData\JsonLd\Algorithms\Compact\Compactor` or `JoliCode\StructuredData\JsonLd\Algorithms\Frame\Framer`, and pass it the JSON-LD document you want to convert. - -For instance, expanding a JSON-LD document looks like this: - -```php -use JoliCode\StructuredData\JsonLd\Algorithms\Expand\Expander; - -$jsonString = '{ - "@context": "https://schema.org", - "@type": "Person", - "name": "John Doe" -}'; - -$expander = new Expander(); -$result = $expander->expand($jsonString); -``` - -The result will be a JSON string containing the expanded JSON-LD document: - -```json -[ - { - "@type": [ - "http://schema.org/Person" - ], - "http://schema.org/name": [ - { - "@value": "John Doe" - } - ] - } -] -``` - -If you want a PHP value (an array or a `stdClass`) instead of a JSON string, set the `encodeResult` parameter to `false` when calling `expand()`. -You can also pass a `ProcessorOptions` object holding the [JSON-LD options](https://www.w3.org/TR/json-ld-api/#the-jsonldoptions-type) if you want to modify the default behavior of the algorithms: +Expansion, compaction, flattening and framing are available as `Expander`, `Compactor`, +`Flattener` and `Framer`: ```php use JoliCode\StructuredData\JsonLd\Algorithms\Expand\Expander; -use JoliCode\StructuredData\JsonLd\Algorithms\JsonLd\ProcessorOptions; - -$jsonString = '{ - "@context": "https://schema.org", - "@type": "Person", - "name": "John Doe" -}'; - -$options = new ProcessorOptions( - ordered: true, - frameExpansion: true, -); -$expander = new Expander(); -$result = $expander->expand($jsonString, options: $options, encodeResult: false); +$expanded = (new Expander())->expand('{"@context": "https://schema.org", "@type": "Person", "name": "John Doe"}'); ``` -### Loading remote contexts +See [the JSON-LD documentation](docs/json-ld.md) for the options and the conformance report. -#### By default, nothing goes out - -A JSON-LD document may point its `@context` at a URL, and the specification requires -that URL to be resolved before the document can be expanded. This library resolves -`https://schema.org` (and its `http` and trailing-slash variants) from the vocabulary -files it ships with, so the most common case by far is covered without a single -outbound request. - -Every other remote context is **refused**. `Validator::audit()` and the four algorithms -issue no network request and read no file unless you say otherwise, and a refused -context raises the error the specification mandates: - -``` -loading remote context failed -``` +### Remote contexts -#### Why unbounded resolution is dangerous - -The `@context` URL comes from the document being processed. As soon as that document is -not fully under your control, the URL is attacker-controlled, and a loader that resolves -anything hands them: - -- **Request forgery.** `http://127.0.0.1:9200/`, `http://169.254.169.254/latest/meta-data/`, - or any host on your internal network, reachable from your server. -- **Network mapping.** Even without seeing the responses, the difference between a - refusal, a timeout, and a success tells them which internal ports are open. -- **Exfiltration.** If the response body of a failed fetch ever finds its way back into - an error message. This is why the message above is opaque: it discloses neither the - body, nor the status code, nor the URL that was tried. -- **Denial of service.** Through a response that never ends or never arrives, or through - a chain of contexts that each pull more contexts (`@import`, alternate locations, - `Link rel="…json-ld#context"` headers). -- **Local file reads.** If a non-http scheme is allowed to reach the PHP stream wrappers: - `file:///var/www/.env`, or `phar://`, which deserializes archive metadata on a mere - stat call. - -#### Widening the policy, safely - -If your documents legitimately reference contexts you trust, allow those hosts, and -nothing else: +`https://schema.org` contexts are resolved from bundled files. Any other remote `@context` is +refused by default. To allow some hosts, install `symfony/http-client` and pass an +allow-list: ```php -use JoliCode\StructuredData\JsonLd\Algorithms\Http\HttpDocumentLoader; -use JoliCode\StructuredData\JsonLd\Algorithms\Http\RemoteContextPolicy; -use JoliCode\StructuredData\Validator; -use Symfony\Component\HttpClient\HttpClient; -use Symfony\Component\HttpClient\NoPrivateNetworkHttpClient; - -// 1. Which hosts do you trust? An explicit list, over https only. -$policy = RemoteContextPolicy::allowHosts('schema.org', 'www.w3.org', 'json-ld.org') - ->withTimeouts(timeout: 2.0, maxDuration: 5.0) - ->withMaxResponseBytes(512 * 1024) - ->withMaxRedirects(3); - -// 2. A second barrier, at the transport level: no private, loopback or link-local -// address, even if a hostile DNS answer points an allowed host at 169.254.169.254. -$httpClient = new NoPrivateNetworkHttpClient(HttpClient::create()); - -// 3. A single injection point covers the whole chain. +$policy = RemoteContextPolicy::allowHosts('www.w3.org', 'json-ld.org'); $validator = new Validator(documentLoader: new HttpDocumentLoader($policy, $httpClient)); -$audit = $validator->audit($document); -``` - -The same constructor argument is available on `Expander`, `Compactor`, `Flattener` and `Framer`: - -```php -$expander = new Expander(documentLoader: new HttpDocumentLoader($policy, $httpClient)); -``` - -Host matching is exact, so allowing `schema.org` does not allow `evil.schema.org.example`. -A URL carrying userinfo (`https://user:pass@schema.org/`) or a non-default port -(`https://schema.org:8080/`) is refused. Only `http` and `https` may ever be allowed, and -`http` requires an explicit `withSchemes('http', 'https')`. The policy is re-checked on -every hop: the URL you asked for, **each intermediate redirect**, each alternate location, -each `Link` header, and the URL a response was ultimately served from. - -The `@context` URL is not the only document-controlled value that can reach the loader: -`Expander::expand()` (and, through it, `Validator::audit()`) also accepts a bare IRI as its -input and will resolve it. That path is bound by the very same policy, so the default -deny-all loader refuses it too - but keep it in mind when you widen the allow-list. - -#### Writing your own loader - -Implement `JoliCode\StructuredData\JsonLd\Algorithms\Http\DocumentLoaderInterface` to resolve contexts -your own way, for instance from a local mirror or a PSR-6 cache: - -```php -interface DocumentLoaderInterface -{ - public function load(string $url): \stdClass; - - public function getCacheNamespace(): string; -} -``` - -Processed contexts are cached for the lifetime of the process, and `getCacheNamespace()` -partitions that cache. Return a value that identifies what your loader is willing to -resolve, so that a context obtained under a permissive strategy can never be served to a -restrictive one. Signal every failure with -`new ContextProcessingException('loading remote context failed')`, and never put anything -from the remote response in that message. - -#### Checklist - -- List the allowed hosts explicitly, and keep the list short. -- Stay on `https` unless a context you genuinely must load is only available over `http`. -- Wrap your client in `NoPrivateNetworkHttpClient`. -- Set a timeout, a max duration, a response size cap and a redirect cap. -- Never return the body of a remote response to your users. - -### Command line interface - -Commands are also available to use the algorithms from the CLI: - -```bash -castor json-ld:expand -castor json-ld:flatten -castor json-ld:compact -castor json-ld:frame ``` -Each prints its output to the console. - -## Testing and QA commands - -All the tasks are defined in the `.castor` directory, one file per namespace. - -The following commands are available to run the QA checks: - -Command | Description ----- | ----- -`castor qa:install` | Installs the QA tooling -`castor qa:update` | Updates the QA tooling -`castor qa:cs` | Fixes coding standards -`castor qa:phpstan` | Runs PHPStan -`castor qa:all` | Runs all QA tasks -`castor qa:earl` | Runs the W3C test suites and writes the EARL conformance report (see [Conformance](#conformance)) - -The following commands build the [demo website](website/README.md), that runs the validator in the browser: - -Command | Description ----- | ----- -`castor website:build` | Builds the demo website: its assets, and PHP as WebAssembly with the validator -`castor website:check` | Checks, in node, that what is built boots, and audits a document -`castor website:serve` | Serves the demo website - -The following commands are available to run the tests: - -Command | Description ----- | ----- -`castor qa:phpunit:prepare` | Downloads the W3C tests suite -`castor qa:phpunit:run` | Runs PHPUnit -`castor qa:phpunit:coverage` | Runs PHPUnit with code coverage (requires the pcov or xdebug extension) -`castor qa:infection` | Runs Infection mutation testing on the validator and mapper layers (requires the pcov or xdebug extension) - -The W3C test suite is pinned to a known-good upstream commit (see `W3C_TEST_SUITE_REF` in `.castor/qa.php`). -To re-download it, or to test against the upstream main branch: - -```bash -castor qa:phpunit:prepare --force -castor qa:phpunit:prepare --force --ref main -``` - -Additional commands are available to run the benchmarks: - -Command | Description ----- | ----- -`castor qa:bench:all` | Runs all the benchmarks -`castor qa:bench:algorithms` | Runs the JSON-LD manipulation algorithms benchmark -`castor qa:bench:validators` | Runs the validators benchmark -`castor qa:bench:validators -d` | Runs the detailed and slow validators benchmark - -The latest figures, and what each of them measures, are in [BENCHMARKS.md](BENCHMARKS.md). - -## Vocabulary generation commands - -The validation classes shipped in `src/Vocabularies/Generated` are generated from the -vocabulary definitions. These commands refresh them: - -Command | Description ----- | ----- -`castor schema-org:update-version` | Bump the schema.org version to the latest release published on GitHub -`castor schema-org:download` | Download the schema.org types definition file and JSON-LD context -`castor schema-org:generate` | Generate the schema.org validation classes -`castor schema-org:download-examples` | Refresh the schema.org examples used by the tests -`castor google:download` | Crawl the Google structured-data documentation -`castor google:generate` | Generate the Google validation classes -`castor google:check` | Check the Google documentation coverage against the curated manifest - -The full procedures are described in the -[schema.org](resources/schema.org/UPGRADE_GUIDELINES.md) and -[Google](resources/google/UPGRADE_GUIDELINES.md) upgrade guidelines. +Read [Loading remote contexts](docs/remote-contexts.md) before widening the policy. ## Contributing -See the [CONTRIBUTING.md](CONTRIBUTING.md) file for more information. - -## Working on the library - -Clone the repository, then install the library's dependencies and the QA tooling: - -```bash -composer install # the library's own dependencies (Composer, not Castor) -castor qa:install # the QA tooling: php-cs-fixer, phpstan, phpunit, phpbench, infection -``` - -### Upgrading schema.org - -Upgrades are driven by the official schema.org release definition file. -The complete upgrade process is documented in: -[resources/schema.org/UPGRADE_GUIDELINES.md](resources/schema.org/UPGRADE_GUIDELINES.md) - -When upgrading, always review the [schema.org release notes](https://schema.org/docs/releases.html) and check that the changes to the `schema-org-baseline.json` test fixture reflect real schema.org changes. - -### Upgrading Google - -The Google validator tracks the [Google structured-data documentation](https://developers.google.com/search/docs/appearance/structured-data/intro-structured-data), which evolves continuously. -The complete upgrade process is documented in: -[resources/google/UPGRADE_GUIDELINES.md](resources/google/UPGRADE_GUIDELINES.md) +See [CONTRIBUTING.md](CONTRIBUTING.md) for the setup, the QA commands and the vocabulary +upgrade procedures. ## License diff --git a/composer.json b/composer.json index bd1c5f3c..710640e2 100644 --- a/composer.json +++ b/composer.json @@ -49,7 +49,7 @@ "symfony/var-dumper": "^8.0" }, "suggest": { - "symfony/http-client": "To resolve remote JSON-LD @context documents (off by default; see the \"Loading remote contexts\" section of the README)." + "symfony/http-client": "To resolve remote JSON-LD @context documents (off by default; see docs/remote-contexts.md)." }, "autoload": { "psr-4": { diff --git a/docs/json-ld.md b/docs/json-ld.md new file mode 100644 index 00000000..64569ec3 --- /dev/null +++ b/docs/json-ld.md @@ -0,0 +1,106 @@ +# JSON-LD 1.1 algorithms + +The library implements the W3C +[JSON-LD 1.1 Processing Algorithms and API](https://www.w3.org/TR/json-ld-api) and +[JSON-LD 1.1 Framing](https://www.w3.org/TR/json-ld11-framing/) Recommendations: + +- [Compaction](https://www.w3.org/TR/json-ld11-api/#compaction-algorithm): `JoliCode\StructuredData\JsonLd\Algorithms\Compact\Compactor` +- [Expansion](https://www.w3.org/TR/json-ld11-api/#expansion-algorithm): `JoliCode\StructuredData\JsonLd\Algorithms\Expand\Expander` +- [Flattening](https://www.w3.org/TR/json-ld11-api/#flattening-algorithm): `JoliCode\StructuredData\JsonLd\Algorithms\Flatten\Flattener` +- [Framing](https://www.w3.org/TR/json-ld11-framing/#framing-algorithm): `JoliCode\StructuredData\JsonLd\Algorithms\Frame\Framer` + +Serialization to and from RDF (`toRdf` / `fromRdf`) is out of scope and not implemented. + +## Usage + +Create an instance of the algorithm and pass it the JSON-LD document you want to convert. +For instance, expanding a document: + +```php +use JoliCode\StructuredData\JsonLd\Algorithms\Expand\Expander; + +$jsonString = '{ + "@context": "https://schema.org", + "@type": "Person", + "name": "John Doe" +}'; + +$expander = new Expander(); +$result = $expander->expand($jsonString); +``` + +The result is a JSON string containing the expanded document: + +```json +[ + { + "@type": [ + "http://schema.org/Person" + ], + "http://schema.org/name": [ + { + "@value": "John Doe" + } + ] + } +] +``` + +If you want a PHP value (an array or a `stdClass`) instead of a JSON string, set the +`encodeResult` parameter to `false`. You can also pass a `ProcessorOptions` object holding +the [JSON-LD options](https://www.w3.org/TR/json-ld-api/#the-jsonldoptions-type) to modify +the default behavior of the algorithms: + +```php +use JoliCode\StructuredData\JsonLd\Algorithms\Expand\Expander; +use JoliCode\StructuredData\JsonLd\Algorithms\JsonLd\ProcessorOptions; + +$options = new ProcessorOptions( + ordered: true, + frameExpansion: true, +); + +$expander = new Expander(); +$result = $expander->expand($jsonString, options: $options, encodeResult: false); +``` + +Remote `@context` URLs other than schema.org are refused by default: see +[Loading remote contexts](remote-contexts.md). + +## Conformance + +Each algorithm is validated against the official W3C test suites +([json-ld-api](https://github.com/w3c/json-ld-api) and +[json-ld-framing](https://github.com/w3c/json-ld-framing)), pinned to a known-good upstream +commit and re-run weekly against `main` to surface drift. + +The full suites pass. The only skipped fixtures are a handful that target JSON-LD +1.0-specific behavior (declared `specVersion: json-ld-1.0` in the upstream manifest), which +this library does not implement; each skip is documented in the corresponding test. + +As of the pinned upstream commits, the manifests list 781 tests, and every test that +applies to a JSON-LD 1.1 processor passes: + +| Suite | Passed | JSON-LD 1.0 only | +| --- | --- | --- | +| expand | 385 | 0 | +| compact | 244 | 2 | +| flatten | 58 | 0 | +| frame | 91 | 1 | + +`castor qa:earl` runs these suites and writes an [EARL](https://www.w3.org/TR/EARL10-Schema/) +report in Turtle, one assertion per manifest entry, in the format the +[W3C JSON-LD implementation report](https://w3c.github.io/json-ld-api/reports/) consolidates. + +## Command line interface + +From a clone of this repository, with [Castor](https://github.com/jolicode/castor/) +installed, the algorithms are available from the command line. Each prints its output to +the console: + +```bash +castor json-ld:expand +castor json-ld:flatten +castor json-ld:compact +castor json-ld:frame +``` diff --git a/docs/remote-contexts.md b/docs/remote-contexts.md new file mode 100644 index 00000000..3ddd9e08 --- /dev/null +++ b/docs/remote-contexts.md @@ -0,0 +1,111 @@ +# Loading remote contexts + +## By default, nothing goes out + +A JSON-LD document may point its `@context` at a URL, and the specification requires +that URL to be resolved before the document can be expanded. This library resolves +`https://schema.org` (and its `http` and trailing-slash variants) from the vocabulary +files it ships with, so the most common case by far is covered without a single +outbound request. + +Every other remote context is **refused**. `Validator::audit()` and the four algorithms +issue no network request and read no file unless you say otherwise, and a refused +context raises the error the specification mandates: + +``` +loading remote context failed +``` + +## Why unbounded resolution is dangerous + +The `@context` URL comes from the document being processed. As soon as that document is +not fully under your control, the URL is attacker-controlled, and a loader that resolves +anything hands them: + +- **Request forgery.** `http://127.0.0.1:9200/`, `http://169.254.169.254/latest/meta-data/`, + or any host on your internal network, reachable from your server. +- **Network mapping.** Even without seeing the responses, the difference between a + refusal, a timeout, and a success tells them which internal ports are open. +- **Exfiltration.** If the response body of a failed fetch ever finds its way back into + an error message. This is why the message above is opaque: it discloses neither the + body, nor the status code, nor the URL that was tried. +- **Denial of service.** Through a response that never ends or never arrives, or through + a chain of contexts that each pull more contexts (`@import`, alternate locations, + `Link rel="…json-ld#context"` headers). +- **Local file reads.** If a non-http scheme is allowed to reach the PHP stream wrappers: + `file:///var/www/.env`, or `phar://`, which deserializes archive metadata on a mere + stat call. + +## Widening the policy, safely + +If your documents legitimately reference contexts you trust, install an HTTP client +(`composer require symfony/http-client`), then allow those hosts, and nothing else: + +```php +use JoliCode\StructuredData\JsonLd\Algorithms\Http\HttpDocumentLoader; +use JoliCode\StructuredData\JsonLd\Algorithms\Http\RemoteContextPolicy; +use JoliCode\StructuredData\Validator; +use Symfony\Component\HttpClient\HttpClient; +use Symfony\Component\HttpClient\NoPrivateNetworkHttpClient; + +// 1. Which hosts do you trust? An explicit list, over https only. +$policy = RemoteContextPolicy::allowHosts('schema.org', 'www.w3.org', 'json-ld.org') + ->withTimeouts(timeout: 2.0, maxDuration: 5.0) + ->withMaxResponseBytes(512 * 1024) + ->withMaxRedirects(3); + +// 2. A second barrier, at the transport level: no private, loopback or link-local +// address, even if a hostile DNS answer points an allowed host at 169.254.169.254. +$httpClient = new NoPrivateNetworkHttpClient(HttpClient::create()); + +// 3. A single injection point covers the whole chain. +$validator = new Validator(documentLoader: new HttpDocumentLoader($policy, $httpClient)); +$audit = $validator->audit($document); +``` + +The same constructor argument is available on `Expander`, `Compactor`, `Flattener` and `Framer`: + +```php +$expander = new Expander(documentLoader: new HttpDocumentLoader($policy, $httpClient)); +``` + +Host matching is exact, so allowing `schema.org` does not allow `evil.schema.org.example`. +A URL carrying userinfo (`https://user:pass@schema.org/`) or a non-default port +(`https://schema.org:8080/`) is refused. Only `http` and `https` may ever be allowed, and +`http` requires an explicit `withSchemes('http', 'https')`. The policy is re-checked on +every hop: the URL you asked for, **each intermediate redirect**, each alternate location, +each `Link` header, and the URL a response was ultimately served from. + +The `@context` URL is not the only document-controlled value that can reach the loader: +`Expander::expand()` (and, through it, `Validator::audit()`) also accepts a bare IRI as its +input and will resolve it. That path is bound by the very same policy, so the default +deny-all loader refuses it too - but keep it in mind when you widen the allow-list. + +## Writing your own loader + +Implement `JoliCode\StructuredData\JsonLd\Algorithms\Http\DocumentLoaderInterface` to resolve contexts +your own way, for instance from a local mirror or a PSR-6 cache: + +```php +interface DocumentLoaderInterface +{ + public function load(string $url): \stdClass; + + public function getCacheNamespace(): string; +} +``` + +Processed contexts are cached for the lifetime of the process, and `getCacheNamespace()` +partitions that cache. Return a value that identifies what your loader is willing to +resolve, so that a context obtained under a permissive strategy can never be served to a +restrictive one. Signal every failure with +`new ContextProcessingException('loading remote context failed')`, and never put anything +from the remote response in that message. + +## Checklist + +- List the allowed hosts explicitly, and keep the list short. +- Stay on `https` unless a context you genuinely must load is only available over `http`. +- Wrap your client in `NoPrivateNetworkHttpClient`. +- Set a timeout, a max duration, a response size cap and a redirect cap. +- Never return the body of a remote response to your users. diff --git a/docs/validator.md b/docs/validator.md new file mode 100644 index 00000000..e0b155d1 --- /dev/null +++ b/docs/validator.md @@ -0,0 +1,154 @@ +# Validating structured data + +To validate a document, use the `JoliCode\StructuredData\Validator` class. It accepts +JSON-LD, microdata and RDFa (schema.org-style RDFa only), either as a JSON-LD document or +embedded in an HTML page. + +## Accepted inputs + +`audit()` takes **the document itself**, as a string - never a URL, never a file path. + +This library deliberately does not guess what a string is, and never fetches anything on +your behalf. Guessing is a security hazard: an application that forwards user input to a +validator would silently offer an attacker a way to reach its internal network +(`http://127.0.0.1:9200/`, cloud metadata endpoints), or to read local files through a +path or a stream wrapper (`/var/www/.env`, `file://`, `phar://`). + +Whether a document may be fetched, from where, and under which restrictions is a decision +only your application can make. So your application is the one that fetches: + +```php +// From a local file - the path comes from you, not from a user +$document = file_get_contents('/path/to/document.html'); + +// From the network - your HTTP client, your allow-list, your timeouts +$document = $httpClient->request('GET', $trustedUrl)->getContent(); + +$audit = $validator->audit($document); +``` + +The same rule applies to the `@context` URLs found *inside* a document: see +[Loading remote contexts](remote-contexts.md). + +## Reading the result + +`audit()` returns a `JoliCode\StructuredData\Audit\Audit` object: + +- `isValid()` returns true if no errors are detected; +- `isFullyValid()` returns true if no errors, no warnings, and no malformed (hence + unusable) data structures were detected; +- `getDiagnostic()` returns the messages (errors and warnings). + +**Keep in mind that a schema.org type is considered valid even with warnings!** + +```php +use JoliCode\StructuredData\Validator; + +$validator = new Validator(); +$audit = $validator->audit($document); + +if (!$audit->isValid()) { + foreach ($audit->getDiagnostic() as $message) { + // [Google warning] DataFeed.dataFeedElement.workExample: Missing recommended property: "sameAs" for the type "Book" + // [Google error] DataFeed.dataFeedElement.workExample.potentialAction.expectsAcceptanceOf: Missing required property: "price" for the type "Offer" when "category" is "purchase" or "rental". + echo $message; + } +} +``` + +`getDiagnostic()` accepts an optional `JoliCode\StructuredData\Audit\AuditOptions` object, +which lets you filter or group the results, or change the return format. See its PHPDoc +for details. + +To access the full parsed PHP tree, use `getTypes()` and inspect the underlying +`MappedType` objects directly. These are low-level objects, but they are the most detailed +information you can get, and they preserve the type hierarchy of the document. For an idea +of what you can do with them, look at the output of the `castor validate` command. + +## Running a single validator + +If you are only interested in the results of a single validator, call `setValidator()` +before auditing: + +```php +use JoliCode\StructuredData\Validator; +use JoliCode\StructuredData\Vocabularies\Validators\Google\GoogleValidator; + +$validator = new Validator(); +$validator->setValidator(GoogleValidator::class); + +$validator->audit($document); +``` + +## Pending vocabulary + +Terms hosted under [pending.schema.org](https://pending.schema.org) are still under +development and may change or be removed. Using them is legitimate, so the validator +accepts them silently by default; opt in to `$reportPendingVocabularyUsage` to get a +warning for each of these usages instead: + +```php +$audit = $validator->audit($document, reportPendingVocabularyUsage: true); +``` + +## Blocks describing the same node + +Google reads a page as one graph: the blocks that share an identifier (a JSON-LD `@id`, +a microdata `itemid`) are one node holding the union of their properties, whatever their +format, and a property one block omits is not missing when another block provides it. + +By default, the validator does the same. The audit is made of items, one per node, listed +by `Audit::getItems()`: the blocks sharing an identifier make one item, the others one +item each. The diagnostics of the audit (`isValid()`, `getDiagnostic()`) are the ones of +the items, so an issue of a merged node is reported once, with the positions of every +block it comes from. + +```php +foreach ($audit->getItems() as $item) { + $item->getType(); // the node: merged from several blocks, or the block itself + $item->isMerged(); // true when the node comes from several blocks + $item->getIdentifier(); // its absolute "@id" or "itemid", if any + $item->getSources(); // the blocks it comes from +} +``` + +Each source block keeps its own audit, as if it were alone in the document: a block may +report a property another block provides, while the merged item, which makes the verdict, +does not. `Audit::getTypes()` still lists the blocks. + +When the blocks disagree on a value, Google keeps every value and reports nothing; the +validator raises a warning on the property of the merged node, with the `Merge` validator +name (`AuditOptions::VALIDATOR_MERGE`), outside the Google diagnostics. Blank node +identifiers are local to their block and never merge, and identifiers are compared as +written. Only the top-level node of each block is merged: nested nodes sharing an +identifier are not. + +Google is not the only consumer of structured data, though. To audit every block on its +own, with no merge, disable the option: + +```php +$audit = $validator->audit($document, mergeNodesSharingAnIdentifier: false); +``` + +Each block is then an item of its own. + +## Command line interface + +From a clone of this repository, with [Castor](https://github.com/jolicode/castor/) +installed, two commands validate a document from the command line or in CI. `castor check` +gives a short verdict, `castor validate` a parsed and colored full audit (it can be pretty +verbose!). Both return a meaningful exit code. + +```bash +castor check +castor validate +castor validate google # a single validator: google or schema-org +castor validate --no-merge # audit every block on its own +``` + +Unlike the `Validator::audit()` API, these commands **do** accept a file path or a URL, +and will read or fetch it for you. That is safe here precisely because the argument comes +from the operator running the command, not from a document being processed - exactly the +distinction drawn in [Accepted inputs](#accepted-inputs). + +Output of the castor validate command diff --git a/resources/google/UPGRADE_GUIDELINES.md b/resources/google/UPGRADE_GUIDELINES.md index 42c2357f..0be708af 100644 --- a/resources/google/UPGRADE_GUIDELINES.md +++ b/resources/google/UPGRADE_GUIDELINES.md @@ -109,7 +109,7 @@ Then run the full suite with `castor qa:phpunit:run`. ## 9. Keep the docs aligned -Check that the command examples in `README.md` and in +Check that the command examples in `docs/`, `CONTRIBUTING.md` and in `resources/google/structured-data/README.md` still match the actual task names and behavior, and update the tasks in `.castor/` if needed. diff --git a/resources/schema.org/UPGRADE_GUIDELINES.md b/resources/schema.org/UPGRADE_GUIDELINES.md index b26c55d1..e63c0499 100644 --- a/resources/schema.org/UPGRADE_GUIDELINES.md +++ b/resources/schema.org/UPGRADE_GUIDELINES.md @@ -102,6 +102,6 @@ shared with the other vocabularies. ## 9. Keep the docs and the commands aligned Update the tasks in `.castor/` if needed, and check that the command examples and the upgrade -notes in `README.md` are still accurate. +notes in `docs/` and `CONTRIBUTING.md` are still accurate. ## 10. Submit a pull request! diff --git a/src/JsonLd/Algorithms/Http/DocumentLoaderInterface.php b/src/JsonLd/Algorithms/Http/DocumentLoaderInterface.php index ae7dca99..1c28d291 100644 --- a/src/JsonLd/Algorithms/Http/DocumentLoaderInterface.php +++ b/src/JsonLd/Algorithms/Http/DocumentLoaderInterface.php @@ -19,7 +19,7 @@ * * Implementations decide what may be resolved. Keep in mind that these URLs come * from the document being processed, so on untrusted input they are attacker - * controlled: see the "Loading remote contexts" section of the README. + * controlled: see docs/remote-contexts.md. */ interface DocumentLoaderInterface { diff --git a/src/Validator.php b/src/Validator.php index 6b85639d..33b3d7cc 100644 --- a/src/Validator.php +++ b/src/Validator.php @@ -63,8 +63,7 @@ class Validator * resolved. Defaults to a loader * that refuses every host, so no * request ever leaves the process. - * See the "Loading remote contexts" - * section of the README. + * See docs/remote-contexts.md. */ public function __construct( private readonly ValidationMapper $validationMapper = new ValidationMapper(), @@ -107,7 +106,7 @@ public function setValidator(string $validator): void * The $document argument is the document itself, not a URL nor a file path: * this library never guesses what an input is, and never fetches it for you. * Deciding what may be fetched, and under which restrictions, belongs to the - * application - see the "Accepted inputs" section of the README. + * application - see the "Accepted inputs" section of docs/validator.md. * * @param bool $reportPendingVocabularyUsage Terms hosted under * pending.schema.org are still @@ -126,7 +125,7 @@ public function setValidator(string $validator): void * false to audit every block on * its own. See the "Blocks * describing the same node" - * section of the README. + * section of docs/validator.md. */ public function audit(string $document, bool $reportPendingVocabularyUsage = false, bool $mergeNodesSharingAnIdentifier = true): Audit { diff --git a/website/README.md b/website/README.md index 8a1ea5bb..96af4a32 100644 --- a/website/README.md +++ b/website/README.md @@ -1,5 +1,8 @@ # Demo website +> [!TIP] +> This demo website is deployed under https://structured-data-validator.jolicode.com/ + A static page that audits a document with the validator, in the browser: PHP is built as WebAssembly, with [lyrixx/php-wasm-builder](https://github.com/lyrixx/php-wasm-builder), and the library of this repository embedded in it. GitHub Pages serves `public/`, deployed from `main` by