diff --git a/docs/data-model.md b/docs/data-model.md index d03005481..1e0f781d9 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -1887,17 +1887,35 @@ An import reads the file as the pod is read, with the same mappers: an IRI that may be short. A file this app exports states short namespaces with a scheme; - a JSON-LD term, `@base` or `@vocab` is refused (`deckFileUnreadable`) - when it's a compact IRI whose prefix, the part before its first `:`, - is a term of the node's contexts. The parser expands such a term - through the other with no bound on how deep, so two terms defined - through each other, or one through itself, never end. A file this + when it's another term of the node's contexts, or a compact IRI whose + prefix, the part before its first `:`, is one. The parser expands + such a term through the other with no bound on how deep, so two terms + defined through each other, or one through itself, never end. A term + defined as one that names a keyword names that keyword too, which the + checks below would read as a plain term. A file this app exports states each namespace in full. With the base bounded and each term defined in full, at most 512 characters, the limits below bound the time; - the parser reads the document as it streams, each object written again with its context first. A parser that held every node to the end would match each against every typed one, in time growing with - the square of the nodes; + the square of the nodes. It gets the text in parts of 4,096 + characters, each in a task of its own, and no more after its first + error, so a parse that fails leaves the tab free after the part it + failed in; +- a JSON-LD keyword given a value the parser reports as an error is + refused (`deckFileUnreadable`) before it's parsed, under an alias + too. These are an `@id` that's no string, or a second one in an + object, and a `@type` that's no string or array of strings. So are an + `@index` that's no string and a `@reverse` that's no object or holds + a keyword. So are an `@included` member that's no node object, a + `@nest` member that's no object or holds `@value` or `@id`, and a + `@list` beside any key but `@index`. A `@set` object's members, and + an array's in it, count as members of the keyword it sits under. The + parser reports such an error and reads on, and each takes it far + longer than a triple, which the bound on the work below doesn't + count. As with the checks below, it covers the whole + document, a `@json` literal too. A file this app exports holds none; - JSON nested deeper than 12 levels is refused (`deckFileUnreadable`), before anything else reads it: the JSON-LD parser takes time far beyond linear in the depth, and a file this app exports nests 5; diff --git a/docs/deck-library.md b/docs/deck-library.md index 2aa638e23..2f076b19f 100644 --- a/docs/deck-library.md +++ b/docs/deck-library.md @@ -1050,7 +1050,8 @@ flowchart LR and `@vocab` are refused (`deckFileUnreadable`) past it, counted with the release's own address. Its terms and prefixes are refused past their bound or when defined through another term, as a deck file's - are, and so is an RDF-star annotation or triple term + are. So are a keyword given a value the parser reports as an error + and an RDF-star annotation or triple term ([data-model.md](data-model.md#decks-as-files)). A Turtle release with a triple term, an annotation or a reifier doesn't parse (`releaseUnreadable`), however it's read: the parser reads none, as diff --git a/packages/solid/src/client/resources.test.ts b/packages/solid/src/client/resources.test.ts index c431d488c..7422b19fc 100644 --- a/packages/solid/src/client/resources.test.ts +++ b/packages/solid/src/client/resources.test.ts @@ -196,3 +196,37 @@ describe("the headers that describe a resource, over the library", () => { expect(handed.every((response) => response.headers.get("Link") === null)).toBe(true); }); }); + +describe("parseQuads, of JSON-LD the parser reports errors in", () => { + /** + * The least CPU time, of three parses of `text`, until the main thread + * is free again: the parser may still work after its promise settles. + */ + async function cpuUntilFree(text: string): Promise { + let least = Infinity; + for (let round = 0; round < 3; round++) { + const started = process.cpuUsage(); + await resources.parseQuads(text, "jsonld", "https://pod.example/").catch(() => []); + await new Promise((resolve) => setTimeout(resolve, 0)); + const { user, system } = process.cpuUsage(started); + least = Math.min(least, user + system); + } + return least; + } + + it("rejects at the first error and reads no further than the part it is in (GHSA-c6fr-r6fm-4hjm)", async () => { + const count = 4_000; + const graph = (node: (at: number) => string) => + `{"@context":{"ex":"https://example.org/ns#"},"@graph":[${Array.from({ length: count }, (_, at) => node(at)).join()}]}`; + const nodes = graph((at) => `{"@id":"ex:s${at}","ex:p":"v${at}"}`); + const errors = graph(() => '{"@id":5}'); + await expect( + resources.parseQuads(nodes, "jsonld", "https://pod.example/"), + ).resolves.toHaveLength(count); + await expect(resources.parseQuads(errors, "jsonld", "https://pod.example/")).rejects.toThrow( + "Found illegal @id '5'", + ); + // The parser would take longer over the errors than over the nodes, did it read them all. + expect(await cpuUntilFree(errors)).toBeLessThan((await cpuUntilFree(nodes)) / 2); + }); +}); diff --git a/packages/solid/src/client/resources.ts b/packages/solid/src/client/resources.ts index e49aa1f77..74d64afc8 100644 --- a/packages/solid/src/client/resources.ts +++ b/packages/solid/src/client/resources.ts @@ -498,9 +498,12 @@ export function parseQuads( * its other keys (refuseRemoteContexts writes it so), or the text does * not parse: otherwise the parser holds every node to the end, then * matches each against every typed one, time growing with their square. - * The text is written in parts (textChunks), and the parser destroyed - * once `onQuad` says not to go on, so that it reads no further than the - * part it is in. + * The text is written in parts (textChunks), each in a task of its own + * (taskQueue), and the parser destroyed once `onQuad` says not to go on + * or at its first error, so that it reads no further than the part it + * is in. The parser reports some errors and reads on, each taking it far + * more time than a triple; and the parts written at once would hold the + * tab for all the work they queue, past the error that ends the parse. */ function parseJsonLd( text: string, @@ -522,13 +525,52 @@ function parseJsonLd( load: (url) => Promise.reject(new Error(`A JSON-LD context elsewhere is not read: ${url}`)), }, }); + let stopped = false; + const stop = () => { + stopped = true; + parser.destroy(); + }; parser.on("data", (quad: Quad) => { - if (!onQuad(quad)) parser.destroy(); + if (!onQuad(quad)) stop(); + }); + parser.on("error", (error: Error) => { + stop(); + onError(error); }); - parser.on("error", onError); parser.on("end", done); - for (const chunk of textChunks(text)) parser.write(chunk); - parser.end(); + const tasks = taskQueue(); + (async () => { + try { + for (const chunk of textChunks(text)) { + parser.write(chunk); + await tasks.next(); + if (stopped) break; + } + } finally { + tasks.close(); + } + if (!stopped) parser.end(); + })().catch(onError); +} + +/** + * A wait for the next task of the event loop, after the microtasks + * queued so far, the parser's work on what it was written among them: + * a message to itself, which a browser delivers at once, where a + * timeout nested this deep waits some milliseconds. + */ +function taskQueue(): { next: () => Promise; close: () => void } { + const channel = new MessageChannel(); + let resume: () => void; + channel.port1.onmessage = () => resume(); + return { + next: () => + new Promise((resolve) => { + resume = resolve; + channel.port2.postMessage(null); + }), + close: () => channel.port1.close(), + }; } /** The URL of the document a dataset was read from or saved to; null for one built here. */ diff --git a/packages/solid/src/jsonLd.test.ts b/packages/solid/src/jsonLd.test.ts index e2f8ff02a..d36391bd7 100644 --- a/packages/solid/src/jsonLd.test.ts +++ b/packages/solid/src/jsonLd.test.ts @@ -8,6 +8,7 @@ import { growth } from "@solid-memo/domain/testing/growth"; import { jsonLdOf, refuseRemoteContexts as refuseIn } from "./jsonLd"; import { termOf } from "./ntriples"; import { MAX_TERM_CHARACTERS } from "./client/readLimits"; +import type { ReadLimits } from "./client/interface"; import { parseQuads } from "./client/resources"; const EX = "https://example.org/ns#"; @@ -43,6 +44,27 @@ async function triplesOf(text: string): Promise { .sort(); } +/** + * The least CPU time, of three reads, that checking a deck file and + * parsing what passes takes, until the main thread is free again: the + * parser may still work after its promise settles. + */ +async function cpuUntilFree(text: string, limits: ReadLimits): Promise { + let least = Infinity; + for (let round = 0; round < 3; round++) { + const started = process.cpuUsage(); + try { + await parseQuads(refuseIn(text, EX, limits), "jsonld", EX, limits); + } catch { + // Refused, or not parsed: the time until then counts all the same. + } + await new Promise((resolve) => setTimeout(resolve, 0)); + const { user, system } = process.cpuUsage(started); + least = Math.min(least, user + system); + } + return least; +} + describe("jsonLdOf", () => { const quads = [ quad(iri(`${EX}a`), RDF_TYPE, iri(`${EX}Thing`)), @@ -297,6 +319,9 @@ describe("refuseRemoteContexts", () => { { ex: EX, p: "ex:p" }, [{ ex: EX }, { "@vocab": "ex:" }], { ex: EX, "@base": "ex:a/" }, + // A term defined as another, which may name a keyword. + { "x:y": "@included", b: "x:y" }, + { "x:y": EX, b: "x:y" }, ]) { expect(() => refuseRemoteContexts(JSON.stringify({ "@context": context, "@id": "n" })), @@ -317,13 +342,12 @@ describe("refuseRemoteContexts", () => { // A file of nothing else, as large as a deck file may be, is refused too. expect(() => refuseRemoteContexts(values(value, 100_000))).toThrow(refused); } - // A string, number or boolean where a node belongs states nothing either. - for (const key of ["@graph", "@included"]) { - const nodes = (count: number) => - `{"@context":{"ex":"${EX}"},"${key}":[${Array(count).fill('"a"').join(",")}]}`; - expect(() => refuseRemoteContexts(nodes(4_096))).not.toThrow(); - expect(() => refuseRemoteContexts(nodes(4_097))).toThrow(refused); - } + // A string, number or boolean where a node belongs states nothing either (in `@included`, it is + // refused as invalid at once). + const nodes = (count: number) => + `{"@context":{"ex":"${EX}"},"@graph":[${Array(count).fill('"a"').join(",")}]}`; + expect(() => refuseRemoteContexts(nodes(4_096))).not.toThrow(); + expect(() => refuseRemoteContexts(nodes(4_097))).toThrow(refused); expect(() => refuseRemoteContexts(`[${Array(4_097).fill("1").join(",")}]`)).toThrow(refused); expect(() => refuseRemoteContexts( @@ -439,6 +463,81 @@ describe("refuseRemoteContexts", () => { } }); + it("refuses a keyword given a value the parser reports as an error and reads on past, however the keyword is named (GHSA-c6fr-r6fm-4hjm)", () => { + const refused = expect.objectContaining({ + code: "deckFileUnreadable", + vars: { reason: "The file gives a JSON-LD keyword a value it may not have." }, + }); + const context = { + ex: EX, + i: "@id", + t: "@type", + inc: "@included", + n: "@nest", + v: "@value", + s: "@set", + }; + const node = (members: object) => JSON.stringify({ "@context": context, ...members }); + for (const members of [ + { "@graph": [{ "@id": 5 }] }, + { "@graph": [{ i: null }] }, + { "@id": ["ex:a"] }, + { "@id": "ex:a", i: "ex:b" }, + { "@id": "ex:a", t: 5 }, + { "@id": "ex:a", "@type": ["ex:T", ["ex:U"]] }, + { "@id": "ex:a", "@index": 5 }, + { "@id": "ex:a", "@reverse": "ex:b" }, + { "@id": "ex:a", "@reverse": { "@id": "ex:b" } }, + { "@id": "ex:a", inc: "s" }, + { "@id": "ex:a", "@included": [{ "@id": "ex:b" }, { v: "s" }] }, + { "@id": "ex:a", "@included": { "@list": [] } }, + { "@id": "ex:a", n: "x" }, + { "@id": "ex:a", "@nest": [{ "ex:p": "s" }, null] }, + { "@id": "ex:a", "@nest": { v: "s" } }, + { "@id": "ex:a", "@nest": { i: "ex:b" } }, + // A `@set` object's members are the keyword's, in an array in it too, under an alias as well. + { "@id": "ex:a", "@included": { "@set": [{ "@value": "v" }] } }, + { "@id": "ex:a", inc: { s: ["s"] } }, + { "@id": "ex:a", "@included": [{ "@set": [[{ "@id": "ex:b" }, "s"]] }] }, + { "@id": "ex:a", "@nest": { "@set": [{ "@value": "v" }] } }, + { "@id": "ex:a", n: { s: { s: ["s"] } } }, + { "@id": "ex:a", "@nest": { "@set": [{ "@id": "ex:b" }] } }, + { "@id": "ex:a", "ex:p": { "@list": ["s"], "@id": "ex:b" } }, + { "@id": "ex:a", "ex:p": { "@list": ["s"], "ex:q": "s" } }, + ]) { + expect(() => refuseRemoteContexts(node(members))).toThrow(refused); + } + // The same keywords given what the parser reads, and a context's own members, which name them. + const valid = node({ + "@id": "ex:a", + t: ["ex:T", "ex:U"], + "@index": "i", + "@reverse": { "ex:p": { i: "ex:b" } }, + inc: [{ i: "ex:c", "ex:p": "s" }, { s: [{ i: "ex:d" }] }], + n: [{ "ex:q": "s", "@type": "ex:V" }, { "@set": [{ "ex:q": "t" }] }], + "ex:r": { "@list": ["s"], "@index": "j" }, + }); + expect(() => refuseRemoteContexts(valid)).not.toThrow(); + }); + + it("refuses up front a file of keywords given values the parser reports as errors, which would take it longer than an exported file at the same limits (GHSA-c6fr-r6fm-4hjm)", async () => { + const quads = 2_000; + const limits = { bytes: 64 * 1024 * 1024, quads, characters: 64 * 1024 * 1024 }; + const exported = jsonLdOf( + Array.from({ length: quads }, (_, at) => + quad(iri(`${EX}s${at % 100}`), `${EX}p`, literal(`v${at}`, at % 2 === 0 ? "en" : "")), + ), + { ex: EX }, + ); + // As many values as the bound on the work let through at these limits before. + const ids = `{"@graph":[${Array(3_400).fill('{"@id":5}').join()}]}`; + const included = `{"@context":{"x:y":"@included","b":"x:y"},"@id":"${EX}x","b":[${Array(3_600).fill('"s"').join()}]}`; + const base = await cpuUntilFree(exported, limits); + for (const text of [ids, included]) { + expect(await cpuUntilFree(text, limits)).toBeLessThan(base); + } + }); + it("gives the document it checked, a key named twice as JSON.parse keeps it", () => { expect(refuseRemoteContexts('{"@context": "https://evil.example/c", "@context": {}}')).toBe( '{"@context":{}}', diff --git a/packages/solid/src/jsonLd.ts b/packages/solid/src/jsonLd.ts index 7126c03a4..4464d3a90 100644 --- a/packages/solid/src/jsonLd.ts +++ b/packages/solid/src/jsonLd.ts @@ -266,11 +266,13 @@ function refuseLongStrings(text: string): void { * which the parser is also told not to read, and a string longer than a * term may be (refuseLongStrings), before JSON.parse reads it, and more * than MAX_EMPTY_VALUES values that make no triple or sit in nested - * arrays (valueTally). Throws deckFileUnreadable, as for text that is no - * JSON at all; and deckFileTooLarge for values that take the parser more - * work than WORK_PER_QUAD for each triple `limits` allow, when there are - * limits: the parser takes time for each value, whether it makes a - * triple or not, the more the deeper it sits (valueTally). + * arrays (valueTally), and a member the parser reports as an error and + * reads on past (refuseInvalid). Throws deckFileUnreadable, as for + * text that is no JSON at all; and deckFileTooLarge for values that + * take the parser more work than WORK_PER_QUAD for each triple `limits` + * allow, when there are limits: the parser takes time for each value, + * whether it makes a triple or not, the more the deeper it sits + * (valueTally). * * Gives the document checked, written again: the parser is to read that * text, not the file's, so that both see the same document. A key named @@ -297,6 +299,8 @@ export function refuseRemoteContexts( const ids = new Set(["@id"]); const givenObjects = new Set(); const tally = valueTally(limits); + // The contexts, whose members are terms and their IRIs, not values the parser reads. + const contexts = new Set(); // A walk of its own, not a recursion, which a file nested deeper than the call stack would end. const pending: Pending[] = [ { value: json, depth: 0, place: "nodes", arrays: 0, key: "other", aliases: new Map() }, @@ -313,16 +317,16 @@ export function refuseRemoteContexts( } // The node's context first, for its aliases to name the keywords of its other keys, and of // the values in it alone: not of another top-level node's, as the parser reads them. - let { aliases } = visit; - if ("@context" in value) { - const context = (value as { "@context": unknown })["@context"]; - const own = new Map(aliases); - for (const [term, aliased] of refuseContext(context, topLevel.has(value), baseUrl)) { - own.set(term, aliased); - if (aliased === "@id") ids.add(term); - } - aliases = own; - } + const aliases = + "@context" in value + ? withContext(value, visit.aliases, { + onTopLevelNode: topLevel.has(value), + baseUrl, + ids, + contexts, + }) + : visit.aliases; + if (!Array.isArray(value) && !contexts.has(value)) refuseInvalid(value, aliases); for (const [key, child] of Object.entries(value)) { if (key === "@annotation") refuseTripleTerms(); if (holdsObject(child)) givenObjects.add(key); @@ -341,6 +345,28 @@ export function refuseRemoteContexts( return written(json); } +/** + * The aliases over an object's members, `aliases` with those its own + * context gives (refuseContext): the terms aliased to `@id` noted in + * `ids`, and the context, and each it lists, in `contexts`. + */ +function withContext( + value: { "@context"?: unknown }, + aliases: ReadonlyMap, + seen: { onTopLevelNode: boolean; baseUrl: string; ids: Set; contexts: Set }, +): ReadonlyMap { + const context = value["@context"]; + for (const one of Array.isArray(context) ? [context, ...context] : [context]) { + seen.contexts.add(one); + } + const own = new Map(aliases); + for (const [term, aliased] of refuseContext(context, seen.onTopLevelNode, seen.baseUrl)) { + own.set(term, aliased); + if (aliased === "@id") seen.ids.add(term); + } + return own; +} + /** * A count of a document's values as refuseRemoteContexts walks them, * which throws deckFileUnreadable at the value past MAX_EMPTY_VALUES that @@ -406,6 +432,85 @@ function placeIn( return { place: place === "nodes" ? "node" : "nested", arrays: arrays + 1 }; } +/** + * Throws deckFileUnreadable for an object's members (`aliases` resolved) + * that the parser reports as an error and then reads on past: an `@id` + * that is no string (one that is an object is a triple term, + * refuseTripleTerms), or two of them in one object; a `@type` that is no + * string or array of strings; an `@index` that is no string; a + * `@reverse` that is no object, or one holding a keyword (a context + * there is refused anyway, refuseContext); an `@included` member that + * is no node object (a value or list object, say); a `@nest` member + * that is no object, or one holding a `@value`, or an `@id` (the node's + * second), a `@set` object's members and an array's in it counting as + * the keyword's (members); and a `@list` beside any key but `@index`. + * Each such error takes the parser far more time than a triple, and + * makes none, so the bound on the values (valueTally) would not bound + * the time. A file this app exports holds none of them. + */ +function refuseInvalid(object: object, aliases: ReadonlyMap): void { + const keywordsOf = (one: object) => Object.keys(one).map((key) => aliases.get(key) ?? key); + // The objects an `@included` or `@nest` holds, each null where it holds no object. + const objects = (child: unknown) => members(child, aliases).map(plainObject); + const holdsNone = (child: unknown, ...keywords: string[]) => + objects(child).every( + (one) => one !== null && !keywordsOf(one).some((key) => keywords.includes(key)), + ); + const valid = ([key, child]: [string, unknown]): boolean => { + switch (aliases.get(key) ?? key) { + case "@id": + return typeof child === "string" || holdsObject(child); + case "@type": + return (Array.isArray(child) ? child : [child]).every((one) => typeof one === "string"); + case "@index": + return typeof child === "string"; + case "@reverse": { + const reverse = plainObject(child); + return reverse !== null && !keywordsOf(reverse).some((key) => key.startsWith("@")); + } + case "@included": + return holdsNone(child, "@value", "@list"); + case "@nest": + return holdsNone(child, "@value", "@id"); + default: + return true; + } + }; + const keywords = keywordsOf(object); + const ids = keywords.filter((keyword) => keyword === "@id").length; + const list = + keywords.includes("@list") && !keywords.every((key) => key === "@list" || key === "@index"); + if (Object.entries(object).every(valid) && ids <= 1 && !list) return; + throw new AppError("deckFileUnreadable", { + reason: "The file gives a JSON-LD keyword a value it may not have.", + }); +} + +/** + * The members of an `@included` or `@nest` value as the parser reads + * them: the elements of an array, and of the array a `@set` object + * holds (`aliases` resolved), one in the next as far as they go. A walk + * of its own, not a recursion, as for refuseRemoteContexts. + */ +function members(value: unknown, aliases: ReadonlyMap): unknown[] { + const found: unknown[] = []; + const pending = [value]; + while (pending.length > 0) { + const one = pending.pop(); + const object = plainObject(one) as Record | null; + const set = Object.keys(object ?? {}).find((key) => (aliases.get(key) ?? key) === "@set"); + if (Array.isArray(one)) for (const element of one) pending.push(element); + else if (set === undefined) found.push(one); + else pending.push(object![set]); + } + return found; +} + +/** The value as an object that is no array, or null. */ +function plainObject(value: unknown): object | null { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value : null; +} + /** Whether a value is an object, or an array that holds one. */ function holdsObject(value: unknown): boolean { const values = Array.isArray(value) ? value : [value]; @@ -511,16 +616,18 @@ function baseLength(baseUrl: string, declared: string[]): number { } /** - * Whether a context's value is a compact IRI whose prefix (the part - * before the first `:`) is a term of the node's contexts. The parser - * expands such a value through that term, and that term's through the - * next, with no bound on how deep, so two terms through each other (or - * one through itself) build an IRI without end. A file this app exports - * states its namespaces in full. + * Whether a context's value is a term of the node's contexts, or a + * compact IRI whose prefix (the part before the first `:`) is one. The + * parser expands such a value through that term, and that term's through + * the next, with no bound on how deep, so two terms through each other + * (or one through itself) build an IRI without end. A term so defined + * as one that names a keyword names that keyword too, which the aliases + * refuseContext gives would miss. A file this app exports states its + * namespaces in full. */ function chained(value: string, terms: Set): boolean { const colon = value.indexOf(":"); - return colon >= 0 && terms.has(value.slice(0, colon)); + return terms.has(value) || (colon >= 0 && terms.has(value.slice(0, colon))); } /**