Repository navigation
Expand file tree
/
Copy pathinstanceChecks.ts
More file actions
456 lines (447 loc) · 17.9 KB
/
Copy pathinstanceChecks.ts
File metadata and controls
456 lines (447 loc) · 17.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
// SPDX-FileCopyrightText: 2026 Anton Wiklund and Solid Memo contributors
//
// SPDX-License-Identifier: MIT
import { planRepair, type Repair, type RepairPlan } from "@solid-memo/domain/repair";
import type { Deck } from "@solid-memo/domain/deck";
import {
canonicalUrlOf,
catalogUrlOf,
historyUrlOf,
instanceDocumentUrls,
isDocumentIn,
} from "@solid-memo/domain/instanceLayout";
import {
summarize,
withUnclaimed,
type DocumentReport,
type UnclaimedDeck,
type ValidationReport,
} from "@solid-memo/domain/validation";
import { deckHealth, type DeckHealth, type DeckTextCheck } from "@solid-memo/domain/deckHealth";
import type { MarkdownFinding } from "@solid-memo/domain/release/problems";
import type { InstanceDigest } from "@solid-memo/domain/studyDigest";
import type {
DeckLibrary,
DeckRepository,
ShapeValidator,
RepairRepository,
DocumentContext,
ReadBudget,
ReleaseDraftRepository,
StorageGateway,
} from "./ports";
import type { AnswerStore } from "./answerStore";
import { AppError } from "@solid-memo/domain/appError";
import { documentUrlOf } from "@solid-memo/domain/subjectUrl";
import { TextMap } from "@solid-memo/domain/pathKeys";
import { mapAtOnce } from "./atOnce";
import { readBudget } from "./readBudget";
import { noting } from "./digestStore";
import type { DigestStore } from "./digestStore";
import { isInInstancePod } from "./releaseDrafts";
export interface InstanceCheckUseCases {
/**
* Developer tool: every document of the instance checked against
* Solid Memo's shapes (docs/validation.md). Reads only. `onChecked` is
* told how many documents are checked of how many, from none.
*/
validateInstance(
instanceUrl: string,
onChecked?: (count: number, of: number) => void,
): Promise<ValidationReport>;
/**
* The check made when an instance is opened: validateInstance, but a
* document still at the version the instance's digest says conformed
* is not checked (nor downloaded) again.
*/
checkInstance(instanceUrl: string): Promise<ValidationReport>;
/**
* Mark a deck's unclaimed documents (the report's `unclaimed`) as the
* deck's, once the user has confirmed they are: from then on they are
* written to and deleted as the deck's. Each in one write made only
* while it is as read; nothing else in it changes. deckGone when the
* instance has no such deck; claimOutsidePod, with nothing written,
* when any of them is outside the instance's folder and in no storage
* of the user's found to hold the instance too (UnclaimedDeck
* `outsidePod`).
*/
claimDeckDocuments(instanceUrl: string, deckUrl: string): Promise<void>;
/**
* Everything wrong with one deck of the instance (domain/deckHealth.ts):
* the check of its entry in the catalog and of its cards and reviews
* documents, made as checkInstance makes it (a document still at the
* version the digest says conformed is not checked again); the sides
* whose language is not stated, of the cards the user may settle
* (unknown when the release the deck was copied from cannot be read);
* the cards that say the same; and what `text` (the markdown package's
* plain text and check, which the caller passes in) finds in its text
* written in Markdown. Writes nothing but the digest's receipts.
*/
checkDeck<F extends MarkdownFinding>(
instanceUrl: string,
deck: Deck,
text: DeckTextCheck<F>,
): Promise<DeckHealth<F>>;
/** What the app can repair of what a check found, and what it leaves to the user. */
planRepair(report: ValidationReport): RepairPlan;
/**
* Apply repairs (the planned ones, or removals the user chose) to the
* instance's documents. A document that its read shows is not to be
* written is left as it is (RepairRepository).
*/
applyRepairs(instanceUrl: string, repairs: readonly Repair[]): Promise<void>;
}
/**
* A test of whether the document at an address is one of these decks'
* unclaimed ones, in whatever spelling: a deck may name one document
* twice, and only the first spelling is listed as unclaimed. Each
* address is compared as a request would send it (canonicalUrlOf), the
* unclaimed ones brought to that form once for the test, so that a
* catalog naming many decks costs one lookup per address, not a
* comparison with every unclaimed document.
*/
function unclaimedTest(unclaimed: readonly UnclaimedDeck[]): (url: string) => boolean {
// Keyed by pieces (TextMap): an address as a request sends it may be longer than V8 hashes all of.
const skipped = new TextMap<true>();
for (const deck of unclaimed) {
for (const document of deck.documents) {
const canonical = canonicalDocumentOf(document);
if (canonical !== null) skipped.set(canonical, true);
}
}
return (url) => {
const canonical = canonicalDocumentOf(url);
return canonical !== null && skipped.has(canonical);
};
}
/**
* A lookup of the decks naming a document as theirs, in whatever spelling
* of its address, each once: found once for all the documents, so that a
* catalog naming many decks costs one lookup per document.
*/
function decksByDocument(decks: readonly Deck[]): (url: string) => readonly Deck[] {
const naming = new TextMap<Deck[]>();
for (const deck of decks) {
const documents = [deck.cardsDocumentUrl, deck.reviewsDocumentUrl].map(canonicalDocumentOf);
for (const url of new Set(documents)) {
if (url === null) continue;
const listed = naming.get(url);
if (listed === undefined) naming.set(url, [deck]);
else listed.push(deck);
}
}
return (url) => {
const canonical = canonicalDocumentOf(url);
return (canonical === null ? undefined : naming.get(canonical)) ?? [];
};
}
/**
* How long one check may read, in all, the decks' documents outside
* their instance's folder, in milliseconds (readBudget): to find which
* are unclaimed, to check them and to read again one that does not
* conform. A catalog may name a thousand of them on servers that answer
* slowly, and every deck page waits on the check. The clock runs only
* while one of those reads is under way, so a slow read of the
* instance's own documents takes nothing from it.
*/
const OUTSIDE_READS_MS = 60_000;
/** The budget a read of the document is made within: the check's for one outside the instance's folder, none for one in it. */
function budgetFor(url: string, instanceUrl: string, budget: ReadBudget): ReadBudget | undefined {
return isDocumentIn(url, instanceUrl) ? undefined : budget;
}
/** The address of the document `url` is in, as a request sends it (canonicalUrlOf); null for one that is no URL. */
function canonicalDocumentOf(url: string): string | null {
return canonicalUrlOf(documentUrlOf(url));
}
/**
* The checks of an instance against Solid Memo's shapes (docs/validation.md):
* the quick one made when it is opened, which spares a document the digest
* says conformed and has not changed since, the full one of every document,
* and one deck's; the decks' unclaimed documents claimed once the user
* confirms they are the deck's; and what the app can repair of what a check
* found (domain/repair.ts).
*/
export function createInstanceCheckUseCases({
deckRepository,
releaseDraftRepository,
answerStore,
shapeValidator,
repairRepository,
deckLibrary,
digestStore,
storageGateway,
webId,
ruleset,
}: {
deckRepository: DeckRepository;
releaseDraftRepository: ReleaseDraftRepository;
answerStore: Pick<AnswerStore, "logMonths">;
shapeValidator: ShapeValidator;
repairRepository: RepairRepository;
deckLibrary: DeckLibrary;
digestStore: Pick<DigestStore, "of" | "remember" | "isInstances">;
storageGateway: Pick<StorageGateway, "discoverStorages">;
/** The WebID of whoever is signed in, asked at each call; none when no one is. */
webId: () => string | undefined;
ruleset: string;
}): InstanceCheckUseCases {
async function checkInstance(instanceUrl: string): Promise<ValidationReport> {
const [decks, drafts, digest] = await Promise.all([
deckRepository.listDecks(instanceUrl),
releaseDraftRepository.documents(instanceUrl),
digestStore.of(instanceUrl),
]);
const budget = readBudget(OUTSIDE_READS_MS);
const unclaimed = await unclaimedDecksOf(instanceUrl, decks, budget);
return withUnclaimed(
await checkDocuments(
instanceUrl,
documentsToCheck(instanceUrl, decks, drafts, unclaimed),
digest,
decks,
budget,
),
unclaimed,
);
}
/**
* A test of which documents may not be claimed for a deck of the
* instance, giving those of `documents` outside its folder and in no
* storage of the user's that holds it too (isInInstancePod): a claim
* lets the app write to a document as the user and delete from it, and
* an instance attached from someone else's pod may name a document of
* the user's own. A storage counts only when it can be found; when none
* can, or no one is signed in, only the instance's own folder does.
* The user's storages are found once for one test.
*/
function outsidePodTest(
instanceUrl: string,
): (documents: readonly string[]) => Promise<string[]> {
const user = webId();
let found: ReturnType<StorageGateway["discoverStorages"]> | undefined;
const storages: Pick<StorageGateway, "discoverStorages"> = {
discoverStorages: (id) => (found ??= storageGateway.discoverStorages(id)),
};
return async (documents) => {
const inside = await Promise.all(
documents.map(async (url) =>
user === undefined
? isDocumentIn(url, instanceUrl)
: isInInstancePod(storages, instanceUrl, url, user),
),
);
return documents.filter((_, index) => !inside[index]);
};
}
/**
* The decks whose documents are unclaimed, each with those documents
* (DeckRepository.unclaimedDocuments) and which of them may not be
* claimed (outsidePodTest). The documents outside the instance's
* folder, which a catalog may name on anyone's server, are read within
* the check's `budget` (OUTSIDE_READS_MS): one not read by then counts
* as unclaimed, so a server that answers slowly holds the check up no
* longer.
*/
async function unclaimedDecksOf(
instanceUrl: string,
decks: readonly Deck[],
budget: ReadBudget,
): Promise<UnclaimedDeck[]> {
const outsidePodOf = outsidePodTest(instanceUrl);
const found = await mapAtOnce(decks, async (deck) => {
const documents = await deckRepository.unclaimedDocuments(deck, budget);
return {
deckUrl: deck.url,
title: deck.title,
documents,
outsidePod: await outsidePodOf(documents),
};
});
return found.filter((deck) => deck.documents.length > 0);
}
/**
* The instance's documents, then its drafts': each with where it is,
* which picks its shapes. An unclaimed document is not checked, in any
* spelling of its address (unclaimedTest): it may be anyone's, and a
* repair of what the check finds writes to it.
*/
function documentsToCheck(
instanceUrl: string,
decks: readonly Deck[],
drafts: readonly string[],
unclaimed: readonly UnclaimedDeck[],
): { url: string; context: Exclude<DocumentContext, "library"> }[] {
const isUnclaimed = unclaimedTest(unclaimed);
return [
...instanceDocumentUrls(instanceUrl, decks)
.filter((url) => !isUnclaimed(url))
.map((url) => ({ url, context: "pod" as const })),
...drafts.map((url) => ({ url, context: "draft" as const })),
];
}
/**
* The check of these documents of the instance, each not checked again
* while still at the version the digest says conformed. A receipt is
* kept only of one that is the instance's (DigestStore.isInstances).
* A document that cannot be read is in the report as one
* (ShapeValidator), so it holds back only what it is part of; so is one
* outside the instance's folder not read within the check's `budget`.
*/
async function checkDocuments(
instanceUrl: string,
urls: readonly { url: string; context: Exclude<DocumentContext, "library"> }[],
digest: InstanceDigest,
decks: readonly Deck[],
budget: ReadBudget,
): Promise<ValidationReport> {
// A few at a time: the catalog says how many there are (MAX_CATALOG_LINKS of each kind at most).
const documents = await mapAtOnce(urls, async ({ url, context }): Promise<DocumentReport> => {
const receipt = digest.receipts[url];
const since = await shapeValidator.validateDocumentSince(
url,
receipt?.conformedTo === ruleset ? receipt.version : undefined,
context,
budgetFor(url, instanceUrl, budget),
);
if (since.unchanged) return { url, status: "checked", subjects: [] };
// A receipt only of a document that is the instance's: one an instance from another's pod names may be the user's own.
if (
since.version !== null &&
summarize(instanceUrl, [since.value]).conforms &&
(await digestStore.isInstances(instanceUrl, url))
) {
digestStore.remember(instanceUrl, noting(url, since.version, { conformedTo: ruleset }));
}
return since.value;
});
return summarize(
instanceUrl,
await withoutUnclaimedOnRead(instanceUrl, decks, documents, budget),
);
}
/**
* The reports without those of a deck's document that its read shows
* is unclaimed for every deck naming it (isUnclaimedAsRead): a repair
* or removal of what the check found would write to a document that
* is no deck's to change, a release in the instance's folder say. Only
* a report with something to repair needs that read, so a check of an
* unchanged instance still downloads nothing. One outside the
* instance's folder is read within the check's `budget`, and its report
* kept when it is not.
*/
async function withoutUnclaimedOnRead(
instanceUrl: string,
decks: readonly Deck[],
documents: readonly DocumentReport[],
budget: ReadBudget,
): Promise<DocumentReport[]> {
const decksOf = decksByDocument(decks);
const left = await mapAtOnce(documents, async (document) => {
const naming = decksOf(document.url);
return (
naming.length > 0 &&
!summarize(instanceUrl, [document]).conforms &&
(await deckRepository.isUnclaimedAsRead(
naming,
document.url,
budgetFor(document.url, instanceUrl, budget),
))
);
});
return documents.filter((_, index) => !left[index]);
}
/** Every document of the instance, its answer log's too: not checked on every visit, being large and growing. */
async function validateInstance(
instanceUrl: string,
onChecked: (count: number, of: number) => void = () => undefined,
): Promise<ValidationReport> {
const [decks, months, drafts] = await Promise.all([
deckRepository.listDecks(instanceUrl),
answerStore.logMonths(instanceUrl),
releaseDraftRepository.documents(instanceUrl),
]);
const budget = readBudget(OUTSIDE_READS_MS);
const unclaimed = await unclaimedDecksOf(instanceUrl, decks, budget);
const urls = [
...documentsToCheck(instanceUrl, decks, drafts, unclaimed),
...months.map((month) => ({
url: historyUrlOf(instanceUrl, month),
context: "pod" as const,
})),
];
let checked = 0;
onChecked(checked, urls.length);
const documents = await mapAtOnce(urls, async ({ url, context }) => {
const document = await shapeValidator.validateDocument(
url,
context,
budgetFor(url, instanceUrl, budget),
);
onChecked(++checked, urls.length);
return document;
});
return withUnclaimed(
summarize(instanceUrl, await withoutUnclaimedOnRead(instanceUrl, decks, documents, budget)),
unclaimed,
);
}
return {
validateInstance,
checkInstance,
async claimDeckDocuments(instanceUrl, deckUrl) {
const deck = (await deckRepository.listDecks(instanceUrl)).find(
(listed) => listed.url === deckUrl,
);
if (deck === undefined) throw new AppError("deckGone", { deck: deckUrl });
const documents = await deckRepository.unclaimedDocuments(deck);
const [outside] = await outsidePodTest(instanceUrl)(documents);
if (outside !== undefined)
throw new AppError("claimOutsidePod", { deck: deck.title, url: outside });
await deckRepository.claimDocuments(deck);
},
async checkDeck(instanceUrl, deck, text) {
const budget = readBudget(OUTSIDE_READS_MS);
const [report, cards, release] = await Promise.all([
Promise.all([
digestStore.of(instanceUrl),
unclaimedDecksOf(instanceUrl, [deck], budget),
]).then(async ([digest, unclaimed]) => {
const isUnclaimed = unclaimedTest(unclaimed);
const urls = [deck.cardsDocumentUrl, deck.reviewsDocumentUrl].filter(
(url) => !isUnclaimed(url),
);
return withUnclaimed(
await checkDocuments(
instanceUrl,
[catalogUrlOf(instanceUrl), ...urls].map((url) => ({
url,
context: "pod" as const,
})),
digest,
[deck],
budget,
),
unclaimed,
);
}),
// Read as the deck's page reads it, within the read's own deadlines: it holds back this deck's health alone.
deckRepository.listCards(deck),
// A release that cannot be read leaves the sides to settle unknown, not the rest of the check.
deck.sourceUrl === undefined
? []
: deckLibrary.fetchLibraryDeck(deck.sourceUrl).then(
(release) => release.cards,
() => undefined,
),
]);
return deckHealth(deck, report, cards, release, text);
},
planRepair(report) {
return planRepair(report);
},
async applyRepairs(instanceUrl, repairs) {
const decks = await deckRepository.listDecks(instanceUrl);
return repairRepository.applyRepairs(repairs, decksByDocument(decks));
},
};
}