From 3259c3fb31f423d562e970f5843d07673d08e2c8 Mon Sep 17 00:00:00 2001 From: raiseCatError <315733358+raiseCatError@users.noreply.github.com> Date: Thu, 22 Oct 2026 02:56:25 +0530 Subject: [PATCH 1/2] Problems: offline static diagnostics for Markdown, HTML and CSS (#466) --- CHANGELOG.md | 1 + docs/design/document-workspace.md | 12 + docs/development/v024-ledger.md | 2 + docs/testing/v024-editor-physical-qa.md | 6 + src/app/TerminalApp.ts | 23 +- src/markdown/diagnostics.ts | 422 ++++++++++++++++++++++++ src/markdown/sourceMap.ts | 5 + src/viewer/DocumentWorkspace.ts | 13 +- src/viewer/EditorPalette.ts | 6 +- src/viewer/EditorPanel.ts | 100 +++++- src/viewer/problems.ts | 108 ++++++ tests/diagnostics.test.ts | 146 ++++++++ tests/problems.test.ts | 230 +++++++++++++ 13 files changed, 1062 insertions(+), 12 deletions(-) create mode 100644 src/markdown/diagnostics.ts create mode 100644 src/viewer/problems.ts create mode 100644 tests/diagnostics.test.ts create mode 100644 tests/problems.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b361e74..c02c1ca5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added - **Hybrid Markdown editing** ([#460](https://github.com/raiseCatError/notMyShell/issues/460)). **F3** in a Markdown document (or `H` from the read-only views) shows it *rendered* — headings, emphasis, links, lists, tables, quotes, code — except for the block your caret (or selection) is in, which shows its exact Markdown source and is fully editable. Move on and it renders again. It is another presentation of the same buffer: every key is the editor's (letters always type; snippets, list/table helpers, undo/redo, hunk review, recovery drafts and Save As all work as in the source editor), the caret lives in source positions, and switching views never changes the text. The smallest safe block is revealed — the whole list for a nested item, the whole table, the whole fence — and a selection shows every block it touches as source. The view is anchored to a source line, so revealing a block does not move what is above it. F3 again returns to the source editor; in the split view (F2) F3 switches the text pane between source and hybrid. Limits: Markdown files only (not HTML); images appear as `[image: alt]`; Up/Down move by source line; pointer input is not handled. +- **Problems: offline static diagnostics** ([#466](https://github.com/raiseCatError/notMyShell/issues/466)). In a Markdown, HTML or CSS document, **F1 then `e`** (the snippet palette's *Problems…* in HTML files; **F1** in a CSS file; **`P`** in the read-only views and the split's preview pane) lists what is wrong: ✖ errors, ▲ warnings, ℹ notes and `?` things that cannot be checked. Markdown: links to files that do not exist, `#fragments` with no heading or id, missing images, images without alt text, undefined `[text][ref]` references and footnotes, unused footnotes, repeated headings. HTML: missing images, stylesheets and scripts, `` without `alt`, form controls without a label, duplicate ids, unresolved `#fragments`, unclosed or unexpected tags. CSS: missing `url()` assets and unbalanced braces. Type to search, Tab filters (all, problems, errors), Enter jumps to the exact text and selects it (Back returns), Del dismisses one, Ctrl+R checks again. Checking starts when you first open Problems, follows the text (typing, undo, snippets, restored drafts, a merge) after a short pause, drops results for text that has since changed, and counts show in the editor's status row. Everything is offline: external URLs are counted, never fetched; no script, style or command is run; files are only looked at (never read except another Markdown file's headings, 256 KiB at most) and only inside the document's project (a path that leaves it, by `..` or a symlink, is reported as not checked); nothing is written. Browser-console diagnostics are a later, separate step. - **Footnote renumbering** ([#463](https://github.com/raiseCatError/notMyShell/issues/463)). *Renumber footnotes* in the snippet palette (F1, then `/` in a Markdown file) numbers the footnotes 1, 2, 3… in the order they are first referenced and updates each definition to match, as one undo step. Named footnotes (`[^note]`), escaped references, code spans and fenced code are left alone; a definition nobody references keeps its number unless a renumbered footnote needs it. - **Split view for Markdown and HTML documents** ([#461](https://github.com/raiseCatError/notMyShell/issues/461)). **F2** (or `s` in the read-only views) in the document workspace shows the editor beside NMSh's own preview of the *same* buffer: the text on the left, the rendered document on the right, updated as you type, without saving. The preview follows the caret to its block (`l` in the preview pane turns that off), **Ctrl+O** moves between the panes, `x` swaps the sides and `<` `>` change their widths (30–70%). Only the text pane is editable; keys typed while the preview has the focus never reach the text. The preview pane has its own find (`/`) and heading prompt (`#`). Below 100 columns only the focused pane is shown at full width, and widening the terminal brings the split back. Esc leaves for the read-only source view with your unsaved text kept; F2 again returns to editing. The browser-quality preview is not used in the split; the pane is NMSh's renderer. Mouse input in the split is not handled yet. - **Heading navigation** ([#462](https://github.com/raiseCatError/notMyShell/issues/462), first slice). In a Markdown or HTML document, **`#`** in the viewer or workspace opens a heading prompt: type to narrow, Up/Down choose, Enter jumps (source or rendered view, any width) and **Backspace** goes back to where you were. In the editor, F1 then `h` (or *Go to heading…* in the snippet palette) lists the headings with their depth; `j` / *Back* returns to the caret position you left, even after edits. `/view README.md#install` opens the file at that heading. Anchors follow GitHub's rules (lower-case, punctuation dropped, `-1`, `-2` for repeats; raw HTML headings listed without an anchor); Jekyll/Kramdown ids are different and are not implemented (see the [architecture note](docs/architecture/document-source-map.md)). Underneath, every parsed Markdown block now knows its source lines and every rendered row its block's line, the shared map the coming split view and hybrid editing build on. diff --git a/docs/design/document-workspace.md b/docs/design/document-workspace.md index 93dbbecd..1b4a2ee2 100644 --- a/docs/design/document-workspace.md +++ b/docs/design/document-workspace.md @@ -51,6 +51,18 @@ Limits: code fences are recognised by a line scan (indented code blocks and fenc **Verified by** `tests/markdownEdit.test.ts` (each helper: empty and multi-line selections, nested and mixed lists, quotes, fences, Unicode, very long lines, one undo step, restored drafts) and `tests/markdownEditorKeys.test.ts` (keys, menu at 30–120 columns, non-Markdown files unchanged, CRLF saved as CRLF). Physical Ghostty/Zed: NOT RUN. +## Problems (static diagnostics) + +`src/markdown/diagnostics.ts` (engine), `src/viewer/problems.ts` (runner) and the *problems* dialog of `EditorPanel.ts` ([#466](https://github.com/raiseCatError/notMyShell/issues/466)). Reuses `parseBlockLines` (code blocks are skipped through the block spans), `headingsFromLines` and `resolveFragment`; no second parser, no language server. + +- **Severities:** `error` (certainly broken: a file that is not there, an undefined reference/footnote/id), `warning` (likely: a missing alt text, an unclosed tag, a fragment other generators might satisfy — a document with raw `id=`/`{#…}` ids gets warnings instead of errors), `info` (a repeated heading), `unverified` (not checked, not claimed wrong: absolute and `~` paths, a path outside the project, a templated attribute, a relative file in a document with no folder yet). External URLs (`http`, `mailto`, `//`, any other scheme) are only counted. `javascript:`, `vbscript:`, `file:` and non-image `data:` links are reported as never followed. +- **Positions:** every finding has a 0-based line and, when it names exact text (a link destination, an attribute value, an id), UTF-16 columns; Enter selects that text. Without a span the caret goes to the start of the line and nothing pretends otherwise. +- **Filesystem:** only through `LocalFs`, created per run for the document's folder with the project root (`documentRoot`: the git project within a dozen levels, else the folder). A path is lexically resolved, then `realpath`ed, and must stay inside the real root; for a missing path the deepest existing ancestor is checked, so a symlinked directory cannot be used to probe outside. Only `lstat` and one bounded read of another Markdown file (headings only). At most 400 distinct paths per run; at most 500 findings; HTML over 1.5 MB is skipped and marked truncated; messages never name a path the document did not itself write. +- **Currency:** checking is off until Problems is first opened. Then `ProblemsRunner` observes the buffer: each change bumps a revision and schedules a run (500 ms); a run snapshots the lines, aborts any run in flight, and its result is dropped if the revision moved. Opening Problems, Ctrl+R, a save and a Save As re-check (files the document names may have changed; no watchers). The runner is disposed with the editor, on reload and on exit, so no timer or observer outlives the document. +- **Surface:** the dialog lists findings by severity then line with counts and the number of external links not checked; Tab cycles all / problems / errors, typing filters on the message, Del dismisses (until the next report), Esc leaves. In source and preview a jump also scrolls the read-only view to the line; hybrid and the split follow the caret. +- **Not provided:** a gutter marker in the text, checks of other languages, CSS property validation, link fragments inside another HTML file, following redirects or fetching anything, quick fixes, and the browser console (a separate, gated slice). +- **Verified by** `tests/diagnostics.test.ts` (each check, hostile paths and symlinks, the filesystem checker, no-folder documents, Unicode and stray CR, HTML and CSS, false-positive stress, bounds) and `tests/problems.test.ts` (runner staleness and disposal, the surface, filters, dismiss, refresh, widths/NO_COLOR/Safe Glyphs, and every mode: source, preview, hybrid, split, edit). + ## Hybrid view `DocumentMode` gains `'hybrid'` (#460), implemented in `src/viewer/HybridView.ts` over the shared source map (`docs/architecture/document-source-map.md`). It is a presentation of the one `TextBuffer`; there is no second editor and no second document. diff --git a/docs/development/v024-ledger.md b/docs/development/v024-ledger.md index 9a46cf2d..926f5225 100644 --- a/docs/development/v024-ledger.md +++ b/docs/development/v024-ledger.md @@ -44,6 +44,8 @@ The paused issues #460–#469 are the milestone (the maintainer superseded the e 8. **#460: hybrid Markdown editing** — branch `feature/v024-hybrid` (from `dev` after #489–#496 merged). Done: hybrid mode, items from the source map, active-block raw editing, anchored viewport, split text-pane toggle, tests, independent review (see its section). **#460 remaining:** images in place, visual-line movement, pointer input, tables as a grid, HTML. **#461** still lacks pointer input, preview→caret sync, vertical orientation and the Chromium preview; **#463** footnote renumbering; **#466**, **#462** (details/Mermaid/math), **#464/#465/#467** not started. 9. **#463: footnote renumbering** — branch `feature/v024-footnotes`, from `dev`. Done (tests in `footnotes.test.ts`). **#463 acceptance:** every implemented item is covered by automated tests; the only open requirement is the physical Ghostty/Zed pass (NOT RUN), so the issue stays open. +10. **#466: offline static diagnostics** — branch `feature/v024-diagnostics`, from `dev`. Done: engine, runner, Problems surface in all modes, tests, independent review (see its section in the design doc). **#466 remaining:** the optional browser console for trusted previews (separate gated slice, nothing started), a gutter marker, further languages. **#462 next:** `
` disclosure, then Mermaid and math (separate slices). + ## Environment notes Scratch and test runs: `t7run`; focused tests via `/Volumes/T7Tmp/ai-dev/runtests.mjs` (see the testing-gotchas memory). `npm ci` was needed in a fresh worktree (the main checkout's `node_modules` is stale). diff --git a/docs/testing/v024-editor-physical-qa.md b/docs/testing/v024-editor-physical-qa.md index b3e783f6..1ee44e03 100644 --- a/docs/testing/v024-editor-physical-qa.md +++ b/docs/testing/v024-editor-physical-qa.md @@ -21,6 +21,12 @@ Automated tests cover the logic headlessly. Everything below is **NOT RUN** unti - [ ] Mermaid and math entries say they are source text; nothing in the preview claims to render them. - [ ] The palette rows and footer fit 30, 50, 80 and 120 columns; NO_COLOR and Safe Glyphs stay legible. +## Problems (#466) +- [ ] F1 then `e` in a Markdown file with a broken link, a missing image and a bad `#fragment`: the list shows them; Enter selects the exact text; Back returns. `P` in the read-only views and in the split's preview pane. +- [ ] An HTML file (palette → Problems) and a CSS file (F1); the status row counts; fixing a finding removes it after a moment; undo brings it back. +- [ ] A symlink out of the project and `/etc/passwd`-style paths are reported as not checked; an external URL is never fetched (watch the network). +- [ ] 30, 50, 80 and 120 columns; NO_COLOR; Safe Glyphs (E/W/i/?). + ## Hybrid view (#460) - [ ] Open a Markdown file, F3 (is F3 delivered by the terminal and by Zed?): headings, bold, links, lists, tables, quotes and fences render; the block under the caret shows its source; moving with the arrows swaps blocks without the screen jumping. - [ ] Type in every kind of block; select across blocks with Shift+Down (both show source); undo/redo; insert a snippet and Tab through it. diff --git a/src/app/TerminalApp.ts b/src/app/TerminalApp.ts index 697dfd06..b9bff7d0 100644 --- a/src/app/TerminalApp.ts +++ b/src/app/TerminalApp.ts @@ -80,6 +80,7 @@ import {createEditor, editorKey, offerDraft, renderEditor, type EditorAction, ty import {createScratchFile, openForEdit, saveFile} from '../viewer/fileEdit.js'; import {headingsOf, resolveFragment} from '../markdown/sourceMap.js'; import {DraftKeeper, DraftStore, type DraftOffer} from '../viewer/draftRecovery.js'; +import {ProblemsRunner} from '../viewer/problems.js'; import {applyReview, beginReview, restartReview, sameDiskVersion} from '../viewer/ReviewPanel.js'; import {bufferText, createDocumentSession, renderDocumentSession, setDocumentMode, syncDocumentViewer, documentSessionKey, documentSessionLook, type DocumentSessionAction, type DocumentMode, type DocumentSession} from '../viewer/DocumentWorkspace.js'; import {formatForPath} from '../viewer/document.js'; @@ -1063,6 +1064,8 @@ export class TerminalApp { private finish!: (exitCode: number) => void; constructor(connection?: SessionConnection, preset?: SessionPreset) { + // A finished check redraws the panel that shows its counts. + ProblemsRunner.notify = () => { if (!this.stopped) this.render(); }; if (preset) { if (!connection || connection.mode !== 'service' || connection.attached) throw new Error('Presets require a new live session.'); this.presetStartup = new PresetStartup(preset); @@ -4601,6 +4604,7 @@ export class TerminalApp { this.drafts().discard(editor); editor.file = saved.file; if (editor.buffer.lines.length === lines.length && editor.buffer.lines.every((line, index) => line === lines[index])) editor.buffer.markSaved(); + editor.problems?.refresh(); editor.note = {text: `Saved as ${saved.file.title}.`, tone: 'success'}; if (session) { session.viewer.note = editor.note.text; syncDocumentViewer(session, session.mode === 'preview'); } if (thenClose) close(); @@ -4614,6 +4618,13 @@ export class TerminalApp { })(); } + /** An editor is going away (closed, reloaded, replaced): its recovery draft goes and its checking and parsed index stop. */ + private releaseEditor(editor: EditorState): void { + this.drafts().discard(editor); + editor.problems?.dispose(); + editor.index?.dispose(); + } + private runDraftAction(editor: EditorState, action: Extract): void { const keeper = this.drafts(); if (action.kind === 'draft') keeper.restored(editor, action.target); @@ -4624,11 +4635,11 @@ export class TerminalApp { /** The editor's requests that touch the disk or the clipboard; the panel shows the outcome. */ private runEditorAction(editor: EditorState, action: EditorAction): void { if (action.kind === 'draft' || action.kind === 'draft-discard' || action.kind === 'draft-keep') { this.runDraftAction(editor, action); return; } - if (action.kind === 'playground') { this.drafts().discard(editor); this.textEditor = undefined; void this.openPlayground(action.choice); return; } + if (action.kind === 'playground') { this.releaseEditor(editor); this.textEditor = undefined; void this.openPlayground(action.choice); return; } if (action.kind === 'save-as') { this.saveAs(editor, action.path, Boolean(action.thenClose), () => this.textEditor === editor, () => { this.textEditor = undefined; this.returnFromPanel(); }); return; } if (editor.file.scratch && (action.kind === 'review' || action.kind === 'review-apply' || action.kind === 'reload' || action.kind === 'save')) { editor.note = {text: 'A Playground document has no file yet: Save As creates one.', tone: 'info'}; return; } if (action.kind === 'review' || action.kind === 'review-apply') { this.reviewDisk(editor, action.kind === 'review-apply', () => this.textEditor === editor); return; } - if (action.kind === 'close') { this.drafts().discard(editor); this.textEditor = undefined; this.returnFromPanel(); return; } + if (action.kind === 'close') { this.releaseEditor(editor); this.textEditor = undefined; this.returnFromPanel(); return; } if (action.kind === 'copy') { void this.copyText(action.text, 'Copied').then(note => { if (note && this.textEditor === editor) { editor.note = {text: note, tone: 'success'}; this.render(); } }); return; @@ -4654,8 +4665,8 @@ export class TerminalApp { // Only what was written is clean: typing during the save stays unsaved. if (editor.buffer.lines.length === lines.length && editor.buffer.lines.every((line, index) => line === lines[index])) editor.buffer.markSaved(); this.drafts().note(editor); - if (action.thenClose) { this.drafts().discard(editor); this.textEditor = undefined; this.returnFromPanel(); } - else editor.note = {text: `Saved ${editor.file.title}.`, tone: 'success'}; + if (action.thenClose) { this.releaseEditor(editor); this.textEditor = undefined; this.returnFromPanel(); } + else { editor.problems?.refresh(); editor.note = {text: `Saved ${editor.file.title}.`, tone: 'success'}; } } else if (result.conflict) { editor.dialog = {kind: 'conflict', focus: 0, conflict: result.conflict, reason: result.reason, ...(action.thenClose ? {thenClose: true} : {})}; } else editor.note = {text: `${result.reason} Your text is still here.`, tone: 'error'}; @@ -4750,7 +4761,7 @@ export class TerminalApp { const session = this.docSession; if (!session) return; this.docSession = undefined; - this.drafts().discard(session.editor); + this.releaseEditor(session.editor); session.preview?.dispose(); this.releaseViewerImages(); this.returnFromPanel(); @@ -4802,6 +4813,7 @@ export class TerminalApp { if (editor.buffer.lines.length === lines.length && editor.buffer.lines.every((line, index) => line === lines[index])) editor.buffer.markSaved(); if (action.thenClose) { this.closeDocumentSession(); this.render(); return; } this.drafts().note(editor); + editor.problems?.refresh(); editor.note = {text: `Saved ${editor.file.title}.`, tone: 'success'}; session.viewer.note = `Saved ${editor.file.title}.`; syncDocumentViewer(session, session.mode === 'preview'); @@ -13965,6 +13977,7 @@ export class TerminalApp { this.docSession?.preview?.dispose(); // Unsaved editor text is written before the window ends, so closing it never costs the text. this.draftKeeper?.dispose(); + this.textEditor?.problems?.dispose(); this.docSession?.editor.problems?.dispose(); void this.browserHost?.shutdown(); this.releaseViewerImages(); this.graphics?.releaseAll(); diff --git a/src/markdown/diagnostics.ts b/src/markdown/diagnostics.ts new file mode 100644 index 00000000..75dff080 --- /dev/null +++ b/src/markdown/diagnostics.ts @@ -0,0 +1,422 @@ +import {lstat, open, realpath} from 'node:fs/promises'; +import {dirname, isAbsolute, relative, resolve, sep} from 'node:path'; +import {parseBlockLines, type Block} from './blocks.js'; +import {headingsFromLines, resolveFragment} from './sourceMap.js'; + +/** + * Offline static diagnostics for Markdown, HTML and CSS documents (#466). Everything here reads the document's own lines (the + * editor's buffer, saved or not) and, only through an injected `LocalFs`, checks that files the document names exist inside + * the document's folder or project. It never fetches a URL, runs a script or a command, follows a link out of the root, + * writes anything or reports a guess as a fact: + * + * error a reference that is certainly broken (the file is not there, the id/footnote/reference is not defined) + * warning likely a problem (a missing alt text, an unclosed tag, a fragment that Jekyll ids might satisfy) + * info worth knowing (a repeated heading) + * unverified could not be checked, and is not claimed to be wrong (an absolute path, a path outside the project, a templated + * attribute, a relative file of a document that has no folder yet) + * + * External URLs are only counted. Positions are source lines and UTF-16 columns of the exact text, so a jump can select it. + */ + +export type Severity = 'error' | 'warning' | 'info' | 'unverified'; + +export interface Diagnostic { + id: string; + code: string; + severity: Severity; + message: string; + /** 0-based source line. */ + line: number; + column?: number; + endColumn?: number; + /** The span is exactly the text at fault; otherwise only the line is known. */ + exact: boolean; +} + +export interface DiagnosticReport { + diagnostics: Diagnostic[]; + /** External references seen and deliberately not checked. */ + external: number; + /** Filesystem checks made. */ + checked: number; + /** Stopped early: too many findings or checks. */ + truncated: boolean; + aborted?: boolean; +} + +export type FileState = 'file' | 'directory' | 'missing' | 'unreadable' | 'outside'; + +export interface LocalFs { + /** The state of a path written in the document (already percent-decoded, relative to the document's folder). */ + stat(path: string): Promise; + /** Up to `maxBytes` of a regular file inside the root, for checking a heading fragment in another Markdown file. */ + readText(path: string, maxBytes: number): Promise; +} + +export const MAX_DIAGNOSTICS = 500; +export const MAX_CHECKS = 400; +const MAX_PATH = 1024; +const MAX_TARGET_BYTES = 256 * 1024; + +/** A checker confined to `root`: lexical `..` and symlinks are both resolved before a path is called inside. */ +export function createLocalFs(docDir: string, root: string): LocalFs { + let realRoot: Promise | undefined; + const rootReal = () => realRoot ??= realpath(root).catch(() => root); + const inside = (base: string, path: string) => { const rel = relative(base, path); return rel === '' || (!rel.startsWith(`..${sep}`) && rel !== '..' && !isAbsolute(rel)); }; + async function locate(path: string): Promise<{state: FileState; real?: string}> { + if (!path || path.length > MAX_PATH || path.includes('\0')) return {state: 'unreadable'}; + const resolved = resolve(docDir, path); + if (!inside(root, resolved)) return {state: 'outside'}; + const base = await rootReal(); + try { + const real = await realpath(resolved); + if (!inside(base, real)) return {state: 'outside'}; + const info = await lstat(real); + return {state: info.isDirectory() ? 'directory' : info.isFile() ? 'file' : 'unreadable', real}; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === 'EACCES' || code === 'EPERM') return {state: 'unreadable'}; + if (code !== 'ENOENT' && code !== 'ENOTDIR') return {state: 'unreadable'}; + // Missing: but a symlinked folder on the way may already lead out of the root. + for (let ancestor = dirname(resolved); ancestor !== dirname(ancestor); ancestor = dirname(ancestor)) { + try { const real = await realpath(ancestor); return {state: inside(base, real) ? 'missing' : 'outside'}; } catch { /* keep climbing */ } + } + return {state: 'missing'}; + } + } + return { + stat: async path => (await locate(path)).state, + async readText(path, maxBytes) { + const found = await locate(path); + if (found.state !== 'file' || !found.real) return undefined; + let handle; + try { handle = await open(found.real, 'r'); const buffer = Buffer.alloc(maxBytes); const { bytesRead } = await handle.read(buffer, 0, maxBytes, 0); return buffer.subarray(0, bytesRead).toString('utf8'); } + catch { return undefined; } finally { await handle?.close().catch(() => undefined); } + }, + }; +} + +// ---- Destinations ----------------------------------------------------------------------------------------------- + +type Target = + | {kind: 'fragment'; fragment: string} + | {kind: 'local'; path: string; fragment?: string} + | {kind: 'external'} + | {kind: 'unsafe'; scheme: string} + | {kind: 'absolute'} + | {kind: 'ignore'}; + +function classify(destination: string): Target { + const text = destination.trim(); + if (!text) return {kind: 'ignore'}; + if (text.startsWith('#')) return text.length > 1 ? {kind: 'fragment', fragment: text.slice(1)} : {kind: 'ignore'}; + const scheme = /^([a-zA-Z][a-zA-Z0-9+.-]*):/u.exec(text)?.[1]?.toLowerCase(); + if (scheme) { + if (['javascript', 'vbscript', 'file'].includes(scheme)) return {kind: 'unsafe', scheme}; + if (scheme === 'data') return /^data:image\//iu.test(text) ? {kind: 'ignore'} : {kind: 'unsafe', scheme}; + if (scheme.length === 1) return {kind: 'absolute'}; + return {kind: 'external'}; + } + if (text.startsWith('//')) return {kind: 'external'}; + if (text.startsWith('/') || text.startsWith('~') || text.startsWith('\\')) return {kind: 'absolute'}; + const hash = text.indexOf('#'); + const withoutFragment = hash >= 0 ? text.slice(0, hash) : text; + const fragment = hash >= 0 ? text.slice(hash + 1) : undefined; + const path = withoutFragment.replace(/\?.*$/u, ''); + if (!path) return fragment ? {kind: 'fragment', fragment} : {kind: 'ignore'}; + let decoded = path; + try { decoded = decodeURIComponent(path); } catch { /* keep it as written */ } + return {kind: 'local', path: decoded, ...(fragment ? {fragment} : {})}; +} + +const TEMPLATED = /\{\{|\{%|<%|\$\{|\{#/u; + +// ---- The run ---------------------------------------------------------------------------------------------------- + +export interface DiagnoseInput { + lines: readonly string[]; + format: 'markdown' | 'html' | 'css'; + fs?: LocalFs; + signal?: AbortSignal; + maxChecks?: number; +} + +class Run { + diagnostics: Diagnostic[] = []; + external = 0; + truncated = false; + private readonly states = new Map>(); + private ids = 0; + constructor(readonly input: DiagnoseInput) {} + get aborted() { return Boolean(this.input.signal?.aborted); } + get checked() { return this.states.size; } + + add(code: string, severity: Severity, message: string, line: number, column?: number, endColumn?: number): void { + if (this.diagnostics.length >= MAX_DIAGNOSTICS) { this.truncated = true; return; } + this.diagnostics.push({id: `${code}:${line}:${column ?? ''}:${this.ids++}`, code, severity, message, line, exact: column !== undefined && endColumn !== undefined, ...(column === undefined ? {} : {column}), ...(endColumn === undefined ? {} : {endColumn})}); + } + + stat(path: string): Promise | undefined { + const known = this.states.get(path); + if (known) return known; + if (!this.input.fs || this.states.size >= (this.input.maxChecks ?? MAX_CHECKS)) { if (this.input.fs) this.truncated = true; return undefined; } + const pending = this.input.fs.stat(path); + this.states.set(path, pending); + return pending; + } + + /** Report a destination written in the document. `kindOfAsset`: a missing file is an error either way; the wording differs. */ + async reference(destination: string, line: number, column: number, headings: ReturnType | undefined, ids: ReadonlySet | undefined, softFragments: boolean): Promise { + const end = column + destination.length; + const target = classify(destination); + switch (target.kind) { + case 'ignore': return; + case 'external': this.external += 1; return; + case 'unsafe': this.add('unsafe-scheme', 'warning', `A ${target.scheme}: link is never followed or checked here.`, line, column, end); return; + case 'absolute': this.add('absolute-path', 'unverified', 'Absolute and home-relative paths are not checked.', line, column, end); return; + case 'fragment': { + const found = headings ? resolveFragment(headings, target.fragment) : undefined; + let decoded = target.fragment; + try { decoded = decodeURIComponent(decoded); } catch { /* as written */ } + if (found || ids?.has(decoded) || ids?.has(target.fragment) || decoded.toLowerCase() === 'top') return; + this.add('broken-fragment', softFragments ? 'warning' : 'error', softFragments ? `No heading or id “${target.fragment}” here (other generators may build ids differently).` : `No heading or id “${target.fragment}” in this document.`, line, column, end); + return; + } + case 'local': { + const pending = this.stat(target.path); + if (!pending) { if (!this.input.fs) this.add('unverified-local', 'unverified', 'Not checked: this document has no folder yet. Save it to check local files.', line, column, end); return; } + const state = await pending; + if (this.aborted) return; + if (state === 'outside') this.add('outside-root', 'unverified', 'Points outside the document\'s folder or project; not checked.', line, column, end); + else if (state === 'unreadable') this.add('unreadable', 'warning', 'This path cannot be read or checked.', line, column, end); + else if (state === 'missing') this.add('missing-file', 'error', `No such file: ${target.path}`, line, column, end); + else if (state === 'file' && target.fragment && /\.(?:md|markdown)$/iu.test(target.path) && this.input.fs) { + const text = await this.input.fs.readText(target.path, MAX_TARGET_BYTES); + if (text !== undefined && !resolveFragment(headingsFromLines(text.split('\n')), target.fragment)) this.add('broken-fragment-in-file', 'warning', `No heading “${target.fragment}” in ${target.path}.`, line, column, end); + } + return; + } + } + } +} + +function codeLines(blocks: readonly Block[], into: Set): void { + for (const block of blocks) { + if (block.kind === 'code' && block.from !== undefined && block.to !== undefined) for (let line = block.from; line < block.to; line += 1) into.add(line); + else if (block.kind === 'quote') codeLines(block.blocks, into); + else if (block.kind === 'list') for (const item of block.items) codeLines(item.blocks, into); + } +} + +const normalize = (label: string) => label.trim().replace(/\s+/gu, ' ').toLowerCase(); +const maskCode = (text: string) => text.replace(/(`+)[\s\S]*?\1/gu, span => ' '.repeat(span.length)); + +async function markdown(run: Run): Promise { + const lines = run.input.lines; + const blocks = parseBlockLines(lines, {html: true}); + const skip = new Set(); + codeLines(blocks, skip); + if (lines[0]?.trim() === '---') for (let at = 1; at < Math.min(lines.length, 200); at += 1) if (/^(?:---|\.\.\.)\s*$/u.test(lines[at]!)) { for (let line = 0; line <= at; line += 1) skip.add(line); break; } + const headings = headingsFromLines(lines, {html: true}); + const seen = new Map(); + for (const heading of headings) { + const key = heading.slug; + if (key === undefined) continue; + const first = seen.get(heading.text.toLowerCase()); + if (first !== undefined) run.add('duplicate-heading', 'info', `Same heading text as line ${first + 1}; its anchor is #${key}.`, heading.line); + else seen.set(heading.text.toLowerCase(), heading.line); + } + const soft = lines.some(line => /\bid\s*=|\{#/u.test(line)); + const definitions = new Set(), footnoteDefs = new Map(); + const refs: Array<{label: string; line: number; start: number; end: number}> = [], notes: Array<{label: string; line: number; start: number; end: number}> = []; + const destinations: Array<{destination: string; line: number; column: number; image?: boolean; alt?: string}> = []; + lines.forEach((text, line) => { + if (skip.has(line) || run.aborted) return; + const masked = maskCode(text); + let from = 0; + const note = /^ {0,3}\[\^([^\]\s]+)\]:/u.exec(masked); + const definition = note ? undefined : /^ {0,3}\[([^\]]+)\]:[ \t]*(<[^>\n]*>|\S+)/du.exec(masked); + if (note) { footnoteDefs.set(note[1]!, line); from = note[0].length; } + if (definition) { + definitions.add(normalize(definition[1]!)); + const raw = definition[2]!; + const start = definition.indices![2]![0] + (raw.startsWith('<') ? 1 : 0); + destinations.push({destination: raw.startsWith('<') ? raw.slice(1, -1) : raw, line, column: start}); + from = definition[0].length; + } + const rest = masked.slice(from); + for (const match of rest.matchAll(/(!?)(?\n]*>|[^\s()]*(?:\([^\s()]*\)[^\s()]*)*)/gdu)) { + const raw = match[3]!; + const [startIndex] = match.indices![3]!; + const angle = raw.startsWith('<'); + destinations.push({destination: angle ? raw.slice(1, -1) : raw, line, column: from + startIndex + (angle ? 1 : 0), ...(match[1] ? {image: true, alt: match[2]!} : {})}); + } + for (const match of rest.matchAll(/(?\s]*)>/gu)].filter(autolink => classify(autolink[1]!).kind === 'external').length; + }); + for (const item of destinations) { + if (run.aborted) return; + if (item.image && !item.alt!.trim()) run.add('image-alt', 'warning', 'This image has no alternative text.', item.line, item.column - 1, item.column - 1 + item.destination.length + 1); + await run.reference(item.destination, item.line, item.column, headings, undefined, soft); + } + for (const ref of refs) if (!definitions.has(normalize(ref.label))) run.add('undefined-reference', 'error', `No link reference definition for “${ref.label}”.`, ref.line, ref.start, ref.end); + const used = new Set(); + for (const note of notes) { used.add(note.label); if (!footnoteDefs.has(note.label)) run.add('undefined-footnote', 'error', `No definition for footnote [^${note.label}].`, note.line, note.start, note.end); } + for (const [label, line] of footnoteDefs) if (!used.has(label)) run.add('unreferenced-footnote', 'warning', `Footnote [^${label}] is defined but never referenced.`, line); +} + +// ---- CSS and HTML ----------------------------------------------------------------------------------------------- + +async function cssUrls(run: Run, text: string, line: number, columnBase: number): Promise { + for (const match of text.matchAll(/url\(\s*(?:"([^"\n]*)"|'([^'\n]*)'|([^)\s'"]*))\s*\)/gdu)) { + const value = match[1] ?? match[2] ?? match[3] ?? ''; + const [start] = match.indices![match[1] !== undefined ? 1 : match[2] !== undefined ? 2 : 3]!; + if (TEMPLATED.test(value)) { run.add('unverified-dynamic', 'unverified', 'A templated address cannot be checked statically.', line, columnBase + start, columnBase + start + value.length); continue; } + await run.reference(value, line, columnBase + start, undefined, undefined, false); + } +} + +function cssSyntax(run: Run, lines: readonly string[]): void { + const stack: Array<{char: string; line: number; column: number}> = []; + let comment = false, quote = ''; + for (let line = 0; line < lines.length; line += 1) { + const text = lines[line]!; + for (let column = 0; column < text.length; column += 1) { + const char = text[column]!; + if (comment) { if (char === '*' && text[column + 1] === '/') { comment = false; column += 1; } continue; } + if (quote) { if (char === '\\') column += 1; else if (char === quote) quote = ''; continue; } + if (char === '/' && text[column + 1] === '*') { comment = true; column += 1; continue; } + if (char === '"' || char === '\'') { quote = char; continue; } + if (char === '{' || char === '(') stack.push({char, line, column}); + else if (char === '}' || char === ')') { + const open = stack.pop(); + if (!open || (open.char === '{') !== (char === '}')) { run.add('css-syntax', 'warning', `Unexpected “${char}”.`, line, column, column + 1); return; } + } + } + quote = ''; + } + const open = stack.at(-1); + if (open) run.add('css-syntax', 'warning', `“${open.char}” is never closed.`, open.line, open.column, open.column + 1); + else if (comment) run.add('css-syntax', 'warning', 'A comment is never closed.', lines.length - 1); +} + +const VOID = new Set(['area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta', 'param', 'source', 'track', 'wbr']); +const OPTIONAL_END = new Set(['p', 'li', 'dt', 'dd', 'tr', 'td', 'th', 'thead', 'tbody', 'tfoot', 'option', 'optgroup', 'colgroup', 'html', 'head', 'body']); +const RAW_TEXT = new Set(['script', 'style', 'textarea', 'title']); +const TAG = /<(\/?)([a-zA-Z][a-zA-Z0-9-]*)((?:\s+[^\s"'<>/=]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s"'<>`=]+))?)*)\s*(\/?)>/gdu; +const ATTRIBUTE = /([^\s"'<>/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'<>`=]+)))?/gdu; +const MAX_HTML_CHARS = 1_500_000; + +async function html(run: Run): Promise { + const lines = run.input.lines; + const starts: number[] = []; + let offset = 0; + for (const text of lines) { starts.push(offset); offset += text.length + 1; } + const source = lines.join('\n'); + if (source.length > MAX_HTML_CHARS) { run.truncated = true; return; } + const where = (position: number) => { let low = 0, high = starts.length - 1; while (low < high) { const mid = (low + high + 1) >> 1; if (starts[mid]! <= position) low = mid; else high = mid - 1; } return {line: low, column: position - starts[low]!}; }; + const text = source.replace(//gu, comment => comment.replace(/[^\n]/gu, ' ')); + interface Tag {name: string; attributes: Map; at: number} + const ids = new Map(), fragments: Array<{value: string; position: number; end: number}> = [], labelsFor = new Set(), controls: Array<{tag: Tag; labelled: boolean}> = []; + const stack: Array<{name: string; at: number}> = []; + let dynamic = 0, inLabel = 0; + const jobs: Array<() => Promise> = []; + let last = 0; + for (const match of text.matchAll(TAG)) { + if (run.aborted) return; + if (match.index! < last) continue; + const closing = Boolean(match[1]); + const name = match[2]!.toLowerCase(); + const attributes = new Map(); + const attributeText = match[3] ?? ''; + const base = match.indices![3]?.[0] ?? match.index!; + for (const attribute of attributeText.matchAll(ATTRIBUTE)) { + const group = attribute.indices![2] ?? attribute.indices![3] ?? attribute.indices![4]; + attributes.set(attribute[1]!.toLowerCase(), {value: attribute[2] ?? attribute[3] ?? attribute[4] ?? '', start: base + (group?.[0] ?? attribute.indices![1]![1]), end: base + (group?.[1] ?? attribute.indices![1]![1])}); + } + const tag: Tag = {name, attributes, at: match.index!}; + const position = where(match.index!); + if (closing) { + if (name === 'label' && inLabel > 0) inLabel -= 1; + const found = stack.map(entry => entry.name).lastIndexOf(name); + if (found < 0) { if (!VOID.has(name)) run.add('unexpected-closing-tag', 'warning', ` has no matching opening tag.`, position.line, position.column, position.column + match[0].length); } + else { for (let at = stack.length - 1; at > found; at -= 1) { const open = stack[at]!; if (!OPTIONAL_END.has(open.name)) { const place = where(open.at); run.add('unclosed-tag', 'warning', `<${open.name}> is never closed.`, place.line, place.column, place.column + open.name.length + 1); } } stack.length = found; } + } else { + if (!VOID.has(name) && !match[4]) { + if (OPTIONAL_END.has(name) && stack.at(-1)?.name === name) stack.pop(); + stack.push({name, at: match.index!}); + } + if (name === 'label') inLabel += 1; + const dynamicHere = [...attributes.values()].some(attribute => TEMPLATED.test(attribute.value)); + if (dynamicHere) { dynamic += 1; const first = [...attributes.values()].find(attribute => TEMPLATED.test(attribute.value))!; const place = where(first.start); run.add('unverified-dynamic', 'unverified', 'A templated attribute cannot be checked statically.', place.line, place.column, place.column + (first.end - first.start)); } + const id = attributes.get('id'); + if (id?.value) { if (ids.has(id.value)) { const place = where(id.start); run.add('duplicate-id', 'error', `The id “${id.value}” is used twice (first on line ${where(ids.get(id.value)!).line + 1}).`, place.line, place.column, place.column + id.value.length); } else ids.set(id.value, id.start); } + const named = attributes.get('name'); + if (name === 'a' && named?.value) ids.set(named.value, named.start); + const asset = (attribute: string) => { + const entry = attributes.get(attribute); + if (!entry || !entry.value || dynamicHere) return; + const place = where(entry.start); + jobs.push(() => run.reference(entry.value, place.line, place.column, undefined, undefined, false)); + }; + if (['img', 'script', 'source', 'video', 'audio', 'embed', 'track', 'iframe'].includes(name)) asset('src'); + if (name === 'link') asset('href'); + if (name === 'img' && !attributes.has('alt')) run.add('image-alt', 'warning', 'This image has no alt attribute (use alt="" for a decorative one).', position.line, position.column, position.column + match[0].length); + if (name === 'a') { const href = attributes.get('href'); if (href && !dynamicHere) { if (href.value.startsWith('#')) fragments.push({value: href.value.slice(1), position: href.start, end: href.end}); else asset('href'); } } + if (name === 'label' && attributes.get('for')) labelsFor.add(attributes.get('for')!.value); + if (['input', 'select', 'textarea'].includes(name) && !(name === 'input' && ['hidden', 'submit', 'button', 'image', 'reset'].includes((attributes.get('type')?.value ?? '').toLowerCase())) && !dynamicHere) { + controls.push({tag, labelled: inLabel > 0 || attributes.has('aria-label') || attributes.has('aria-labelledby') || attributes.has('title')}); + } + if (RAW_TEXT.has(name) && !match[4]) { + // The text of a script, style, textarea or title is not markup: skip it, but read url() addresses out of a style. + const contentStart = match.index! + match[0].length; + const rest = text.slice(contentStart); + const end = new RegExp(``, 'iu').exec(rest); + const content = end ? rest.slice(0, end.index) : rest; + if (name === 'style') { const origin = where(contentStart); content.split('\n').forEach((part, index) => jobs.push(() => cssUrls(run, part, origin.line + index, index === 0 ? origin.column : 0))); } + last = contentStart + content.length + (end ? end[0].length : 0); + if (end) stack.pop(); + } + } + } + for (const open of stack) if (!OPTIONAL_END.has(open.name)) { const place = where(open.at); run.add('unclosed-tag', 'warning', `<${open.name}> is never closed.`, place.line, place.column, place.column + open.name.length + 1); } + for (const fragment of fragments) { + let value = fragment.value; + try { value = decodeURIComponent(value); } catch { /* as written */ } + if (!value || value.toLowerCase() === 'top' || ids.has(value)) continue; + const place = where(fragment.position); + run.add('broken-fragment', 'error', `No element with the id “${value}” in this document.`, place.line, place.column, place.column + (fragment.end - fragment.position)); + } + for (const control of controls) { + const id = control.tag.attributes.get('id')?.value; + if (control.labelled || (id && labelsFor.has(id))) continue; + const place = where(control.tag.at); + run.add('form-label', 'warning', `This <${control.tag.name}> has no label (use