Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 5 additions & 6 deletions .github/actions/setup/action.yml
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
name: Setup & Install Dependencies
description: Sets Node.js up and installs project dependencies
description: Sets Bun up and installs project dependencies
runs:
using: composite
steps:
- name: Setup Node.js
uses: actions/setup-node@v7
- name: Setup Bun
uses: oven-sh/setup-bun@v2
with:
node-version-file: package.json
cache: npm
bun-version-file: package.json

- name: Install dependencies
shell: bash
run: npm ci
run: bun install --frozen-lockfile
2 changes: 1 addition & 1 deletion .github/workflows/continuous-link-linting.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,4 @@ jobs:

# we intentionally do not restore cache here
- name: Lint links
run: npm run lint:links
run: bun run lint:links
18 changes: 8 additions & 10 deletions .github/workflows/fix-and-lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -89,11 +89,11 @@ jobs:

- name: Format Code
shell: bash
run: npm run format
run: bun run format

- name: Generate UUIDs
shell: bash
run: npm run gen:uuids
run: bun run gen:uuids

- name: Commit Changes
id: commit-changes
Expand Down Expand Up @@ -137,18 +137,16 @@ jobs:

- parallel:
- name: Lint code
run: npm run lint:ultracite
run: bun run lint:ultracite

- name: Lint tasks
run: npm run lint:tasks
run: bun run lint:tasks

- name: Lint schema
run: npm run lint:schema
run: bun run lint:schema

- name: Lint UUIDs
run: npm run lint:uuids
- name: Lint node types
run: npm run lint:node-types
run: bun run lint:uuids

lint-links:
name: Lint Links
Expand Down Expand Up @@ -177,7 +175,7 @@ jobs:
cache-

- name: Lint links
run: npm run lint:links
run: bun run lint:links

check-types:
name: Check Typescript Types
Expand All @@ -198,7 +196,7 @@ jobs:
- uses: ./.github/actions/setup

- name: Generate types
run: npm run gen:types
run: bun run gen:types

- name: Ensure types are up to date
run: |
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/push-task-list-changes.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ jobs:
- uses: ./.github/actions/setup

- name: Generate list
run: npm run gen:list:official
run: bun run gen:list:official

- name: Checkout plugin repository
uses: actions/checkout@v7
Expand Down
4 changes: 2 additions & 2 deletions bin/frac.mts
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
* Passing `--lint` as the **first** argument will cause the program to act as a very simple
* linter, not modifying any file and exiting with an error code if any issues were found.
*
* Usage: `npx tsx frac.mts [--lint] <file-glob1> [<file-glob2> ...]`
* Example: `npx tsx frac.mts tasks/*.json`
* Usage: `bun frac.mts [--lint] <file-glob1> [<file-glob2> ...]`
* Example: `bun frac.mts tasks/*.json`
*/
import { readFile, writeFile } from 'node:fs/promises';
import { performance } from 'node:perf_hooks';
Expand Down
147 changes: 136 additions & 11 deletions bin/validate-links.mts
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,22 @@ import { closest } from 'fastest-levenshtein';
import { Glob } from 'glob';
import type { TaskTier } from '@/types.js';

const WIKI_ORIGIN = 'https://oldschool.runescape.wiki';
const WIKI_API_URL = `${WIKI_ORIGIN}/api.php`;
const IMAGE_SEARCH_BASE_URL =
'https://oldschool.runescape.wiki/?title=Special%3ASearch&profile=advanced&fulltext=1&ns6=1&search=';
const CACHE_DIR = './.cache/links';
const IMAGE_CONTENT_TYPE_REGEX = /^image\/(png|gif)/;
const HTML_CONTENT_TYPE_REGEX = /^text\/html/;
const HTTP_STATUS_OK = 200;
const WIKI_API_BATCH_SIZE = 50;
const WIKI_TITLE_ALIAS_MAX_DEPTH = 5;

type WikiPage = { missing?: string; invalid?: string; title?: string };
type WikiQuery = {
pages?: Record<string, WikiPage>;
normalized?: { from: string; to: string }[];
redirects?: { from: string; to: string }[];
};

const shouldFix = process.argv.slice(2).includes('--fix');

Expand Down Expand Up @@ -44,16 +54,14 @@ for await (const tierFile of tierWalker) {
let hasErrors = false;

console.log(`Checking ${wikiLinks.size} wiki links...`);
for (const link of wikiLinks) {
if (!(await validateLink(link, HTML_CONTENT_TYPE_REGEX))) {
hasErrors = true;
console.error(`- Invalid link: ${link}`);
}
for (const link of await validateWikiLinks([...wikiLinks])) {
hasErrors = true;
console.error(`- Invalid link: ${link}`);
}

console.log(`Checking ${imageLinks.size} image links...`);
for (const link of imageLinks) {
if (!(await validateLink(link, IMAGE_CONTENT_TYPE_REGEX))) {
if (!(await validateImageLink(link))) {
console.error(`- Invalid link: ${link}`);

if (shouldFix) {
Expand All @@ -78,19 +86,133 @@ function cacheKey(link: string): string {
return hash('sha1', link);
}

async function validateLink(link: string, contentTypeRegex: RegExp): Promise<boolean> {
async function markCached(link: string): Promise<void> {
await writeFile(`${CACHE_DIR}/${cacheKey(link)}`, '');
}

function wikiTitleFromLink(link: string): string | null {
let url: URL;
try {
url = new URL(link);
} catch {
return null;
}

if (url.origin !== WIKI_ORIGIN) {
return null;
}

const match = url.pathname.match(/^\/w\/(.+)$/);
if (!match?.[1]) {
return null;
}

return decodeURIComponent(match[1].replaceAll('_', ' '));
}

function existingWikiTitles(query: WikiQuery): Set<string> {
const existingTitles = new Set<string>();
for (const page of Object.values(query.pages ?? {})) {
if (page.missing != null || page.invalid != null || !page.title) {
continue;
}
existingTitles.add(page.title);
}
return existingTitles;
}

function wikiTitleAliases(query: WikiQuery): Map<string, string> {
const titleAliases = new Map<string, string>();
for (const { from, to } of query.normalized ?? []) {
titleAliases.set(from, to);
}
for (const { from, to } of query.redirects ?? []) {
titleAliases.set(from, to);
}
return titleAliases;
}

function resolveWikiTitle(title: string, titleAliases: Map<string, string>): string {
let resolved = title;
for (let depth = 0; depth < WIKI_TITLE_ALIAS_MAX_DEPTH; depth++) {
const next = titleAliases.get(resolved);
if (!next) {
return resolved;
}
resolved = next;
}
return resolved;
}

async function validateWikiBatch(batch: { link: string; title: string }[]): Promise<string[]> {
const titles = batch.map(({ title }) => title);
const url = new URL(WIKI_API_URL);
url.searchParams.set('action', 'query');
url.searchParams.set('format', 'json');
url.searchParams.set('redirects', '1');
url.searchParams.set('titles', titles.join('|'));

const res = await fetch(url);
if (res.status !== HTTP_STATUS_OK) {
return batch.map(({ link }) => link);
}

const data = (await res.json()) as { query?: WikiQuery };
const query = data.query ?? {};
const existingTitles = existingWikiTitles(query);
const titleAliases = wikiTitleAliases(query);
const invalid: string[] = [];

for (const { link, title } of batch) {
const resolved = resolveWikiTitle(title, titleAliases);
if (existingTitles.has(resolved)) {
await markCached(link);
continue;
}
invalid.push(link);
}

return invalid;
}

async function validateWikiLinks(links: string[]): Promise<string[]> {
const invalid: string[] = [];
const pending: { link: string; title: string }[] = [];

for (const link of links) {
if (cachedKeys.has(cacheKey(link))) {
continue;
}

const title = wikiTitleFromLink(link);
if (!title) {
invalid.push(link);
continue;
}

pending.push({ link, title });
}

for (let i = 0; i < pending.length; i += WIKI_API_BATCH_SIZE) {
invalid.push(...(await validateWikiBatch(pending.slice(i, i + WIKI_API_BATCH_SIZE))));
}

return invalid;
}

async function validateImageLink(link: string): Promise<boolean> {
if (cachedKeys.has(cacheKey(link))) {
return true;
}

const res = await fetch(link, { method: 'HEAD' });
const contentType = res.headers.get('Content-Type');

if (res.status !== HTTP_STATUS_OK || !contentType?.match(contentTypeRegex)) {
if (res.status !== HTTP_STATUS_OK || !contentType?.match(IMAGE_CONTENT_TYPE_REGEX)) {
return false;
}

await writeFile(`${CACHE_DIR}/${cacheKey(link)}`, '');
await markCached(link);
return true;
}

Expand All @@ -102,7 +224,10 @@ async function proposeImageLinkFix(link: string): Promise<string | null> {
const body = await res.text();

const matchedLinks = [...body.matchAll(/<a href="\/w\/File:(.{5,50})\.(?:png|gif)" title=/g)];
const suggestedLinks = matchedLinks.map((match) => link.replaceAll(baseName, match[1]));
const suggestedLinks = matchedLinks.flatMap((match) => {
const fileName = match[1];
return fileName ? [link.replaceAll(baseName, fileName)] : [];
});
if (suggestedLinks.length === 0) {
return null;
}
Expand Down
Loading
Loading