adding SSRF protection

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Change Log
 
+## 6.13.0
+
+- Added URL and IP validation to block non-HTTP(S) URLs and private, loopback, link-local, reserved, and mapped IP addresses. See `lib/isPublicAddress.ts`.
+- Updated `lib/request.ts` to check DNS results at connection time and validate each redirect destination. Caller-provided fetch options can’t restore automatic redirect handling or replace the guarded dispatcher.
+- `lib/request.ts`: Always uses OGS’s DNS-guarded Undici Agent, rather than switching to a global dispatcher that may have changed after module initialization.
+- Blacklist now applies to redirect destinations.
+- Updating dependencies to fix npm vulnerabilities
+
 ## 6.12.0
 
 - CI/CD now supports node26!

README.md

@@ -60,14 +60,16 @@ Check the return for a ```success``` flag. If success is set to true, then the u
 | html                 | You can pass in an HTML string to run ogs on it. (use without options.url)                                                                |                                                                                     |          |
 | fetchOptions         | Options that are used by the Fetch API                                                                                                    | {}                                                                                  |          |
 | timeout              | Request timeout for Fetch (Default is 10 seconds)                                                                                         | 10                                                                                  |          |
-| blacklist            | Pass in an array of sites you don't want ogs to run on.                                                                                   | []                                                                                  |          |
+| blacklist            | Pass in site values to block, including redirect destinations.                                                                            | []                                                                                  |          |
 | onlyGetOpenGraphInfo | Only fetch open graph info and don't fall back on anything else. Also accepts an array of properties for which no fallback should be used | false                                                                               |          |
 | customMetaTags       | Here you can define custom meta tags you want to scrape.                                                                                  | []                                                                                  |          |
 | urlValidatorSettings | Sets the options used by validator.js for testing the URL                                                                                 | [Here](https://github.com/jshemas/openGraphScraper/blob/master/lib/utils.ts#L4-L17) |          |
 | jsonLDOptions        | Sets the options used when parsing JSON-LD data                                                                                           |                                                                            |          |
 
 Note: `open-graph-scraper` uses the [Fetch API](https://nodejs.org/dist/latest-v18.x/docs/api/globals.html#fetch) for requests and most of [Fetch's options](https://developer.mozilla.org/en-US/docs/Web/API/fetch#options) should work as `open-graph-scraper`'s `fetchOptions` options.
 
+For SSRF protection, requests are limited to HTTP(S) destinations whose resolved IP addresses are public. Redirect destinations are checked before they are requested. Redirect handling cannot be overridden, and custom Undici dispatchers supplied in `fetchOptions` or configured globally are ignored because they could bypass these protections.
+
 ## Types And Import Example
 
 ```javascript

SECURITY.md

@@ -0,0 +1,26 @@
+# Security Policy
+
+## Supported Versions
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 6.x.x   | :white_check_mark: |
+| < 5.0   | :x:                |
+
+## Reporting a Vulnerability
+
+We value the insights of the security community and encourage responsible disclosure. If you discover a vulnerability, please do not use public GitHub issues, discussions, or pull requests to report it. 
+
+### How to Disclose
+Instead of a public post, please report vulnerabilities through the following channels:
+
+* **GitHub Private Vulnerability Reporting:** Navigate to the "Security" tab of this repository, select "Vulnerability reporting" from the sidebar, and click "Report a vulnerability".
+
+### What to Include
+To help us understand and fix the issue quickly, please include:
+1. **Description:** The type of issue (e.g., SQL injection, Remote Code Execution).
+2. **Location:** The affected file paths, branches, or direct URLs.
+3. **Steps to Reproduce:** A clear description or proof-of-concept (PoC) script.
+4. **Impact:** How an attacker could exploit this vulnerability.
+
+Thank you for helping keep our software secure!

index.ts

@@ -16,7 +16,7 @@ import type {
  * @param {object} [options.customMetaTags] - Here you can define custom meta tags you want to scrape.
  * @param {object} [options.fetchOptions] - Sets the options used by fetch for the http requests
  * @param {object} [options.urlValidatorSettings] - Sets the options used by validator.js for testing the URL
- * @param {string[]} [options.blacklist] - Pass in an array of sites you don't want ogs to run on.
+ * @param {string[]} [options.blacklist] - Pass in site values to block, including redirect destinations.
  * @param {string} [options.html] - You can pass in an HTML string to run ogs on it. (use without options.url)
  * @param {number} [options.timeout] - Number of seconds before the fetch request ends. (default is 10 seconds)
  * @param {string} options.url - URL of the site. (Required)

lib/isPublicAddress.ts

@@ -0,0 +1,95 @@
+import { isIP } from 'node:net';
+import { lookup } from 'node:dns/promises';
+
+function isBlockedIPv4(address: string) {
+  const octets = address.split('.').map(Number);
+  const [first, second, third] = octets;
+
+  return first === 0
+    || first === 10
+    || (first === 100 && second >= 64 && second <= 127)
+    || first === 127
+    || (first === 169 && second === 254)
+    || (first === 172 && second >= 16 && second <= 31)
+    || (first === 192 && second === 0 && third === 0)
+    || (first === 192 && second === 0 && third === 2)
+    || (first === 192 && second === 88 && third === 99)
+    || (first === 192 && second === 168)
+    || (first === 198 && (second === 18 || second === 19))
+    || (first === 198 && second === 51 && third === 100)
+    || (first === 203 && second === 0 && third === 113)
+    || first >= 224;
+}
+
+function parseIPv6(address: string) {
+  let normalized = address.toLowerCase();
+  const ipv4Suffix = normalized.match(/(\d{1,3}(?:\.\d{1,3}){3})$/)?.[1];
+
+  if (ipv4Suffix) {
+    const octets = ipv4Suffix.split('.').map(Number);
+    if (octets.some((octet) => octet > 255)) return undefined;
+    const high = ((octets[0] ?? 0) * 256) + (octets[1] ?? 0);
+    const low = ((octets[2] ?? 0) * 256) + (octets[3] ?? 0);
+    normalized = normalized.replace(ipv4Suffix, `${high.toString(16)}:${low.toString(16)}`);
+  }
+
+  const halves = normalized.split('::');
+  if (halves.length > 2) return undefined;
+  const left = halves[0] ? halves[0].split(':') : [];
+  const right = halves[1] ? halves[1].split(':') : [];
+  const zeroCount = 8 - left.length - right.length;
+  if ((halves.length === 1 && zeroCount !== 0) || (halves.length === 2 && zeroCount < 1)) return undefined;
+
+  const groups = [
+    ...left,
+    ...Array.from({ length: zeroCount }, () => '0'),
+    ...right,
+  ].map((group) => Number.parseInt(group, 16));
+  if (groups.length !== 8 || groups.some((group) => !Number.isInteger(group) || group < 0 || group > 0xffff)) {
+    return undefined;
+  }
+
+  return groups;
+}
+
+function isBlockedIPv6(address: string) {
+  const groups = parseIPv6(address);
+  if (!groups) return true;
+
+  const first = groups[0] ?? 0;
+  const second = groups[1] ?? 0;
+
+  return Math.floor(first / 0x2000) !== 1
+    || (first === 0x2001 && second <= 0x01ff)
+    || (first === 0x2001 && second === 0x0db8)
+    || first === 0x2002
+    || (first === 0x3fff && second <= 0x0fff);
+}
+
+export function isPublicAddress(address: string) {
+  const version = isIP(address);
+  if (version === 4) return !isBlockedIPv4(address);
+  if (version === 6) return !isBlockedIPv6(address);
+  return false;
+}
+
+export async function assertPublicUrl(input: string) {
+  const url = new URL(input);
+  if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+    throw new Error('Only HTTP and HTTPS URLs are allowed');
+  }
+
+  const hostname = url.hostname.replace(/^\[|\]$/g, '');
+  if (hostname.includes('%')) throw new Error('URL must not contain a scoped IP address');
+
+  const version = isIP(hostname);
+  if (version) {
+    if (!isPublicAddress(hostname)) throw new Error('Requests to non-public IP addresses are not allowed');
+    return;
+  }
+
+  const addresses = await lookup(hostname, { all: true, verbatim: true });
+  if (addresses.length === 0 || addresses.some(({ address }) => !isPublicAddress(address))) {
+    throw new Error('Requests to hosts resolving to non-public IP addresses are not allowed');
+  }
+}

lib/openGraphScraper.ts

@@ -2,6 +2,7 @@ import extractMetaTags from './extract';
 import requestAndResultsFormatter from './request';
 import {
   defaultUrlValidatorSettings,
+  isBlacklistedUrl,
   isCustomMetaTagsValid,
   isThisANonHTMLUrl,
   optionSetup,
@@ -38,8 +39,7 @@ export default async function setOptionsAndReturnOpenGraphResults(ogsOptions: Op
   // trying to limit non html pages
   if (isThisANonHTMLUrl(options.url)) throw new Error('Must scrape an HTML page');
 
-  // eslint-disable-next-line max-len
-  if (options?.blacklist?.some((blacklistedHostname) => options.url?.includes(blacklistedHostname))) {
+  if (isBlacklistedUrl(options.url, options.blacklist)) {
     throw new Error('Host name has been black listed');
   }
 

lib/request.ts

@@ -1,8 +1,123 @@
-import { fetch } from 'undici';
+import { lookup as dnsLookup } from 'node:dns';
+import {
+  Agent, fetch, Headers, type RequestInit,
+} from 'undici';
 import { decode } from 'iconv-lite';
 import { CheerioAPI, load } from 'cheerio';
 import chardet from 'chardet';
 import type { OpenGraphScraperOptions } from './types';
+import { assertPublicUrl, isPublicAddress } from './isPublicAddress';
+import { isBlacklistedUrl } from './utils';
+
+const redirectStatuses = new Set([301, 302, 303, 307, 308]);
+
+function createRequestDispatcher() {
+  const dispatcher = new Agent({
+    connect: {
+      lookup(hostname, options, callback) {
+        dnsLookup(hostname, { all: true, verbatim: true }, (error, addresses) => {
+          if (error) {
+            callback(error, '', 0);
+            return;
+          }
+          if (addresses.some(({ address }) => !isPublicAddress(address))) {
+            callback(Object.assign(new Error('Requests to hosts resolving to non-public IP addresses are not allowed'), {
+              code: 'EACCES',
+            }), '', 0);
+            return;
+          }
+
+          let family: number | undefined;
+          switch (options.family) {
+            case 'IPv4':
+              family = 4;
+              break;
+            case 'IPv6':
+              family = 6;
+              break;
+            case undefined:
+              family = undefined;
+              break;
+            default:
+              family = options.family;
+          }
+          const matchingAddresses = addresses.filter(({ family: addressFamily }) => (
+            family === undefined || family === 0 || addressFamily === family
+          ));
+          if (matchingAddresses.length === 0) {
+            callback(Object.assign(new Error('No public IP addresses found for host'), { code: 'ENOTFOUND' }), '', 0);
+            return;
+          }
+          if (options.all) {
+            callback(null, matchingAddresses);
+            return;
+          }
+
+          const address = matchingAddresses[0];
+          if (!address) {
+            callback(Object.assign(new Error('No public IP addresses found for host'), { code: 'ENOTFOUND' }), '', 0);
+            return;
+          }
+          callback(null, address.address, address.family);
+        });
+      },
+    },
+  });
+  return { dispatcher, close: () => dispatcher.close() };
+}
+
+async function fetchWithPublicDestinationGuard(url: string, options: RequestInit, blacklist: string[] = []) {
+  const redirectMode = options.redirect ?? 'follow';
+  async function fetchRedirect(currentUrl: string, redirects: number, requestOptions: RequestInit): Promise<Response> {
+    if (isBlacklistedUrl(currentUrl, blacklist)) {
+      throw new Error('Host name has been black listed');
+    }
+    await assertPublicUrl(currentUrl);
+    const response = await fetch(currentUrl, { ...requestOptions, redirect: 'manual' });
+    const location = response.headers.get('location');
+    if (!location || !redirectStatuses.has(response.status)) return response;
+    if (redirectMode === 'manual') return response;
+    if (redirectMode === 'error') {
+      await response.body?.cancel();
+      throw new Error('Request was redirected');
+    }
+    if (redirects >= 20) {
+      await response.body?.cancel();
+      throw new Error('Maximum redirect count exceeded');
+    }
+
+    const nextUrl = new URL(location, currentUrl).toString();
+    const nextOptions: RequestInit = { ...requestOptions };
+    const headers = new Headers(requestOptions.headers);
+    const currentOrigin = new URL(currentUrl).origin;
+    const nextOrigin = new URL(nextUrl).origin;
+
+    if (currentOrigin !== nextOrigin) {
+      headers.delete('authorization');
+      headers.delete('cookie');
+      headers.delete('cookie2');
+      headers.delete('host');
+      headers.delete('proxy-authorization');
+    }
+
+    const method = (requestOptions.method ?? 'GET').toUpperCase();
+    if (((response.status === 301 || response.status === 302) && method === 'POST')
+      || (response.status === 303 && method !== 'GET' && method !== 'HEAD')) {
+      nextOptions.method = 'GET';
+      delete nextOptions.body;
+      headers.delete('content-encoding');
+      headers.delete('content-language');
+      headers.delete('content-location');
+      headers.delete('content-type');
+    }
+
+    nextOptions.headers = headers;
+    await response.body?.cancel();
+    return fetchRedirect(nextUrl, redirects + 1, nextOptions);
+  }
+
+  return fetchRedirect(url, 0, options);
+}
 
 /**
  * checks if an element exists
@@ -47,21 +162,20 @@ function getCharset(body: string, buffer: ArrayBuffer, $: CheerioAPI) {
 export default async function requestAndResultsFormatter(options: OpenGraphScraperOptions) {
   let body;
   let response;
+  const requestDispatcher = createRequestDispatcher();
   try {
     // eslint-disable-next-line no-control-regex
     const isLatin1 = /^[\u0000-\u00ff]{0,}$/;
 
     let url = options.url ?? '';
     if (!isLatin1.test(url)) url = encodeURI(url);
 
-    response = await fetch(
-      url ?? '',
-      {
-        signal: AbortSignal.timeout((options.timeout ?? 10) * 1000),
-        ...options.fetchOptions,
-        headers: { Origin: url ?? '', Accept: 'text/html', ...options.fetchOptions?.headers },
-      },
-    );
+    response = await fetchWithPublicDestinationGuard(url, {
+      signal: AbortSignal.timeout((options.timeout ?? 10) * 1000),
+      ...options.fetchOptions,
+      dispatcher: requestDispatcher.dispatcher,
+      headers: { Origin: url, Accept: 'text/html', ...options.fetchOptions?.headers },
+    }, options.blacklist);
 
     const bodyArrayBuffer = await response.arrayBuffer();
     const bodyText = Buffer.from(bodyArrayBuffer).toString('utf-8');
@@ -111,6 +225,8 @@ export default async function requestAndResultsFormatter(options: OpenGraphScrap
   } catch (error) {
     if (error instanceof Error && error.message === 'fetch failed') throw error.cause;
     throw error;
+  } finally {
+    await requestDispatcher.close();
   }
 
   return { body, response };

lib/types.ts

@@ -22,7 +22,7 @@ export type OnlyGetOpenGraphInfoItem = 'image' | 'title' | 'description' | 'loca
  *
  * @typeParam {string} url - URL of the site. (Required)
  * @typeParam {string} [html] - You can pass in an HTML string to run ogs on it. (use without options.url)
- * @typeParam {string[]} [blacklist] - Pass in an array of sites you don't want ogs to run on.
+ * @typeParam {string[]} [blacklist] - Pass in site values to block, including redirect destinations.
  * @typeParam {boolean | OnlyGetOpenGraphInfoItem[]} [onlyGetOpenGraphInfo] - Only fetch open graph info and don't fall back on anything else.
  * @typeParam {CustomMetaTags} [customMetaTags] - Here you can define custom meta tags you want to scrape.
  * @typeParam {Request} [fetchOptions] - The options passed into fetch.

lib/utils.ts

@@ -54,6 +54,17 @@ export function validateAndFormatURL(url: string, urlValidatorSettings: Validato
   return { url: isUrlValid(url, urlValidatorSettings) ? coerceUrl(url) : null };
 }
 
+/**
+ * Checks whether a URL contains a blacklisted value.
+ *
+ * @param {string} url - url to be checked
+ * @param {string[]} blacklist - values that should be blocked
+ * @return {boolean} whether the URL is blacklisted
+ */
+export function isBlacklistedUrl(url: string, blacklist: string[] = []): boolean {
+  return blacklist.some((blacklistedHostname) => url.includes(blacklistedHostname));
+}
+
 /**
  * Finds the image type from a given url
  *