diff --git a/scrapegraphai/docloaders/chromium.py b/scrapegraphai/docloaders/chromium.py index a452af1ea..e703a889a 100644 --- a/scrapegraphai/docloaders/chromium.py +++ b/scrapegraphai/docloaders/chromium.py @@ -6,6 +6,7 @@ from langchain_core.documents import Document from ..utils import Proxy, dynamic_import, get_logger, parse_or_search_proxy +from ..utils.url_safety import validate_url logger = get_logger("web-loader") @@ -36,6 +37,16 @@ def _warn_on_error_status(response: Any, url: str) -> None: ) +async def _block_unsafe_request(route: Any) -> None: + """Abort browser requests whose host resolves to a metadata address.""" + try: + await asyncio.to_thread(validate_url, route.request.url) + except ValueError: + await route.abort() + else: + await route.continue_() + + class ChromiumLoader: """Scrapes HTML pages from URLs using a (headless) instance of the Chromium web driver with proxy protection. @@ -276,8 +287,10 @@ async def ascrape_playwright_scroll( raise ValueError(f"Invalid browser name: {browser_name}") context = await browser.new_context() await Malenia.apply_stealth(context) + await context.route("**/*", _block_unsafe_request) page = await context.new_page() response = await page.goto(url, wait_until="domcontentloaded") + await asyncio.to_thread(validate_url, page.url) _warn_on_error_status(response, url) await page.wait_for_load_state(self.load_state) @@ -332,6 +345,7 @@ async def ascrape_playwright_scroll( break results = await page.content() + await asyncio.to_thread(validate_url, page.url) break except (aiohttp.ClientError, asyncio.TimeoutError, Exception) as e: @@ -390,11 +404,14 @@ async def ascrape_playwright(self, url: str, browser_name: str = "chromium") -> ignore_https_errors=True, ) await Malenia.apply_stealth(context) + await context.route("**/*", _block_unsafe_request) page = await context.new_page() response = await page.goto(url, wait_until="domcontentloaded") + await asyncio.to_thread(validate_url, page.url) _warn_on_error_status(response, url) await page.wait_for_load_state(self.load_state) results = await page.content() + await asyncio.to_thread(validate_url, page.url) logger.info("Content scraped") await browser.close() return results @@ -448,10 +465,13 @@ async def ascrape_with_js_support( context = await browser.new_context( storage_state=self.storage_state ) + await context.route("**/*", _block_unsafe_request) page = await context.new_page() response = await page.goto(url, wait_until="networkidle") + await asyncio.to_thread(validate_url, page.url) _warn_on_error_status(response, url) results = await page.content() + await asyncio.to_thread(validate_url, page.url) logger.info("Content scraped after JavaScript rendering") return results except (aiohttp.ClientError, asyncio.TimeoutError, Exception) as e: diff --git a/scrapegraphai/nodes/fetch_node.py b/scrapegraphai/nodes/fetch_node.py index c55b96f64..2eb6b26e7 100644 --- a/scrapegraphai/nodes/fetch_node.py +++ b/scrapegraphai/nodes/fetch_node.py @@ -13,6 +13,7 @@ from ..docloaders import ChromiumLoader from ..utils.cleanup_html import cleanup_html from ..utils.convert_to_md import convert_to_md +from ..utils.url_safety import validate_redirect, validate_url from .base_node import BaseNode @@ -276,17 +277,25 @@ def handle_web_source(self, state, source): dict: The updated state with the processed content. Raises: - ValueError: If the fetched HTML content is empty or contains only whitespace. + ValueError: If the fetched HTML content is empty or contains only whitespace, + or if the URL points to a link-local or cloud metadata address. """ + if not (self.browser_base or self.scrape_do): + validate_url(source) + self.logger.info(f"--- (Fetching HTML from: {source}) ---") if self.use_soup: # Apply configured timeout to blocking HTTP requests. If timeout is None, # don't pass the timeout argument (requests will block until completion). if self.timeout is None: - response = requests.get(source) + response = requests.get(source, hooks={"response": validate_redirect}) else: - response = requests.get(source, timeout=self.timeout) + response = requests.get( + source, + timeout=self.timeout, + hooks={"response": validate_redirect}, + ) if response.status_code == 200: if not response.text.strip(): raise ValueError("No HTML body content found in the response.") diff --git a/scrapegraphai/utils/url_safety.py b/scrapegraphai/utils/url_safety.py new file mode 100644 index 000000000..3ea0a6519 --- /dev/null +++ b/scrapegraphai/utils/url_safety.py @@ -0,0 +1,61 @@ +""" +Module for refusing scrape targets that point at cloud metadata endpoints +""" + +import ipaddress +import socket +from urllib.parse import urljoin, urlparse + +# Metadata services that sit outside the link-local ranges +_METADATA_ADDRESSES = { + ipaddress.ip_address("fd00:ec2::254"), + ipaddress.ip_address("100.100.100.200"), +} + + +def validate_url(url: str) -> None: + """ + Raises ValueError if the host of the url resolves to a link-local address + (169.254.0.0/16, fe80::/10) or to a known cloud metadata address. + Hosts that do not resolve are left to fail when they are fetched. + + Args: + url (str): The url about to be fetched. + + Raises: + ValueError: If any address the host resolves to is not allowed. + """ + hostname = urlparse(url).hostname + if not hostname: + return + + try: + # urllib3 drops an IPv6 zone id (fe80::1%25eth0) before connecting + infos = socket.getaddrinfo(hostname.split("%")[0], None) + except socket.gaierror: + return + + for info in infos: + address = ipaddress.ip_address(info[4][0].split("%")[0]) + if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped: + address = address.ipv4_mapped + if address.is_link_local or address in _METADATA_ADDRESSES: + raise ValueError( + "The url points to a link-local or cloud metadata address " + "and will not be fetched." + ) + + +def validate_redirect(response, **kwargs) -> None: + """ + Response hook for requests that checks the target of a redirect before + requests follows it. + + Args: + response: The response that was just received. + + Raises: + ValueError: If the redirect target is not allowed. + """ + if response.is_redirect: + validate_url(urljoin(response.url, response.headers["location"])) diff --git a/tests/test_chromium.py b/tests/test_chromium.py index f1b343977..04b637ef3 100644 --- a/tests/test_chromium.py +++ b/tests/test_chromium.py @@ -24,10 +24,12 @@ def __init__(self): class MockContext: def __init__(self): self.new_page = AsyncMock() + self.route = AsyncMock() class MockPage: def __init__(self): + self.url = "http://example.com/" self.goto = AsyncMock() self.wait_for_load_state = AsyncMock() self.content = AsyncMock() @@ -245,6 +247,8 @@ async def test_ascrape_with_js_support_retry_failure(monkeypatch): # Create dummy classes to simulate failure in page.goto class DummyPage: + url = "http://example.com/" + async def goto(self, url, wait_until): raise asyncio.TimeoutError("Forced timeout") @@ -255,6 +259,9 @@ async def content(self): return "Dummy" class DummyContext: + async def route(self, pattern, handler): + return + async def new_page(self): return DummyPage() @@ -370,6 +377,8 @@ async def test_ascrape_playwright_retry_failure(monkeypatch): # Dummy classes to simulate persistent failure in page.goto for ascrape_playwright class DummyPage: + url = "http://example.com/" + async def goto(self, url, wait_until): raise asyncio.TimeoutError("Forced timeout in goto") @@ -380,6 +389,9 @@ async def content(self): return "This should not be returned" class DummyContext: + async def route(self, pattern, handler): + return + async def new_page(self): return DummyPage() @@ -578,6 +590,8 @@ async def test_ascrape_with_js_support_retry_success(monkeypatch): attempt_count = {"count": 0} class DummyPage: + url = "http://example.com/" + async def goto(self, url, wait_until): if attempt_count["count"] < 1: attempt_count["count"] += 1 @@ -591,6 +605,9 @@ async def content(self): return "Success on retry" class DummyContext: + async def route(self, pattern, handler): + return + async def new_page(self): return DummyPage() @@ -727,6 +744,8 @@ async def test_ascrape_with_js_support_storage_state(monkeypatch): """Test that ascrape_with_js_support passes the storage_state to the new_context call.""" class DummyPage: + url = "http://example.com/" + async def goto(self, url, wait_until): return @@ -737,6 +756,9 @@ async def content(self): return "Storage State Tested" class DummyContext: + async def route(self, pattern, handler): + return + async def new_page(self): return DummyPage() @@ -806,6 +828,8 @@ async def test_ascrape_playwright_browser_config(monkeypatch): captured_kwargs = {} class DummyPage: + url = "http://example.com/" + async def goto(self, url, wait_until): return @@ -816,6 +840,9 @@ async def content(self): return "Config Tested" class DummyContext: + async def route(self, pattern, handler): + return + async def new_page(self): return DummyPage() @@ -884,6 +911,8 @@ async def test_ascrape_playwright_scroll_retry_failure(monkeypatch): # Dummy page that always raises Timeout on goto class DummyPage: + url = "http://example.com/" + async def goto(self, url, wait_until): raise asyncio.TimeoutError("Simulated timeout in goto") @@ -900,6 +929,9 @@ async def content(self): mouse = AsyncMock() class DummyContext: + async def route(self, pattern, handler): + return + async def new_page(self): return DummyPage() @@ -971,6 +1003,8 @@ async def test_ascrape_with_js_support_calls_close(monkeypatch): close_called_flag = {"called": False} class DummyPage: + url = "http://example.com/" + async def goto(self, url, wait_until): return @@ -981,6 +1015,9 @@ async def content(self): return "Dummy Content" class DummyContext: + async def route(self, pattern, handler): + return + async def new_page(self): return DummyPage() @@ -1148,6 +1185,8 @@ async def test_ascrape_with_js_support_exception_cleanup(monkeypatch): close_called_flag = {"called": False} class DummyPage: + url = "http://example.com/" + async def goto(self, url, wait_until): raise asyncio.TimeoutError("Forced timeout") @@ -1158,6 +1197,9 @@ async def content(self): return "No Content" class DummyContext: + async def route(self, pattern, handler): + return + async def new_page(self): return DummyPage() @@ -1266,6 +1308,8 @@ async def test_ascrape_playwright_scroll_exception_cleanup(monkeypatch): close_called = {"called": False} class DummyPage: + url = "http://example.com/" + async def goto(self, url, wait_until): raise asyncio.TimeoutError("Simulated timeout in goto") @@ -1282,6 +1326,9 @@ async def evaluate(self, script): mouse.wheel = AsyncMock() class DummyContext: + async def route(self, pattern, handler): + return + async def new_page(self): return DummyPage() @@ -1332,6 +1379,8 @@ async def test_ascrape_with_js_support_non_timeout_retry(monkeypatch): attempt = {"count": 0} class DummyPage: + url = "http://example.com/" + async def goto(self, url, wait_until): if attempt["count"] < 1: attempt["count"] += 1 @@ -1344,6 +1393,9 @@ async def content(self): return "Success after non-timeout retry" class DummyContext: + async def route(self, pattern, handler): + return + async def new_page(self): return DummyPage() @@ -1414,6 +1466,8 @@ async def dummy_apply_stealth(context): ) class DummyPage: + url = "http://example.com/" + async def goto(self, url, wait_until): return @@ -1424,6 +1478,9 @@ async def content(self): return "Stealth Applied Content" class DummyContext: + async def route(self, pattern, handler): + return + async def new_page(self): return DummyPage() @@ -1531,6 +1588,8 @@ async def test_ascrape_with_js_support_browser_error_cleanup(monkeypatch): close_called = {"called": False} class DummyPage: + url = "http://example.com/" + async def goto(self, url, wait_until): raise aiohttp.ClientError("Navigation error") @@ -1541,6 +1600,9 @@ async def content(self): return "Error Content" class DummyContext: + async def route(self, pattern, handler): + return + async def new_page(self): return DummyPage() @@ -1605,6 +1667,8 @@ async def test_ascrape_playwright_stealth_exception_cleanup(monkeypatch): fail_flag = {"closed": False} class DummyPage: + url = "http://example.com/" + async def goto(self, url, wait_until): return @@ -1615,6 +1679,9 @@ async def content(self): return "Content" class DummyContext: + async def route(self, pattern, handler): + return + async def new_page(self): return DummyPage() @@ -1665,6 +1732,8 @@ async def test_ascrape_with_js_support_value_error_success(monkeypatch): attempt_count = {"count": 0} class DummyPage: + url = "http://example.com/" + async def goto(self, url, wait_until): if attempt_count["count"] < 1: attempt_count["count"] += 1 @@ -1678,6 +1747,9 @@ async def content(self): return "Success after ValueError" class DummyContext: + async def route(self, pattern, handler): + return + async def new_page(self): return DummyPage() @@ -1722,6 +1794,8 @@ async def test_ascrape_with_js_support_value_error_failure(monkeypatch): """Test that ascrape_with_js_support raises RuntimeError after exhausting retries on persistent ValueError.""" class DummyPage: + url = "http://example.com/" + async def goto(self, url, wait_until): raise ValueError("Persistent value error") @@ -1732,6 +1806,9 @@ async def content(self): return "Should not reach here" class DummyContext: + async def route(self, pattern, handler): + return + async def new_page(self): return DummyPage() @@ -1803,6 +1880,8 @@ async def test_ascrape_with_js_support_browser_name_override_new(monkeypatch): """ class DummyPage: + url = "http://example.com/" + async def goto(self, url, wait_until): return @@ -1813,6 +1892,9 @@ async def content(self): return "Firefox content" class DummyContext: + async def route(self, pattern, handler): + return + async def new_page(self): return DummyPage() @@ -2026,6 +2108,8 @@ async def dummy_ascrape(url, browser_name="chromium"): ) class DummyPage: + url = "http://example.com/" + async def goto(self, url, wait_until=None): return @@ -2036,6 +2120,9 @@ async def wait_for_load_state(self, state=None): return class DummyContext: + async def route(self, pattern, handler): + return + def __init__(self): self.new_page_called = False diff --git a/tests/test_error_page_detection.py b/tests/test_error_page_detection.py index 3c402ebd6..4fc6848cf 100644 --- a/tests/test_error_page_detection.py +++ b/tests/test_error_page_detection.py @@ -29,6 +29,7 @@ class _MockPage: def __init__(self, status): + self.url = "https://example.com/" response = MagicMock() response.status = status self.goto = AsyncMock(return_value=response) @@ -46,6 +47,7 @@ def playwright_with_status(): def _factory(status): page = _MockPage(status) context = MagicMock() + context.route = AsyncMock() context.new_page = AsyncMock(return_value=page) browser = MagicMock() browser.new_context = AsyncMock(return_value=context) diff --git a/tests/utils/test_url_safety.py b/tests/utils/test_url_safety.py new file mode 100644 index 000000000..19aced5b1 --- /dev/null +++ b/tests/utils/test_url_safety.py @@ -0,0 +1,220 @@ +import asyncio +import socket +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from scrapegraphai.docloaders.chromium import ChromiumLoader, _block_unsafe_request +from scrapegraphai.nodes.fetch_node import FetchNode +from scrapegraphai.utils.url_safety import validate_url + + +@pytest.mark.parametrize( + "url", + [ + "http://169.254.169.254/latest/meta-data/iam/security-credentials/", + "http://2852039166/latest/meta-data/", + "http://0xa9fea9fe/latest/meta-data/", + "http://[::ffff:169.254.169.254]/latest/meta-data/", + "http://[fe80::1]/", + "http://[fe80::1%25eth0]/", + "http://[fd00:ec2::254]/latest/meta-data/", + "http://100.100.100.200/latest/meta-data/", + ], +) +def test_validate_url_rejects_metadata_addresses(url): + with pytest.raises(ValueError): + validate_url(url) + + +@pytest.mark.parametrize( + "url", + [ + "http://127.0.0.1:8000/", + "http://localhost:11434/", + "http://192.168.1.10/", + "https://93.184.216.34/", + "not a url", + ], +) +def test_validate_url_allows_other_addresses(url): + validate_url(url) + + +def test_validate_url_checks_resolved_address(): + resolved = [(2, 1, 6, "", ("169.254.169.254", 0))] + with patch("socket.getaddrinfo", return_value=resolved): + with pytest.raises(ValueError): + validate_url("http://metadata.example.com/") + + +def test_validate_url_rejects_if_any_resolved_address_is_blocked(): + resolved = [ + (2, 1, 6, "", ("93.184.216.34", 0)), + (2, 1, 6, "", ("169.254.169.254", 0)), + ] + with patch("socket.getaddrinfo", return_value=resolved): + with pytest.raises(ValueError): + validate_url("http://dual.example.com/") + + +def test_validate_url_ignores_unresolvable_host(): + with patch("socket.getaddrinfo", side_effect=socket.gaierror): + validate_url("http://does-not-resolve.invalid/") + + +@pytest.mark.parametrize("use_soup", [True, False]) +def test_fetch_node_does_not_fetch_metadata_address(use_soup): + node = FetchNode( + input="url", + output=["doc"], + node_config={"use_soup": use_soup, "headless": True}, + ) + source = "http://169.254.169.254/latest/meta-data/" + with ( + patch("scrapegraphai.nodes.fetch_node.requests.get") as get, + patch("scrapegraphai.nodes.fetch_node.ChromiumLoader") as loader, + ): + with pytest.raises(ValueError): + node.execute({"url": source}) + get.assert_not_called() + loader.assert_not_called() + + +class _RedirectHandler(BaseHTTPRequestHandler): + hits = [] + + def do_GET(self): + self.hits.append(self.path) + self.send_response(302) + self.send_header("Location", "http://169.254.169.254/latest/meta-data/") + self.end_headers() + + def log_message(self, *args): + pass + + +def test_fetch_node_does_not_follow_redirect_to_metadata_address(): + _RedirectHandler.hits = [] + server = HTTPServer(("127.0.0.1", 0), _RedirectHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + node = FetchNode( + input="url", + output=["doc"], + node_config={"use_soup": True, "timeout": 2}, + ) + source = f"http://127.0.0.1:{server.server_address[1]}/" + try: + with pytest.raises(ValueError): + node.execute({"url": source}) + finally: + server.shutdown() + server.server_close() + assert _RedirectHandler.hits == ["/"] + + +@pytest.mark.parametrize( + "url, blocked", + [ + ("http://169.254.169.254/latest/meta-data/", True), + ("http://127.0.0.1:8000/", False), + ], +) +def test_browser_requests_to_metadata_address_are_aborted(url, blocked): + route = MagicMock() + route.request.url = url + route.abort = AsyncMock() + route.continue_ = AsyncMock() + asyncio.run(_block_unsafe_request(route)) + assert route.abort.called is blocked + assert route.continue_.called is not blocked + + +def _mock_playwright(page): + context = MagicMock() + context.route = AsyncMock() + context.new_page = AsyncMock(return_value=page) + browser = MagicMock() + browser.new_context = AsyncMock(return_value=context) + browser.close = AsyncMock() + playwright = MagicMock() + playwright.chromium.launch = AsyncMock(return_value=browser) + manager = MagicMock() + manager.__aenter__ = AsyncMock(return_value=playwright) + manager.__aexit__ = AsyncMock(return_value=False) + return manager, context + + +def _mock_page(url, content="secret"): + page = MagicMock() + page.goto = AsyncMock() + page.url = url + page.content = AsyncMock(return_value=content) + page.wait_for_load_state = AsyncMock() + page.evaluate = AsyncMock(return_value=1000) + page.mouse.wheel = AsyncMock() + return page + + +def _failure_message(manager, method): + # The scroll loader reports failures as an "Error: ..." string instead of raising + try: + result = _run_loader(manager, method) + except RuntimeError as exc: + return str(exc) + assert result.startswith("Error:"), result + assert "secret" not in result + return result + + +def _run_loader(manager, method="ascrape_playwright"): + loader = ChromiumLoader(["http://example.com/"], retry_limit=1) + with ( + patch("playwright.async_api.async_playwright", return_value=manager), + patch("undetected_playwright.Malenia.apply_stealth", new=AsyncMock()), + ): + return asyncio.run(getattr(loader, method)("http://example.com/")) + + +@pytest.mark.parametrize( + "method", + [ + "ascrape_playwright", + "ascrape_playwright_scroll", + "ascrape_with_js_support", + ], +) +def test_chromium_loader_rejects_navigation_that_lands_on_metadata_address(method): + page = _mock_page("http://169.254.169.254/latest/meta-data/") + manager, _ = _mock_playwright(page) + assert "metadata" in _failure_message(manager, method) + + +@pytest.mark.parametrize( + "method", + [ + "ascrape_playwright", + "ascrape_playwright_scroll", + "ascrape_with_js_support", + ], +) +def test_chromium_loader_rejects_navigation_after_goto_returned(method): + page = _mock_page("http://example.com/") + + async def navigate_away(): + page.url = "http://169.254.169.254/latest/meta-data/" + return "secret" + + page.content = AsyncMock(side_effect=navigate_away) + manager, _ = _mock_playwright(page) + assert "metadata" in _failure_message(manager, method) + + +def test_chromium_loader_registers_request_guard_and_returns_safe_page(): + page = _mock_page("http://example.com/", "ok") + manager, context = _mock_playwright(page) + assert _run_loader(manager) == "ok" + context.route.assert_awaited_once_with("**/*", _block_unsafe_request)