diff --git a/docs/guides/architecture_overview.mdx b/docs/guides/architecture_overview.mdx index edfe5564f2..1554add5f1 100644 --- a/docs/guides/architecture_overview.mdx +++ b/docs/guides/architecture_overview.mdx @@ -120,8 +120,6 @@ class BasicCrawlingContext class HttpCrawlingContext -class HttpCrawlingResult - class ParsedHttpCrawlingContext class ParselCrawlingContext @@ -152,8 +150,6 @@ class StagehandCrawlingContext BasicCrawlingContext --|> HttpCrawlingContext -HttpCrawlingResult --|> HttpCrawlingContext - HttpCrawlingContext --|> ParsedHttpCrawlingContext ParsedHttpCrawlingContext --|> ParselCrawlingContext diff --git a/docs/upgrading/upgrading_to_v2.md b/docs/upgrading/upgrading_to_v2.md index ef2a766c21..033593d30a 100644 --- a/docs/upgrading/upgrading_to_v2.md +++ b/docs/upgrading/upgrading_to_v2.md @@ -326,7 +326,7 @@ The crawler now reads a stall from the status, so it also sees a `ThrottlingRequ ### Responses are read lazily -`HttpClient.crawl` and `HttpClient.stream` are removed. `send_request` returns the response as soon as its headers arrive and reads the body only on `read` or `read_stream`. A response whose body you don't read keeps its connection until you call `aclose`. Use the response as an async context manager to close it: +`HttpClient.crawl` and `HttpClient.stream` are removed, and so is `HttpCrawlingResult`, which `crawl` returned. `HttpCrawlingContext` still has `http_response`. `http_response` now comes after the fields of `BasicCrawlingContext` instead of before them, so if you build a crawling context yourself, pass the fields by keyword. `send_request` returns the response as soon as its headers arrive and reads the body only on `read` or `read_stream`. A response whose body you don't read keeps its connection until you call `aclose`. Use the response as an async context manager to close it: ```python # v1 diff --git a/src/crawlee/crawlers/__init__.py b/src/crawlee/crawlers/__init__.py index cf75a3e84e..66987df426 100644 --- a/src/crawlee/crawlers/__init__.py +++ b/src/crawlee/crawlers/__init__.py @@ -4,7 +4,7 @@ from ._abstract_http import AbstractHttpCrawler, AbstractHttpParser, HttpCrawlerOptions, ParsedHttpCrawlingContext from ._basic import BasicCrawler, BasicCrawlerOptions, BasicCrawlingContext, ContextPipeline from ._file_download import FileDownloadCrawler, FileDownloadCrawlingContext -from ._http import HttpCrawler, HttpCrawlingContext, HttpCrawlingResult +from ._http import HttpCrawler, HttpCrawlingContext _install_import_hook(__name__) @@ -129,7 +129,6 @@ 'HttpCrawler', 'HttpCrawlerOptions', 'HttpCrawlingContext', - 'HttpCrawlingResult', 'ParsedHttpCrawlingContext', 'ParselCrawler', 'ParselCrawlingContext', diff --git a/src/crawlee/crawlers/_abstract_http/_http_crawling_context.py b/src/crawlee/crawlers/_abstract_http/_http_crawling_context.py index e687e16a68..141704246e 100644 --- a/src/crawlee/crawlers/_abstract_http/_http_crawling_context.py +++ b/src/crawlee/crawlers/_abstract_http/_http_crawling_context.py @@ -1,11 +1,13 @@ from __future__ import annotations from dataclasses import dataclass, fields -from typing import Generic, Self, TypeVar +from typing import TYPE_CHECKING, Generic, Self, TypeVar from crawlee._types import BasicCrawlingContext, EnqueueLinksFunction, ExtractLinksFunction, PageSnapshot from crawlee._utils.docs import docs_group -from crawlee.http_clients import HttpCrawlingResult, HttpResponse + +if TYPE_CHECKING: + from crawlee.http_clients import HttpResponse TParseResult = TypeVar('TParseResult') TSelectResult = TypeVar('TSelectResult') @@ -13,9 +15,12 @@ @dataclass(frozen=True) @docs_group('Crawling contexts') -class HttpCrawlingContext(BasicCrawlingContext, HttpCrawlingResult): +class HttpCrawlingContext(BasicCrawlingContext): """The crawling context used by the `AbstractHttpCrawler`.""" + http_response: HttpResponse + """The HTTP response received from the server.""" + @classmethod def from_basic_crawling_context(cls, context: BasicCrawlingContext, http_response: HttpResponse) -> Self: """Initialize a new instance from an existing `BasicCrawlingContext`.""" diff --git a/src/crawlee/crawlers/_http/__init__.py b/src/crawlee/crawlers/_http/__init__.py index e43279c906..fb0ccaf8a5 100644 --- a/src/crawlee/crawlers/_http/__init__.py +++ b/src/crawlee/crawlers/_http/__init__.py @@ -1,10 +1,8 @@ from crawlee.crawlers._abstract_http._http_crawling_context import HttpCrawlingContext -from crawlee.http_clients import HttpCrawlingResult from ._http_crawler import HttpCrawler __all__ = [ 'HttpCrawler', 'HttpCrawlingContext', - 'HttpCrawlingResult', ] diff --git a/src/crawlee/http_clients/__init__.py b/src/crawlee/http_clients/__init__.py index 8fecff89ed..3b7bad92e8 100644 --- a/src/crawlee/http_clients/__init__.py +++ b/src/crawlee/http_clients/__init__.py @@ -2,7 +2,7 @@ from crawlee._utils.try_import import try_import as _try_import # These imports have only mandatory dependencies, so they are imported directly. -from ._base import HttpClient, HttpCrawlingResult, HttpResponse +from ._base import HttpClient, HttpResponse from ._impit import ImpitHttpClient _install_import_hook(__name__) @@ -19,7 +19,6 @@ __all__ = [ 'CurlImpersonateHttpClient', 'HttpClient', - 'HttpCrawlingResult', 'HttpResponse', 'HttpxHttpClient', 'ImpitHttpClient', diff --git a/src/crawlee/http_clients/_base.py b/src/crawlee/http_clients/_base.py index 3fa4354996..e439fc4ed0 100644 --- a/src/crawlee/http_clients/_base.py +++ b/src/crawlee/http_clients/_base.py @@ -2,7 +2,6 @@ import asyncio from abc import ABC, abstractmethod -from dataclasses import dataclass from datetime import timedelta from http import HTTPStatus from time import monotonic @@ -168,19 +167,6 @@ async def __aexit__( await self.aclose() -@dataclass(frozen=True) -@docs_group('Crawling contexts') -class HttpCrawlingResult: - """Result of an HTTP-only crawl. - - Mainly for the purpose of composing specific crawling contexts (e.g. `BeautifulSoupCrawlingContext`, - `ParselCrawlingContext`, ...). - """ - - http_response: HttpResponse - """The HTTP response received from the server.""" - - @docs_group('HTTP clients') class HttpClient(ABC): """An abstract base class for HTTP clients used in crawlers (`BasicCrawler` subclasses).