From fe72b4c9ee2322f440b6727078e57a4cbdffeccd Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Tue, 3 Nov 2026 15:45:13 +0000 Subject: [PATCH 1/3] replace `persist_cookies_per_session` with a `save_response_cookies` crawler option --- docs/03_concepts/15_cookie_management.mdx | 4 ++-- docs/06_upgrading/upgrading_to_v2.md | 16 +++++++++++++++- src/crawlee/_types.py | 2 +- .../_abstract_http/_abstract_http_crawler.py | 16 +++++++++++++++- src/crawlee/crawlers/_http/_http_crawler.py | 4 ++-- .../_playwright/_playwright_crawler.py | 10 +++++++++- .../_playwright/_playwright_http_client.py | 2 +- .../crawlers/_stagehand/_stagehand_crawler.py | 3 +++ src/crawlee/http_clients/_base.py | 19 ++++--------------- src/crawlee/http_clients/_curl_impersonate.py | 3 --- src/crawlee/http_clients/_httpx.py | 3 --- src/crawlee/http_clients/_impit.py | 3 --- .../_stagehand/test_stagehand_crawler.py | 7 +++++++ 13 files changed, 59 insertions(+), 33 deletions(-) diff --git a/docs/03_concepts/15_cookie_management.mdx b/docs/03_concepts/15_cookie_management.mdx index cfd93421f0..973fc1a600 100644 --- a/docs/03_concepts/15_cookie_management.mdx +++ b/docs/03_concepts/15_cookie_management.mdx @@ -17,7 +17,7 @@ import PersistCookies from '!!raw-loader!roa-loader!./code_examples/cookie_manag Cookies carry login state, consent flags, CSRF tokens, and other per-user data that a site expects to see on every request. -In Crawlee, cookies are stored on the `Session` that handles a request, in a `SessionCookies` jar. When a request runs on a session, the HTTP client sends the session's cookies with the request and writes any `Set-Cookie` from the response back onto the same jar. The next request on that session carries the updated cookies. +In Crawlee, cookies are stored on the `Session` that handles a request, in a `SessionCookies` jar. When a request runs on a session, the HTTP client sends the session's cookies with the request and writes any `Set-Cookie` from the response back onto the same jar. The next request on that session carries the updated cookies. To keep the response cookies out of the session, set `save_response_cookies` to `False` on the crawler. The session's cookies are still sent. Cookies follow the session, not the URL. Two requests share cookies when they run on the same session. For how sessions are created and rotated, see the [Session management](./session-management) guide. @@ -49,7 +49,7 @@ To start every session with a known set of cookies, for example a consent flag o ## Cookies with PlaywrightCrawler -`PlaywrightCrawler` keeps the session jar and the browser context in sync. Before navigation, the session's cookies are loaded into the browser context. After the handler returns, cookies from the browser context are merged back onto the session. The merge captures whatever the page picked up through `Set-Cookie`, JavaScript, or redirects. +`PlaywrightCrawler` keeps the session jar and the browser context in sync. Before navigation, the session's cookies are loaded into the browser context. After the handler returns, cookies from the browser context are merged back onto the session. The merge captures whatever the page picked up through `Set-Cookie`, JavaScript, or redirects. With `save_response_cookies` set to `False`, the crawler skips the merge. The first request receives a cookie in the browser. Crawlee merges it onto the session, so a later request on the same session sees it. diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md index 4b90cdcf52..ad82f2ceb8 100644 --- a/docs/06_upgrading/upgrading_to_v2.md +++ b/docs/06_upgrading/upgrading_to_v2.md @@ -389,7 +389,21 @@ A `Cookie` header passed with a request or set on the client is merged with the `send_request` and `context.send_request` take a new `cookies` argument, the cookies to send and store instead of the session cookies for a single call. To send no session cookies, pass `cookies=SessionCookies()`. With `PlaywrightCrawler`, the response cookies still reach the session through the browser context. -Without a session, or with `persist_cookies_per_session=False`, the cookies a server sets during a redirect chain are sent with the following requests of that chain. In v1, `ImpitHttpClient` and `HttpxHttpClient` didn't send them. +Without a session, or with `save_response_cookies=False` on the crawler, the cookies a server sets during a redirect chain are sent with the following requests of that chain. In v1, `ImpitHttpClient` and `HttpxHttpClient` didn't send them. + +### `save_response_cookies` replaces `persist_cookies_per_session` + +`persist_cookies_per_session` is removed from the HTTP clients. To stop saving the response cookies to the session, pass `save_response_cookies=False` to the crawler. The session cookies are still sent. `PlaywrightCrawler` and `StagehandCrawler` take the same option, which stops saving the cookies of the page to the session after the request handler. + +```python +# v1 +crawler = ParselCrawler(http_client=ImpitHttpClient(persist_cookies_per_session=False)) + +# v2 +crawler = ParselCrawler(save_response_cookies=False) +``` + +With `BasicCrawler` and the HTTP crawlers, `context.send_request` always saves the response cookies to the session, unless you pass other cookies in its `cookies` argument. In v1, `persist_cookies_per_session=False` applied to `context.send_request` too. With `PlaywrightCrawler` and its default HTTP client, `context.send_request` stores the response cookies in the browser context, so `save_response_cookies=False` keeps them out of the session as well. ### Some client arguments are ignored diff --git a/src/crawlee/_types.py b/src/crawlee/_types.py index 541e246f6f..3456d939e5 100644 --- a/src/crawlee/_types.py +++ b/src/crawlee/_types.py @@ -593,7 +593,7 @@ def __call__( payload: The payload to include in the request. cookies: The cookies to send and store instead of the cookies of the session. With `PlaywrightCrawler`, the response cookies are stored in the browser context as well, and so in the session after the - request handler. + request handler unless `save_response_cookies` is disabled. timeout: Maximum time allowed to receive the response, including all redirects, and to read its body. Without it, the default timeout of the HTTP client applies. diff --git a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py index 4144723fd6..2cd008913f 100644 --- a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py +++ b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py @@ -16,6 +16,7 @@ from crawlee._utils.urls import convert_to_absolute_url, is_url_absolute, to_absolute_url_iterator from crawlee.crawlers._basic import BasicCrawler, BasicCrawlerOptions, ContextPipeline from crawlee.errors import RequestThrottledError, SessionError +from crawlee.sessions import SessionCookies from crawlee.statistics import StatisticsState from ._http_crawling_context import HttpCrawlingContext, ParsedHttpCrawlingContext, TParseResult, TSelectResult @@ -45,6 +46,11 @@ class HttpCrawlerOptions( navigation_timeout: NotRequired[timedelta | None] """Timeout for the HTTP request.""" + save_response_cookies: NotRequired[bool] + """Whether to save the cookies set by the responses to the session. The session cookies are sent either way. The + option doesn't apply to `context.send_request`. + """ + @docs_group('Crawlers') class AbstractHttpCrawler( @@ -71,10 +77,12 @@ def __init__( *, parser: AbstractHttpParser[TParseResult, TSelectResult], navigation_timeout: timedelta | None = None, + save_response_cookies: bool = True, **kwargs: Unpack[BasicCrawlerOptions[TCrawlingContext, StatisticsState]], ) -> None: self._parser = parser self._navigation_timeout = navigation_timeout or timedelta(minutes=1) + self._save_response_cookies = save_response_cookies self._pre_navigation_hooks: list[Callable[[BasicCrawlingContext], Awaitable[None]]] = [] self._post_navigation_hooks: list[Callable[[HttpCrawlingContext], Awaitable[None]]] = [] self._shared_navigation_timeouts: dict[int, SharedTimeout] = {} @@ -104,7 +112,7 @@ class _ParsedHttpCrawler(AbstractHttpCrawler): def __init__( self, parser: AbstractHttpParser[TParseResult, TSelectResult] = static_parser, # ty: ignore[invalid-parameter-default] - **kwargs: Unpack[BasicCrawlerOptions[ParsedHttpCrawlingContext[TParseResult]]], + **kwargs: Unpack[HttpCrawlerOptions[ParsedHttpCrawlingContext[TParseResult]]], ) -> None: kwargs['_context_pipeline'] = self._create_static_content_crawler_pipeline() super().__init__( @@ -297,6 +305,11 @@ async def _make_http_request(self, context: BasicCrawlingContext) -> AsyncGenera Yields: The original crawling context enhanced by HTTP response. """ + # Without saving, the session cookies are still sent, but the response cookies go to a copy. + cookies = None + if context.session is not None and not self._save_response_cookies: + cookies = SessionCookies(context.session.cookies) + async with self._shared_navigation_timeouts[id(context.request)] as remaining_timeout: response = await self._http_client.send_request( context.request.url, @@ -304,6 +317,7 @@ async def _make_http_request(self, context: BasicCrawlingContext) -> AsyncGenera headers=context.request.headers, payload=context.request.payload, session=context.session, + cookies=cookies, proxy_info=context.proxy_info, timeout=remaining_timeout, ) diff --git a/src/crawlee/crawlers/_http/_http_crawler.py b/src/crawlee/crawlers/_http/_http_crawler.py index d2fd82f172..3c985a33a4 100644 --- a/src/crawlee/crawlers/_http/_http_crawler.py +++ b/src/crawlee/crawlers/_http/_http_crawler.py @@ -10,7 +10,7 @@ if TYPE_CHECKING: from typing import Unpack - from crawlee.crawlers import BasicCrawlerOptions + from crawlee.crawlers import HttpCrawlerOptions @docs_group('Crawlers') @@ -48,7 +48,7 @@ async def request_handler(context: HttpCrawlingContext) -> None: def __init__( self, - **kwargs: Unpack[BasicCrawlerOptions[ParsedHttpCrawlingContext[bytes]]], + **kwargs: Unpack[HttpCrawlerOptions[ParsedHttpCrawlingContext[bytes]]], ) -> None: """Initialize a new instance. diff --git a/src/crawlee/crawlers/_playwright/_playwright_crawler.py b/src/crawlee/crawlers/_playwright/_playwright_crawler.py index 711872375a..07e4e76903 100644 --- a/src/crawlee/crawlers/_playwright/_playwright_crawler.py +++ b/src/crawlee/crawlers/_playwright/_playwright_crawler.py @@ -123,6 +123,7 @@ def __init__( headless: bool | None = None, use_incognito_pages: bool | None = None, navigation_timeout: timedelta | None = None, + save_response_cookies: bool = True, **kwargs: Unpack[BasicCrawlerOptions[TCrawlingContext, StatisticsState]], ) -> None: """Initialize a new instance. @@ -155,9 +156,11 @@ def __init__( the request handler) goto_options: Additional options to pass to Playwright's `Page.goto()` method. The `timeout` option is not supported, use `navigation_timeout` instead. + save_response_cookies: Whether to save the cookies of the page to the session after the request handler. kwargs: Additional keyword arguments to pass to the underlying `BasicCrawler`. """ self._shared_navigation_timeouts: dict[int, SharedTimeout] = {} + self._save_response_cookies = save_response_cookies if browser_pool: # Raise an exception if browser_pool is provided together with other browser-related arguments. @@ -584,7 +587,7 @@ async def _execute_post_navigation_hooks(self, context: TPostNavContext) -> Asyn async def _create_crawling_context(self, context: TPostNavContext) -> AsyncGenerator[TCrawlingContext, None]: yield self._build_crawling_context(context) - if context.session: + if context.session and self._save_response_cookies: pw_cookies = await self._get_cookies(context.page) context.session.cookies.set_cookies_from_playwright_format(pw_cookies) @@ -668,3 +671,8 @@ class PlaywrightCrawlerOptions( It is intended for typing forwarded `__init__` arguments in the subclasses. """ + + # Kept out of `_PlaywrightCrawlerAdditionalOptions`, since `AdaptivePlaywrightCrawler` passes those to its browser + # sub-crawler only, while its static sub-crawler would keep saving the response cookies. + save_response_cookies: NotRequired[bool] + """Whether to save the cookies of the page to the session after the request handler.""" diff --git a/src/crawlee/crawlers/_playwright/_playwright_http_client.py b/src/crawlee/crawlers/_playwright/_playwright_http_client.py index 0e3d0d6c7f..cf7b577fa1 100644 --- a/src/crawlee/crawlers/_playwright/_playwright_http_client.py +++ b/src/crawlee/crawlers/_playwright/_playwright_http_client.py @@ -71,7 +71,7 @@ async def send_request( try: # The browser context holds the cookies of the session, added before the navigation, with the changes the # page made since. The browser stores the response cookies there too, and the crawler saves them to the - # session after the request handler. + # session after the request handler unless `save_response_cookies` is disabled. if page is not None and cookies is None: context_cookies = await page.context.cookies() cookies = SessionCookies() diff --git a/src/crawlee/crawlers/_stagehand/_stagehand_crawler.py b/src/crawlee/crawlers/_stagehand/_stagehand_crawler.py index c4f3cf9bd0..ac714a1508 100644 --- a/src/crawlee/crawlers/_stagehand/_stagehand_crawler.py +++ b/src/crawlee/crawlers/_stagehand/_stagehand_crawler.py @@ -101,6 +101,7 @@ def __init__( browser_new_context_options: dict[str, Any] | None = None, goto_options: GotoOptions | None = None, navigation_timeout: timedelta | None = None, + save_response_cookies: bool = True, **kwargs: Unpack[BasicCrawlerOptions[StagehandCrawlingContext, StatisticsState]], ) -> None: """Initialize a new instance. @@ -124,6 +125,7 @@ def __init__( supported - use `navigation_timeout` instead. navigation_timeout: Timeout for the navigation phase (from opening the page to calling the request handler). Defaults to one minute. + save_response_cookies: Whether to save the cookies of the page to the session after the request handler. kwargs: Additional keyword arguments forwarded to `BasicCrawler`. """ if browser_pool is not None: @@ -164,6 +166,7 @@ def __init__( browser_pool=browser_pool, goto_options=goto_options, navigation_timeout=navigation_timeout, + save_response_cookies=save_response_cookies, **kwargs, ) diff --git a/src/crawlee/http_clients/_base.py b/src/crawlee/http_clients/_base.py index e4e84a2232..a49ec9f703 100644 --- a/src/crawlee/http_clients/_base.py +++ b/src/crawlee/http_clients/_base.py @@ -194,19 +194,16 @@ class HttpClient(ABC): def __init__( self, *, - persist_cookies_per_session: bool = True, follow_redirects: bool = True, headers: Mapping[str, str] | None = None, ) -> None: """Initialize a new instance. Args: - persist_cookies_per_session: Whether to persist cookies per HTTP session. follow_redirects: Whether `send_request` follows redirects. Without following them, it returns the first response as it is. headers: The headers to include in every request. The headers passed to `send_request` win over them. """ - self._persist_cookies_per_session = persist_cookies_per_session self._follow_redirects = follow_redirects self._headers = HttpHeaders(headers or {}) @@ -244,11 +241,9 @@ async def send_request( method: The HTTP method to use. headers: The headers to include in the request. payload: The data to be sent as the request body. - session: The session whose cookies are sent. The response cookies are stored in it when - `persist_cookies_per_session` is enabled. Otherwise, and without a session, they are kept only for the - redirect chain of this request. - cookies: The cookies to send and store instead of the cookies of `session`, even with - `persist_cookies_per_session` disabled. + session: The session whose cookies are sent and stored. Without a session or `cookies`, the response + cookies are kept only for the redirect chain of this request. + cookies: The cookies to send and store instead of the cookies of `session`. proxy_info: The information about the proxy to be used. timeout: Maximum time allowed to receive the response, including all redirects, and to read its body. Without it, the default timeout of the HTTP client applies. @@ -267,13 +262,7 @@ async def send_request( headers = self._headers | HttpHeaders(headers or {}) if cookies is None: - if session is None: - cookies = SessionCookies() - elif self._persist_cookies_per_session: - cookies = session.cookies - else: - # The cookies of the session are still sent, but the response cookies go to a copy. - cookies = SessionCookies(session.cookies) + cookies = session.cookies if session is not None else SessionCookies() # `encoded=True` keeps the path and the query byte for byte, including a safe `%2F` in query. current_url = _with_ascii_host(URL(url, encoded=True)) diff --git a/src/crawlee/http_clients/_curl_impersonate.py b/src/crawlee/http_clients/_curl_impersonate.py index f36155ac34..0deccb8e12 100644 --- a/src/crawlee/http_clients/_curl_impersonate.py +++ b/src/crawlee/http_clients/_curl_impersonate.py @@ -170,14 +170,12 @@ class CurlImpersonateHttpClient(HttpClient): def __init__( self, *, - persist_cookies_per_session: bool = True, follow_redirects: bool = True, **async_session_kwargs: Any, ) -> None: """Initialize a new instance. Args: - persist_cookies_per_session: Whether to persist cookies per HTTP session. follow_redirects: Whether to follow HTTP redirects. async_session_kwargs: Additional keyword arguments for `curl_cffi.requests.AsyncSession`. Some of them are ignored with a warning. Cookies are kept in sessions, so `cookies` is ignored. The `auth` and `params` @@ -186,7 +184,6 @@ def __init__( and `max_redirects` are ignored. """ super().__init__( - persist_cookies_per_session=persist_cookies_per_session, follow_redirects=follow_redirects, headers=async_session_kwargs.pop('headers', None), ) diff --git a/src/crawlee/http_clients/_httpx.py b/src/crawlee/http_clients/_httpx.py index 2f9db5d752..0810826458 100644 --- a/src/crawlee/http_clients/_httpx.py +++ b/src/crawlee/http_clients/_httpx.py @@ -115,7 +115,6 @@ class HttpxHttpClient(HttpClient): def __init__( self, *, - persist_cookies_per_session: bool = True, http1: bool = True, http2: bool = True, verify: str | bool | SSLContext = True, @@ -126,7 +125,6 @@ def __init__( """Initialize a new instance. Args: - persist_cookies_per_session: Whether to persist cookies per HTTP session. http1: Whether to enable HTTP/1.1 support. http2: Whether to enable HTTP/2 support. verify: SSL certificates used to verify the identity of requested hosts. @@ -141,7 +139,6 @@ def __init__( `limits` argument applies per proxy, because every proxy gets a connection pool of its own. """ super().__init__( - persist_cookies_per_session=persist_cookies_per_session, follow_redirects=follow_redirects, headers=async_client_kwargs.pop('headers', None), ) diff --git a/src/crawlee/http_clients/_impit.py b/src/crawlee/http_clients/_impit.py index 6694229470..7e3312ada9 100644 --- a/src/crawlee/http_clients/_impit.py +++ b/src/crawlee/http_clients/_impit.py @@ -113,7 +113,6 @@ class ImpitHttpClient(HttpClient): def __init__( self, *, - persist_cookies_per_session: bool = True, http3: bool = False, verify: bool = True, browser: Browser | None = 'firefox', @@ -123,7 +122,6 @@ def __init__( """Initialize a new instance. Args: - persist_cookies_per_session: Whether to persist cookies per HTTP session. http3: Whether to enable HTTP/3 support. verify: SSL certificates used to verify the identity of requested hosts. browser: Browser to impersonate. @@ -133,7 +131,6 @@ def __init__( by `send_request`, so `max_redirects` is ignored. """ super().__init__( - persist_cookies_per_session=persist_cookies_per_session, follow_redirects=follow_redirects, headers=async_client_kwargs.pop('headers', None), ) diff --git a/tests/unit/crawlers/_stagehand/test_stagehand_crawler.py b/tests/unit/crawlers/_stagehand/test_stagehand_crawler.py index 091df63977..2604ca53da 100644 --- a/tests/unit/crawlers/_stagehand/test_stagehand_crawler.py +++ b/tests/unit/crawlers/_stagehand/test_stagehand_crawler.py @@ -41,6 +41,13 @@ def test_init_with_valid_browser_pool() -> None: StagehandCrawler(browser_pool=pool) +def test_forwards_save_response_cookies() -> None: + """`save_response_cookies` reaches the `PlaywrightCrawler` that saves the cookies of the page.""" + crawler = StagehandCrawler(save_response_cookies=False) + + assert crawler._save_response_cookies is False + + def test_import_error_handled() -> None: blocked = { mod_name: None for mod_name in sys.modules if mod_name == 'stagehand' or mod_name.startswith('stagehand.') From 7f0fe1b3b171c44058aa2096212bca40795337f6 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Thu, 5 Nov 2026 15:19:19 +0000 Subject: [PATCH 2/3] add `save_response_cookies` to `AdaptivePlaywrightCrawler` --- docs/06_upgrading/upgrading_to_v2.md | 2 +- .../_adaptive_playwright_crawler.py | 9 +++++ .../_playwright/_playwright_crawler.py | 2 +- .../test_adaptive_playwright_crawler.py | 35 +++++++++++++++++++ 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md index ad82f2ceb8..c5d40579a7 100644 --- a/docs/06_upgrading/upgrading_to_v2.md +++ b/docs/06_upgrading/upgrading_to_v2.md @@ -393,7 +393,7 @@ Without a session, or with `save_response_cookies=False` on the crawler, the coo ### `save_response_cookies` replaces `persist_cookies_per_session` -`persist_cookies_per_session` is removed from the HTTP clients. To stop saving the response cookies to the session, pass `save_response_cookies=False` to the crawler. The session cookies are still sent. `PlaywrightCrawler` and `StagehandCrawler` take the same option, which stops saving the cookies of the page to the session after the request handler. +`persist_cookies_per_session` is removed from the HTTP clients. To stop saving the response cookies to the session, pass `save_response_cookies=False` to the crawler. The session cookies are still sent. `PlaywrightCrawler` and `StagehandCrawler` take the same option, which stops saving the cookies of the page to the session after the request handler. In `AdaptivePlaywrightCrawler`, the option applies to both the HTTP and the browser crawling. ```python # v1 diff --git a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py index fa9607bcef..8f8999e58f 100644 --- a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py +++ b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py @@ -125,6 +125,7 @@ def __init__( result_checker: Callable[[RequestHandlerRunResult], bool] | None = None, result_comparator: Callable[[RequestHandlerRunResult, RequestHandlerRunResult], bool] | None = None, playwright_crawler_specific_kwargs: _PlaywrightCrawlerAdditionalOptions | None = None, + save_response_cookies: bool = True, statistics: Statistics[AdaptivePlaywrightCrawlerStatisticState] | None = None, **kwargs: Unpack[_BasicCrawlerOptions], ) -> None: @@ -140,6 +141,8 @@ def __init__( static_parser: Implementation of `AbstractHttpParser`. Parser that will be used for static crawling. static_crawler_specific_kwargs: `AbstractHttpCrawler` only kwargs that are passed to the sub crawler. playwright_crawler_specific_kwargs: `PlaywrightCrawler` only kwargs that are passed to the sub crawler. + save_response_cookies: Whether to save the response cookies to the session, in both the static and the + browser sub crawler. statistics: A custom `Statistics[AdaptivePlaywrightCrawlerStatisticState]` instance, allowing the use of non-default configuration. kwargs: Additional keyword arguments to pass to the underlying `BasicCrawler`. @@ -174,10 +177,12 @@ def __init__( static_crawler = static_crawler_class( parser=static_parser, + save_response_cookies=save_response_cookies, statistics=_NonPersistentStatistics(), **basic_crawler_kwargs_for_static_crawler, ) playwright_crawler = PlaywrightCrawler( + save_response_cookies=save_response_cookies, statistics=_NonPersistentStatistics(), **playwright_crawler_specific_kwargs, **basic_crawler_kwargs_for_pw_crawler, @@ -241,6 +246,7 @@ def with_beautifulsoup_static_parser( result_comparator: Callable[[RequestHandlerRunResult, RequestHandlerRunResult], bool] | None = None, parser_type: BeautifulSoupParserType = 'lxml', playwright_crawler_specific_kwargs: _PlaywrightCrawlerAdditionalOptions | None = None, + save_response_cookies: bool = True, statistics: Statistics[StatisticsState] | None = None, **kwargs: Unpack[_BasicCrawlerOptions], ) -> AdaptivePlaywrightCrawler[ParsedHttpCrawlingContext[BeautifulSoup], BeautifulSoup, Tag]: @@ -255,6 +261,7 @@ def with_beautifulsoup_static_parser( result_comparator=result_comparator, static_parser=BeautifulSoupParser(parser=parser_type), playwright_crawler_specific_kwargs=playwright_crawler_specific_kwargs, + save_response_cookies=save_response_cookies, statistics=adaptive_statistics, **kwargs, ) @@ -267,6 +274,7 @@ def with_parsel_static_parser( result_checker: Callable[[RequestHandlerRunResult], bool] | None = None, result_comparator: Callable[[RequestHandlerRunResult, RequestHandlerRunResult], bool] | None = None, playwright_crawler_specific_kwargs: _PlaywrightCrawlerAdditionalOptions | None = None, + save_response_cookies: bool = True, statistics: Statistics[StatisticsState] | None = None, **kwargs: Unpack[_BasicCrawlerOptions], ) -> AdaptivePlaywrightCrawler[ParsedHttpCrawlingContext[Selector], Selector, Selector]: @@ -281,6 +289,7 @@ def with_parsel_static_parser( result_comparator=result_comparator, static_parser=ParselParser(), playwright_crawler_specific_kwargs=playwright_crawler_specific_kwargs, + save_response_cookies=save_response_cookies, statistics=adaptive_statistics, **kwargs, ) diff --git a/src/crawlee/crawlers/_playwright/_playwright_crawler.py b/src/crawlee/crawlers/_playwright/_playwright_crawler.py index 07e4e76903..e5a561d422 100644 --- a/src/crawlee/crawlers/_playwright/_playwright_crawler.py +++ b/src/crawlee/crawlers/_playwright/_playwright_crawler.py @@ -673,6 +673,6 @@ class PlaywrightCrawlerOptions( """ # Kept out of `_PlaywrightCrawlerAdditionalOptions`, since `AdaptivePlaywrightCrawler` passes those to its browser - # sub-crawler only, while its static sub-crawler would keep saving the response cookies. + # sub-crawler only. It takes `save_response_cookies` as its own argument and passes it to both sub-crawlers. save_response_cookies: NotRequired[bool] """Whether to save the cookies of the page to the session after the request handler.""" diff --git a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py index e571334b6e..dffa6a9a3e 100644 --- a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py +++ b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py @@ -17,6 +17,41 @@ async def test_accepts_concurrency_system() -> None: assert crawler.concurrency_system is concurrency_system +@pytest.mark.parametrize( + ('rendering_type', 'save_response_cookies', 'expected_cookies'), + [ + pytest.param('static', True, {'a'}, id='static, saved'), + pytest.param('static', False, set(), id='static, not saved'), + pytest.param('client only', True, {'a'}, id='browser, saved'), + pytest.param('client only', False, set(), id='browser, not saved'), + ], +) +async def test_save_response_cookies( + *, rendering_type: RenderingType, save_response_cookies: bool, expected_cookies: set[str], server_url: URL +) -> None: + """Test that `save_response_cookies` controls saving the response cookies to the session in both sub crawlers.""" + predictor = _SimpleRenderingTypePredictor( + rendering_types=cycle([rendering_type]), detection_probability_recommendation=cycle([0]) + ) + + async with SessionPool(max_pool_size=1) as session_pool: + crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser( + rendering_type_predictor=predictor, + save_response_cookies=save_response_cookies, + session_pool=session_pool, + ) + + @crawler.router.default_handler + async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: + pass + + await crawler.run([str(server_url.with_path('set_cookies').extend_query(a=1))]) + + session = await session_pool.get_session() + + assert {cookie['name'] for cookie in session.cookies.get_cookies_as_dicts()} == expected_cookies + + @pytest.mark.parametrize( 'optional_module_name', [ From 734da3efbb864bf28934e9533e76adcdb6e224e3 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Thu, 5 Nov 2026 20:02:40 +0000 Subject: [PATCH 3/3] restructure tests --- .../test_adaptive_playwright_crawler.py | 35 ++++++++ .../crawlers/_http/test_http_crawler.py | 41 +++++++++- .../_playwright/test_playwright_crawler.py | 22 +++++ .../http_clients/test_http_clients.py | 80 +------------------ .../test_adaptive_playwright_crawler.py | 35 -------- 5 files changed, 99 insertions(+), 114 deletions(-) diff --git a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py index 37e9db54bc..813fe6e50e 100644 --- a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py +++ b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py @@ -1100,3 +1100,38 @@ async def handler(context: AdaptivePlaywrightCrawlingContext) -> None: await crawler.run(test_urls[:1]) assert [c.args[0] for c in rendering_types.call_args_list] == ['static', 'client only'] + + +@pytest.mark.parametrize( + ('rendering_type', 'save_response_cookies', 'expected_cookies'), + [ + pytest.param('static', True, {'a'}, id='static, saved'), + pytest.param('static', False, set(), id='static, not saved'), + pytest.param('client only', True, {'a'}, id='browser, saved'), + pytest.param('client only', False, set(), id='browser, not saved'), + ], +) +async def test_save_response_cookies( + *, rendering_type: RenderingType, save_response_cookies: bool, expected_cookies: set[str], server_url: URL +) -> None: + """Test that `save_response_cookies` controls saving the response cookies to the session in both sub crawlers.""" + predictor = _SimpleRenderingTypePredictor( + rendering_types=cycle([rendering_type]), detection_probability_recommendation=cycle([0]) + ) + + async with SessionPool(max_pool_size=1) as session_pool: + crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser( + rendering_type_predictor=predictor, + save_response_cookies=save_response_cookies, + session_pool=session_pool, + ) + + @crawler.router.default_handler + async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: + pass + + await crawler.run([str(server_url.with_path('set_cookies').extend_query(a=1))]) + + session = await session_pool.get_session() + + assert {cookie['name'] for cookie in session.cookies.get_cookies_as_dicts()} == expected_cookies diff --git a/tests/integration/crawlers/_http/test_http_crawler.py b/tests/integration/crawlers/_http/test_http_crawler.py index 99877c5053..b11f56eca3 100644 --- a/tests/integration/crawlers/_http/test_http_crawler.py +++ b/tests/integration/crawlers/_http/test_http_crawler.py @@ -14,7 +14,7 @@ from crawlee.crawlers import HttpCrawler from crawlee.errors import PersistentRateLimitError from crawlee.request_loaders import ThrottlingRequestManager, _throttling_request_manager -from crawlee.sessions import SessionPool +from crawlee.sessions import CookieParam, SessionPool from crawlee.statistics import Statistics from crawlee.storages import RequestQueue from tests.integration.server_endpoints import HELLO_WORLD @@ -586,6 +586,45 @@ async def handler(context: HttpCrawlingContext) -> None: } +async def test_save_response_cookies_disabled(server_url: URL) -> None: + """Test that `save_response_cookies=False` keeps the response cookies out of the session, but sends its cookies.""" + preset_cookie = CookieParam(name='preset', value='1', domain=server_url.host or '') + crawler = HttpCrawler( + session_pool=SessionPool(max_pool_size=1, create_session_settings={'cookies': [preset_cookie]}), + save_response_cookies=False, + ) + handler_call = Mock() + + @crawler.router.default_handler + async def handler(context: HttpCrawlingContext) -> None: + session_cookies = context.session.cookies.get_cookies_as_dicts() if context.session else [] + handler_call( + sent_cookies=json.loads(await context.http_response.read())['cookies'], + session_cookies={cookie['name'] for cookie in session_cookies}, + ) + + # The cookie set by the redirecting response is still sent to the redirect target. + await crawler.run([str(server_url.with_path('set_cookies').extend_query(a=1))]) + + handler_call.assert_called_once_with(sent_cookies={'preset': '1', 'a': '1'}, session_cookies={'preset'}) + + +async def test_context_send_request_saves_cookies(server_url: URL) -> None: + """Test that `context.send_request` stores response cookies in the session with `save_response_cookies=False`.""" + crawler = HttpCrawler(session_pool=SessionPool(max_pool_size=1), save_response_cookies=False) + handler_call = Mock() + + @crawler.router.default_handler + async def handler(context: HttpCrawlingContext) -> None: + await context.send_request(str(server_url.with_path('set_cookies').extend_query(b=2))) + session_cookies = context.session.cookies.get_cookies_as_dicts() if context.session else [] + handler_call(session_cookies={cookie['name'] for cookie in session_cookies}) + + await crawler.run([str(server_url / 'cookies')]) + + handler_call.assert_called_once_with(session_cookies={'b'}) + + async def test_get_snapshot(server_url: URL) -> None: crawler = HttpCrawler() diff --git a/tests/integration/crawlers/_playwright/test_playwright_crawler.py b/tests/integration/crawlers/_playwright/test_playwright_crawler.py index 9979cd67fa..b98a4c2728 100644 --- a/tests/integration/crawlers/_playwright/test_playwright_crawler.py +++ b/tests/integration/crawlers/_playwright/test_playwright_crawler.py @@ -545,6 +545,28 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: assert session_cookies == {'check': 'test'} +async def test_save_response_cookies_disabled(server_url: URL) -> None: + """Test that `save_response_cookies=False` keeps the cookies of the page out of the session.""" + async with SessionPool(max_pool_size=1) as session_pool: + crawler = PlaywrightCrawler(session_pool=session_pool, save_response_cookies=False) + handler_call = Mock() + + @crawler.router.default_handler + async def request_handler(context: PlaywrightCrawlingContext) -> None: + await context.page.context.add_cookies([{'name': 'check', 'value': 'test', 'url': str(server_url)}]) + handler_call( + session_id=context.session.id if context.session else None, + page_cookies={cookie['name'] for cookie in await context.page.context.cookies()}, + ) + + await crawler.run([str(server_url.with_path('set_cookies').extend_query(a=1))]) + + session = await session_pool.get_session() + + handler_call.assert_called_once_with(session_id=session.id, page_cookies={'a', 'check'}) + assert session.cookies.get_cookies_as_dicts() == [] + + async def test_send_request_sends_page_cookies(server_url: URL) -> None: """Test that `context.send_request` sends the cookies set by the navigation and by the page in the handler.""" crawler = PlaywrightCrawler() diff --git a/tests/integration/http_clients/test_http_clients.py b/tests/integration/http_clients/test_http_clients.py index 0bd662c90b..17c1c4847f 100644 --- a/tests/integration/http_clients/test_http_clients.py +++ b/tests/integration/http_clients/test_http_clients.py @@ -30,12 +30,9 @@ @pytest.fixture async def custom_http_client(request: SubRequest) -> AsyncGenerator[HttpClient]: - """Helper fixture to reduce code duplication. + """Enter the client passed by indirect parametrization and leave its context after the test.""" - If clients are not initialized, create their default instances. - Return client in active context, leave the context after the test.""" - - client = request.param if isinstance(request.param, HttpClient) else request.param() + client = request.param async with client as _: yield _ @@ -478,27 +475,6 @@ async def test_every_response_cookie_stored(http_client: HttpClient, server_url: assert {item['name'] for item in session.cookies.get_cookies_as_dicts()} == {'a', 'b', 'c'} -@pytest.mark.parametrize( - 'custom_http_client', - [ - pytest.param(lambda: CurlImpersonateHttpClient(persist_cookies_per_session=False), id='curl'), - pytest.param(lambda: HttpxHttpClient(persist_cookies_per_session=False), id='httpx'), - pytest.param(lambda: ImpitHttpClient(persist_cookies_per_session=False), id='impit'), - ], - indirect=['custom_http_client'], -) -async def test_cookies_not_persisted(custom_http_client: HttpClient, server_url: URL) -> None: - """Test that `persist_cookies_per_session` keeps the session jar untouched.""" - session = Session() - - async with await custom_http_client.send_request( - str((server_url / 'set_cookies').with_query(a='1')), session=session - ): - pass - - assert session.cookies.get_cookies_as_dicts() == [] - - async def test_cookies_argument_replaces_session_cookies(http_client: HttpClient, server_url: URL) -> None: """Test that the cookies passed in `cookies` are sent and stored instead of the cookies of the session.""" session = Session(cookies=[CookieParam(name='from_session', value='1', domain=server_url.host or '')]) @@ -515,58 +491,6 @@ async def test_cookies_argument_replaces_session_cookies(http_client: HttpClient assert {item['name'] for item in session.cookies.get_cookies_as_dicts()} == {'from_session'} -@pytest.mark.parametrize( - 'custom_http_client', - [ - pytest.param(lambda: CurlImpersonateHttpClient(persist_cookies_per_session=False), id='curl'), - pytest.param(lambda: HttpxHttpClient(persist_cookies_per_session=False), id='httpx'), - pytest.param(lambda: ImpitHttpClient(persist_cookies_per_session=False), id='impit'), - ], - indirect=['custom_http_client'], -) -async def test_cookies_argument_stored_when_not_persisted(custom_http_client: HttpClient, server_url: URL) -> None: - """Test that the response cookies are stored in `cookies` even with `persist_cookies_per_session` disabled.""" - cookies = SessionCookies() - - async with await custom_http_client.send_request( - str((server_url / 'set_cookies').with_query(a='1')), cookies=cookies - ): - pass - - assert {item['name'] for item in cookies.get_cookies_as_dicts()} == {'a'} - - -@pytest.mark.parametrize( - 'custom_http_client', - [ - pytest.param(lambda: CurlImpersonateHttpClient(persist_cookies_per_session=False), id='curl'), - pytest.param(lambda: HttpxHttpClient(persist_cookies_per_session=False), id='httpx'), - pytest.param(lambda: ImpitHttpClient(persist_cookies_per_session=False), id='impit'), - ], - indirect=['custom_http_client'], -) -async def test_cookies_sent_when_not_persisted(custom_http_client: HttpClient, server_url: URL) -> None: - """Test that `persist_cookies_per_session` gates storing the response cookies, not sending the session ones.""" - session = Session(cookies=[CookieParam(name='from_jar', value='1', domain=server_url.host or '')]) - - async with await custom_http_client.send_request(str(server_url / 'cookies'), session=session) as response: - assert (await read_json(response))['cookies'] == {'from_jar': '1'} - - -async def test_redirect_cookies_sent_when_not_persisted(server_url: URL, http_client_class: type[HttpClient]) -> None: - """Test that with `persist_cookies_per_session` off the cookies set during a redirect chain reach its next hops.""" - session = Session() - - async with ( - http_client_class(persist_cookies_per_session=False) as client, - await client.send_request(str((server_url / 'set_cookies').with_query(a='1')), session=session) as response, - ): - cookies = (await read_json(response))['cookies'] - - assert cookies == {'a': '1'} - assert session.cookies.get_cookies_as_dicts() == [] - - async def test_cookie_header_rebuilt_per_hop(http_client: HttpClient, server_url: URL) -> None: """Test that the `Cookie` header of one hop does not reach a hop whose URL the cookie does not match.""" session = Session( diff --git a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py index dffa6a9a3e..e571334b6e 100644 --- a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py +++ b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py @@ -17,41 +17,6 @@ async def test_accepts_concurrency_system() -> None: assert crawler.concurrency_system is concurrency_system -@pytest.mark.parametrize( - ('rendering_type', 'save_response_cookies', 'expected_cookies'), - [ - pytest.param('static', True, {'a'}, id='static, saved'), - pytest.param('static', False, set(), id='static, not saved'), - pytest.param('client only', True, {'a'}, id='browser, saved'), - pytest.param('client only', False, set(), id='browser, not saved'), - ], -) -async def test_save_response_cookies( - *, rendering_type: RenderingType, save_response_cookies: bool, expected_cookies: set[str], server_url: URL -) -> None: - """Test that `save_response_cookies` controls saving the response cookies to the session in both sub crawlers.""" - predictor = _SimpleRenderingTypePredictor( - rendering_types=cycle([rendering_type]), detection_probability_recommendation=cycle([0]) - ) - - async with SessionPool(max_pool_size=1) as session_pool: - crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser( - rendering_type_predictor=predictor, - save_response_cookies=save_response_cookies, - session_pool=session_pool, - ) - - @crawler.router.default_handler - async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: - pass - - await crawler.run([str(server_url.with_path('set_cookies').extend_query(a=1))]) - - session = await session_pool.get_session() - - assert {cookie['name'] for cookie in session.cookies.get_cookies_as_dicts()} == expected_cookies - - @pytest.mark.parametrize( 'optional_module_name', [