From 8f20695fab3fce2c3470338b22932f388566f4e1 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Mon, 2 Nov 2026 15:36:04 +0000 Subject: [PATCH 1/2] add a `cookies` argument to `send_request` Co-authored-by: Vlada Dusek --- docs/upgrading/upgrading_to_v2.md | 4 +- src/crawlee/_types.py | 6 +- src/crawlee/crawlers/_basic/_basic_crawler.py | 4 +- .../_playwright/_playwright_http_client.py | 58 ++++++++++++------- src/crawlee/http_clients/_base.py | 18 +++--- src/crawlee/http_clients/_httpx.py | 4 +- .../crawlers/_basic/test_basic_crawler.py | 5 +- .../_playwright/test_playwright_crawler.py | 20 ++++++- .../test_playwright_http_client.py | 54 +++++++++++++++++ tests/unit/http_clients/test_http_clients.py | 39 ++++++++++++- 10 files changed, 176 insertions(+), 36 deletions(-) diff --git a/docs/upgrading/upgrading_to_v2.md b/docs/upgrading/upgrading_to_v2.md index 12ee4fe7c8..57af046eeb 100644 --- a/docs/upgrading/upgrading_to_v2.md +++ b/docs/upgrading/upgrading_to_v2.md @@ -387,13 +387,15 @@ The `headers` argument of a client still applies to every request. Its `Authoriz A `Cookie` header passed with a request or set on the client is merged with the session cookies while the redirect chain stays on its origin. A cookie in the header wins over a session cookie of the same name and isn't stored in the session. In v1, `ImpitHttpClient`, `HttpxHttpClient` and `context.send_request` of `PlaywrightCrawler` sent the header instead of the session cookies, so an empty `Cookie` header kept them out. `CurlImpersonateHttpClient` sent the session cookies in a second `Cookie` header. +`send_request` and `context.send_request` take a new `cookies` argument, the cookies to send and store instead of the session cookies for a single call. To send no session cookies, pass `cookies=SessionCookies()`. With `PlaywrightCrawler`, the response cookies still reach the session through the browser context. + Without a session, or with `persist_cookies_per_session=False`, the cookies a server sets during a redirect chain are sent with the following requests of that chain. In v1, `ImpitHttpClient` and `HttpxHttpClient` didn't send them. ### Some client arguments are ignored The following keyword arguments of the underlying libraries are now ignored with a warning: -- `cookies` of every client and `cookie_jar` of `ImpitHttpClient`, since cookies are kept in sessions. +- `cookies` of every client and `cookie_jar` of `ImpitHttpClient`, since cookies are kept in sessions. To send other cookies with a single call, pass `cookies` to `send_request`. - `auth` and `params` of `HttpxHttpClient` and `CurlImpersonateHttpClient`, which would reach every request of a redirect chain. Pass the credentials in the `Authorization` header and the query in the URL instead. - `max_redirects` of every client and `allow_redirects` of `CurlImpersonateHttpClient`, since `send_request` follows the redirects. diff --git a/src/crawlee/_types.py b/src/crawlee/_types.py index 5b40157cc0..541e246f6f 100644 --- a/src/crawlee/_types.py +++ b/src/crawlee/_types.py @@ -22,7 +22,7 @@ from crawlee.configuration import Configuration from crawlee.http_clients import HttpResponse from crawlee.proxy_configuration import ProxyInfo - from crawlee.sessions import Session + from crawlee.sessions import Session, SessionCookies from crawlee.storage_backends import StorageBackend from crawlee.storages import KeyValueStore @@ -581,6 +581,7 @@ def __call__( method: HttpMethod = 'GET', payload: HttpPayload | None = None, headers: HttpHeaders | dict[str, str] | None = None, + cookies: SessionCookies | None = None, timeout: timedelta | None = None, ) -> Coroutine[None, None, HttpResponse]: """Call send request function. @@ -590,6 +591,9 @@ def __call__( method: The HTTP method to use. headers: The headers to include in the request. payload: The payload to include in the request. + cookies: The cookies to send and store instead of the cookies of the session. With `PlaywrightCrawler`, + the response cookies are stored in the browser context as well, and so in the session after the + request handler. timeout: Maximum time allowed to receive the response, including all redirects, and to read its body. Without it, the default timeout of the HTTP client applies. diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index 2f4ff4b8ec..f41ed1702b 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -101,7 +101,7 @@ from crawlee.http_clients import HttpClient, HttpResponse from crawlee.proxy_configuration import ProxyConfiguration, ProxyInfo from crawlee.request_loaders import RequestManager - from crawlee.sessions import Session + from crawlee.sessions import Session, SessionCookies from crawlee.statistics import FinalStatistics from crawlee.storage_backends import StorageBackend from crawlee.storage_backends.models import DatasetItemsListPage @@ -1390,6 +1390,7 @@ async def send_request( method: HttpMethod = 'GET', payload: HttpPayload | None = None, headers: HttpHeaders | dict[str, str] | None = None, + cookies: SessionCookies | None = None, timeout: timedelta | None = None, ) -> HttpResponse: response = await self._http_client.send_request( @@ -1398,6 +1399,7 @@ async def send_request( payload=payload, headers=headers, session=session, + cookies=cookies, proxy_info=proxy_info, timeout=timeout, ) diff --git a/src/crawlee/crawlers/_playwright/_playwright_http_client.py b/src/crawlee/crawlers/_playwright/_playwright_http_client.py index be8bef2596..0e3d0d6c7f 100644 --- a/src/crawlee/crawlers/_playwright/_playwright_http_client.py +++ b/src/crawlee/crawlers/_playwright/_playwright_http_client.py @@ -8,8 +8,7 @@ from crawlee.crawlers._playwright._types import PlaywrightHttpResponse from crawlee.http_clients import HttpClient -from crawlee.sessions import Session -from crawlee.sessions._cookies import PlaywrightCookieParam +from crawlee.sessions._cookies import PlaywrightCookieParam, SessionCookies if TYPE_CHECKING: from collections.abc import AsyncGenerator @@ -20,10 +19,14 @@ from crawlee._types import HttpHeaders, HttpMethod, HttpPayload from crawlee.http_clients import HttpResponse from crawlee.proxy_configuration import ProxyInfo + from crawlee.sessions import Session _browser_page_context_var: contextvars.ContextVar[Page | None] = contextvars.ContextVar('browser_context', default=None) +# Set while `send_request` sends the cookies passed by the caller instead of the cookies of the browser context. +_caller_cookies_var: contextvars.ContextVar[bool] = contextvars.ContextVar('caller_cookies', default=False) + @asynccontextmanager async def browser_page_context(page: Page) -> AsyncGenerator[None, None]: @@ -55,6 +58,7 @@ async def send_request( headers: HttpHeaders | dict[str, str] | None = None, payload: HttpPayload | None = None, session: Session | None = None, + cookies: SessionCookies | None = None, proxy_info: ProxyInfo | None = None, timeout: timedelta | None = None, ) -> HttpResponse: @@ -62,26 +66,31 @@ async def send_request( # https://github.com/apify/crawlee-python/issues/1055 page = _browser_page_context_var.get() - - # The browser context holds the cookies of the session, added before the navigation, with the changes the page - # made since. The browser stores the response cookies there too, and the crawler saves them to the session after - # the request handler. - if page is not None: - context_cookies = await page.context.cookies() - session = Session() - session.cookies.set_cookies_from_playwright_format( - [PlaywrightCookieParam(**cookie) for cookie in context_cookies] + token = _caller_cookies_var.set(cookies is not None) + + try: + # The browser context holds the cookies of the session, added before the navigation, with the changes the + # page made since. The browser stores the response cookies there too, and the crawler saves them to the + # session after the request handler. + if page is not None and cookies is None: + context_cookies = await page.context.cookies() + cookies = SessionCookies() + cookies.set_cookies_from_playwright_format( + [PlaywrightCookieParam(**cookie) for cookie in context_cookies] + ) + + return await super().send_request( + url, + method=method, + headers=headers, + payload=payload, + session=session, + cookies=cookies, + proxy_info=proxy_info, + timeout=timeout, ) - - return await super().send_request( - url, - method=method, - headers=headers, - payload=payload, - session=session, - proxy_info=proxy_info, - timeout=timeout, - ) + finally: + _caller_cookies_var.reset(token) @override async def _fetch( @@ -100,11 +109,16 @@ async def _fetch( if browser_context is None: raise RuntimeError('Unable to create an `APIRequestContext` outside the browser context') + fetch_headers = dict(headers) + if _caller_cookies_var.get(): + # Without a `Cookie` header, Playwright sends the cookies of the browser context. + fetch_headers.setdefault('cookie', '') + # Proxies appropriate to the browser context are used response = await browser_context.request.fetch( url_or_request=url, method=method.lower(), - headers=dict(headers) or None, + headers=fetch_headers or None, data=payload, timeout=timeout.total_seconds() * 1000 if timeout is not None else None, # Redirects are followed hop by hop by `send_request`. diff --git a/src/crawlee/http_clients/_base.py b/src/crawlee/http_clients/_base.py index f7b2bab622..e4e84a2232 100644 --- a/src/crawlee/http_clients/_base.py +++ b/src/crawlee/http_clients/_base.py @@ -226,6 +226,7 @@ async def send_request( headers: HttpHeaders | dict[str, str] | None = None, payload: HttpPayload | None = None, session: Session | None = None, + cookies: SessionCookies | None = None, proxy_info: ProxyInfo | None = None, timeout: timedelta | None = None, ) -> HttpResponse: @@ -246,6 +247,8 @@ async def send_request( session: The session whose cookies are sent. The response cookies are stored in it when `persist_cookies_per_session` is enabled. Otherwise, and without a session, they are kept only for the redirect chain of this request. + cookies: The cookies to send and store instead of the cookies of `session`, even with + `persist_cookies_per_session` disabled. proxy_info: The information about the proxy to be used. timeout: Maximum time allowed to receive the response, including all redirects, and to read its body. Without it, the default timeout of the HTTP client applies. @@ -263,13 +266,14 @@ async def send_request( # The headers of the client go through the redirect rules below too, so its credentials never leave the origin. headers = self._headers | HttpHeaders(headers or {}) - if session is None: - cookies = SessionCookies() - elif self._persist_cookies_per_session: - cookies = session.cookies - else: - # The cookies of the session are still sent, but the response cookies go to a copy. - cookies = SessionCookies(session.cookies) + if cookies is None: + if session is None: + cookies = SessionCookies() + elif self._persist_cookies_per_session: + cookies = session.cookies + else: + # The cookies of the session are still sent, but the response cookies go to a copy. + cookies = SessionCookies(session.cookies) # `encoded=True` keeps the path and the query byte for byte, including a safe `%2F` in query. current_url = _with_ascii_host(URL(url, encoded=True)) diff --git a/src/crawlee/http_clients/_httpx.py b/src/crawlee/http_clients/_httpx.py index b7d305333b..2f9db5d752 100644 --- a/src/crawlee/http_clients/_httpx.py +++ b/src/crawlee/http_clients/_httpx.py @@ -26,7 +26,7 @@ from crawlee._types import HttpMethod, HttpPayload from crawlee.proxy_configuration import ProxyInfo - from crawlee.sessions import Session + from crawlee.sessions import Session, SessionCookies logger = getLogger(__name__) @@ -182,6 +182,7 @@ async def send_request( headers: HttpHeaders | dict[str, str] | None = None, payload: HttpPayload | None = None, session: Session | None = None, + cookies: SessionCookies | None = None, proxy_info: ProxyInfo | None = None, timeout: timedelta | None = None, ) -> HttpResponse: @@ -196,6 +197,7 @@ async def send_request( headers=self._combine_headers(headers), payload=payload, session=session, + cookies=cookies, proxy_info=proxy_info, timeout=timeout, ) diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index 7149415976..9bba8d1ae0 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -49,7 +49,7 @@ StalledStatus, ThrottlingRequestManager, ) -from crawlee.sessions import Session, SessionPool +from crawlee.sessions import Session, SessionCookies, SessionPool from crawlee.statistics import FinalStatistics, Statistics, StatisticsState from crawlee.storage_backends import FileSystemStorageBackend, MemoryStorageBackend from crawlee.storages import Dataset, KeyValueStore, RequestQueue @@ -652,6 +652,7 @@ async def test_send_request_forwards_timeout_to_http_client() -> None: http_client.send_request.return_value = response crawler = BasicCrawler(http_client=http_client, use_session_pool=False, max_requests_per_crawl=1) timeout = timedelta(seconds=12) + cookies = SessionCookies({'name': 'value'}) result = None @crawler.router.default_handler @@ -662,6 +663,7 @@ async def handler(context: BasicCrawlingContext) -> None: method='POST', payload=b'Hello, world!', headers={'Content-Type': 'text/plain'}, + cookies=cookies, timeout=timeout, ) @@ -674,6 +676,7 @@ async def handler(context: BasicCrawlingContext) -> None: payload=b'Hello, world!', headers={'Content-Type': 'text/plain'}, session=None, + cookies=cookies, proxy_info=None, timeout=timeout, ) diff --git a/tests/unit/crawlers/_playwright/test_playwright_crawler.py b/tests/unit/crawlers/_playwright/test_playwright_crawler.py index b9ebf1fbc5..5c5fd84ad9 100644 --- a/tests/unit/crawlers/_playwright/test_playwright_crawler.py +++ b/tests/unit/crawlers/_playwright/test_playwright_crawler.py @@ -39,7 +39,7 @@ from crawlee.fingerprint_suite._header_generator import fingerprint_browser_type_from_playwright_browser_type from crawlee.http_clients import ImpitHttpClient from crawlee.proxy_configuration import ProxyConfiguration -from crawlee.sessions import Session, SessionPool +from crawlee.sessions import CookieParam, Session, SessionCookies, SessionPool from crawlee.statistics import Statistics from crawlee.statistics._error_snapshotter import ErrorSnapshotter from crawlee.storages import RequestQueue @@ -608,6 +608,24 @@ async def second_handler(context: PlaywrightCrawlingContext) -> None: assert sent_cookies == [{}] +async def test_send_request_cookies_argument_replaces_page_cookies(server_url: URL) -> None: + """Test that `context.send_request` with `cookies` sends none of the cookies of the browser context.""" + crawler = PlaywrightCrawler() + sent_cookies: list[dict[str, str]] = [] + + @crawler.router.default_handler + async def request_handler(context: PlaywrightCrawlingContext) -> None: + cookies = SessionCookies([CookieParam(name='from_argument', value='1', domain=server_url.host or '')]) + matching = await context.send_request(str(server_url / 'cookies'), cookies=cookies) + sent_cookies.append(json.loads(await matching.read())['cookies']) + empty = await context.send_request(str(server_url / 'cookies'), cookies=SessionCookies()) + sent_cookies.append(json.loads(await empty.read())['cookies']) + + await crawler.run([str(server_url.with_path('set_cookies').extend_query(a=1))]) + + assert sent_cookies == [{'from_argument': '1'}, {}] + + async def test_read_write_cookies(server_url: URL) -> None: """Test that cookies are reloaded correctly.""" async with SessionPool(max_pool_size=1) as session_pool: diff --git a/tests/unit/crawlers/_playwright/test_playwright_http_client.py b/tests/unit/crawlers/_playwright/test_playwright_http_client.py index 18472fe1a1..b9a1aa37e3 100644 --- a/tests/unit/crawlers/_playwright/test_playwright_http_client.py +++ b/tests/unit/crawlers/_playwright/test_playwright_http_client.py @@ -9,6 +9,7 @@ from crawlee.crawlers._playwright._playwright_http_client import PlaywrightHttpClient, browser_page_context from crawlee.crawlers._playwright._types import PlaywrightHttpResponse from crawlee.http_clients import HttpResponse +from crawlee.sessions import CookieParam, SessionCookies def mock_page() -> Mock: @@ -71,6 +72,59 @@ async def test_send_request_follows_redirects_hop_by_hop() -> None: assert all(call.kwargs['max_redirects'] == 0 for call in page.request.fetch.await_args_list) +async def test_send_request_sends_cookies_argument_instead_of_context_cookies() -> None: + """Test that the cookies passed in `cookies` are sent instead of the cookies of the browser context.""" + page = mock_page() + cookies = SessionCookies([CookieParam(name='from_argument', value='1', domain='example.com')]) + client = PlaywrightHttpClient() + + with patch.object( + PlaywrightHttpResponse, + 'from_playwright_response', + new=AsyncMock(return_value=mock_response(200, 'https://example.com/')), + ): + async with browser_page_context(page): + await client.send_request('https://example.com/', cookies=cookies) + + page.context.cookies.assert_not_awaited() + assert page.request.fetch.call_args.kwargs['headers'] == {'cookie': 'from_argument=1'} + + +async def test_cookies_argument_keeps_context_cookies_out() -> None: + """Test that the empty `Cookie` header keeping the browser context cookies out is sent only with `cookies`.""" + page = mock_page() + client = PlaywrightHttpClient() + + with patch.object( + PlaywrightHttpResponse, + 'from_playwright_response', + new=AsyncMock(return_value=mock_response(200, 'https://example.com/')), + ): + async with browser_page_context(page): + await client.send_request('https://example.com/', cookies=SessionCookies()) + await client.send_request('https://example.com/') + + assert [call.kwargs['headers'] for call in page.request.fetch.await_args_list] == [{'cookie': ''}, None] + + +async def test_cookies_argument_keeps_context_cookies_out_cross_origin() -> None: + """Test that the browser context adds no cookies of its own after a redirect to another origin.""" + redirect = mock_response(302, 'https://example.com/start', location='https://other.com/final') + final = mock_response(200, 'https://other.com/final') + page = mock_page() + cookies = SessionCookies([CookieParam(name='from_argument', value='1', domain='example.com')]) + client = PlaywrightHttpClient() + + with patch.object(PlaywrightHttpResponse, 'from_playwright_response', new=AsyncMock(side_effect=[redirect, final])): + async with browser_page_context(page): + await client.send_request('https://example.com/start', cookies=cookies) + + assert [call.kwargs['headers'] for call in page.request.fetch.await_args_list] == [ + {'cookie': 'from_argument=1'}, + {'cookie': ''}, + ] + + async def test_send_request_zero_timeout_expires_immediately() -> None: """A `timedelta(0)` timeout must expire immediately rather than being forwarded as "disable timeout".""" page = mock_page() diff --git a/tests/unit/http_clients/test_http_clients.py b/tests/unit/http_clients/test_http_clients.py index 3df37ae724..0d95c77ab0 100644 --- a/tests/unit/http_clients/test_http_clients.py +++ b/tests/unit/http_clients/test_http_clients.py @@ -20,7 +20,7 @@ from crawlee.http_clients._base import _is_cross_origin, _redirect_method from crawlee.http_clients._httpx import _HttpxResponse from crawlee.http_clients._impit import _ImpitResponse -from crawlee.sessions import CookieParam, Session +from crawlee.sessions import CookieParam, Session, SessionCookies from tests.unit.server import generate_file_content from tests.unit.server_endpoints import HELLO_WORLD @@ -614,6 +614,43 @@ async def test_cookies_not_persisted(custom_http_client: HttpClient, server_url: assert session.cookies.get_cookies_as_dicts() == [] +async def test_cookies_argument_replaces_session_cookies(http_client: HttpClient, server_url: URL) -> None: + """Test that the cookies passed in `cookies` are sent and stored instead of the cookies of the session.""" + session = Session(cookies=[CookieParam(name='from_session', value='1', domain=server_url.host or '')]) + cookies = SessionCookies([CookieParam(name='from_argument', value='1', domain=server_url.host or '')]) + + async with await http_client.send_request( + str((server_url / 'set_cookies').with_query(a='1')), + session=session, + cookies=cookies, + ) as response: + assert (await read_json(response))['cookies'] == {'from_argument': '1', 'a': '1'} + + assert {item['name'] for item in cookies.get_cookies_as_dicts()} == {'from_argument', 'a'} + assert {item['name'] for item in session.cookies.get_cookies_as_dicts()} == {'from_session'} + + +@pytest.mark.parametrize( + 'custom_http_client', + [ + pytest.param(lambda: CurlImpersonateHttpClient(persist_cookies_per_session=False), id='curl'), + pytest.param(lambda: HttpxHttpClient(persist_cookies_per_session=False), id='httpx'), + pytest.param(lambda: ImpitHttpClient(persist_cookies_per_session=False), id='impit'), + ], + indirect=['custom_http_client'], +) +async def test_cookies_argument_stored_when_not_persisted(custom_http_client: HttpClient, server_url: URL) -> None: + """Test that the response cookies are stored in `cookies` even with `persist_cookies_per_session` disabled.""" + cookies = SessionCookies() + + async with await custom_http_client.send_request( + str((server_url / 'set_cookies').with_query(a='1')), cookies=cookies + ): + pass + + assert {item['name'] for item in cookies.get_cookies_as_dicts()} == {'a'} + + @pytest.mark.parametrize( 'custom_http_client', [ From feca766b03f107aa25d126d11afbefab6e10bc9a Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 3 Nov 2026 14:35:24 +0100 Subject: [PATCH 2/2] test: rename send_request argument forwarding test --- tests/unit/crawlers/_basic/test_basic_crawler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index 9bba8d1ae0..f1f72d65c4 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -646,7 +646,8 @@ async def handler(context: BasicCrawlingContext) -> None: assert content_type == 'application/json' -async def test_send_request_forwards_timeout_to_http_client() -> None: +async def test_send_request_forwards_arguments_to_http_client() -> None: + """Test that `context.send_request` passes all its arguments to the HTTP client.""" http_client = AsyncMock(spec=HttpClient) response = Mock(spec=HttpResponse) http_client.send_request.return_value = response