feat(http-clients)!: Add a cookies argument to send_request

docs/upgrading/upgrading_to_v2.md

@@ -387,13 +387,15 @@ The `headers` argument of a client still applies to every request. Its `Authoriz
 
 A `Cookie` header passed with a request or set on the client is merged with the session cookies while the redirect chain stays on its origin. A cookie in the header wins over a session cookie of the same name and isn't stored in the session. In v1, `ImpitHttpClient`, `HttpxHttpClient` and `context.send_request` of `PlaywrightCrawler` sent the header instead of the session cookies, so an empty `Cookie` header kept them out. `CurlImpersonateHttpClient` sent the session cookies in a second `Cookie` header.
 
+`send_request` and `context.send_request` take a new `cookies` argument, the cookies to send and store instead of the session cookies for a single call. To send no session cookies, pass `cookies=SessionCookies()`. With `PlaywrightCrawler`, the response cookies still reach the session through the browser context.
+
 Without a session, or with `persist_cookies_per_session=False`, the cookies a server sets during a redirect chain are sent with the following requests of that chain. In v1, `ImpitHttpClient` and `HttpxHttpClient` didn't send them.
 
 ### Some client arguments are ignored
 
 The following keyword arguments of the underlying libraries are now ignored with a warning:
 
-- `cookies` of every client and `cookie_jar` of `ImpitHttpClient`, since cookies are kept in sessions.
+- `cookies` of every client and `cookie_jar` of `ImpitHttpClient`, since cookies are kept in sessions. To send other cookies with a single call, pass `cookies` to `send_request`.
 - `auth` and `params` of `HttpxHttpClient` and `CurlImpersonateHttpClient`, which would reach every request of a redirect chain. Pass the credentials in the `Authorization` header and the query in the URL instead.
 - `max_redirects` of every client and `allow_redirects` of `CurlImpersonateHttpClient`, since `send_request` follows the redirects.
 

src/crawlee/_types.py

@@ -22,7 +22,7 @@
     from crawlee.configuration import Configuration
     from crawlee.http_clients import HttpResponse
     from crawlee.proxy_configuration import ProxyInfo
-    from crawlee.sessions import Session
+    from crawlee.sessions import Session, SessionCookies
     from crawlee.storage_backends import StorageBackend
     from crawlee.storages import KeyValueStore
 
@@ -581,6 +581,7 @@ def __call__(
         method: HttpMethod = 'GET',
         payload: HttpPayload | None = None,
         headers: HttpHeaders | dict[str, str] | None = None,
+        cookies: SessionCookies | None = None,
         timeout: timedelta | None = None,
     ) -> Coroutine[None, None, HttpResponse]:
         """Call send request function.
@@ -590,6 +591,9 @@ def __call__(
             method: The HTTP method to use.
             headers: The headers to include in the request.
             payload: The payload to include in the request.
+            cookies: The cookies to send and store instead of the cookies of the session. With `PlaywrightCrawler`,
+                the response cookies are stored in the browser context as well, and so in the session after the
+                request handler.
             timeout: Maximum time allowed to receive the response, including all redirects, and to read its body.
                 Without it, the default timeout of the HTTP client applies.
 

src/crawlee/crawlers/_basic/_basic_crawler.py

@@ -101,7 +101,7 @@
     from crawlee.http_clients import HttpClient, HttpResponse
     from crawlee.proxy_configuration import ProxyConfiguration, ProxyInfo
     from crawlee.request_loaders import RequestManager
-    from crawlee.sessions import Session
+    from crawlee.sessions import Session, SessionCookies
     from crawlee.statistics import FinalStatistics
     from crawlee.storage_backends import StorageBackend
     from crawlee.storage_backends.models import DatasetItemsListPage
@@ -1390,6 +1390,7 @@ async def send_request(
             method: HttpMethod = 'GET',
             payload: HttpPayload | None = None,
             headers: HttpHeaders | dict[str, str] | None = None,
+            cookies: SessionCookies | None = None,
             timeout: timedelta | None = None,
         ) -> HttpResponse:
             response = await self._http_client.send_request(
@@ -1398,6 +1399,7 @@ async def send_request(
                 payload=payload,
                 headers=headers,
                 session=session,
+                cookies=cookies,
                 proxy_info=proxy_info,
                 timeout=timeout,
             )

src/crawlee/crawlers/_playwright/_playwright_http_client.py

@@ -8,8 +8,7 @@
 
 from crawlee.crawlers._playwright._types import PlaywrightHttpResponse
 from crawlee.http_clients import HttpClient
-from crawlee.sessions import Session
-from crawlee.sessions._cookies import PlaywrightCookieParam
+from crawlee.sessions._cookies import PlaywrightCookieParam, SessionCookies
 
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator
@@ -20,10 +19,14 @@
     from crawlee._types import HttpHeaders, HttpMethod, HttpPayload
     from crawlee.http_clients import HttpResponse
     from crawlee.proxy_configuration import ProxyInfo
+    from crawlee.sessions import Session
 
 
 _browser_page_context_var: contextvars.ContextVar[Page | None] = contextvars.ContextVar('browser_context', default=None)
 
+# Set while `send_request` sends the cookies passed by the caller instead of the cookies of the browser context.
+_caller_cookies_var: contextvars.ContextVar[bool] = contextvars.ContextVar('caller_cookies', default=False)
+
 
 @asynccontextmanager
 async def browser_page_context(page: Page) -> AsyncGenerator[None, None]:
@@ -55,33 +58,39 @@ async def send_request(
         headers: HttpHeaders | dict[str, str] | None = None,
         payload: HttpPayload | None = None,
         session: Session | None = None,
+        cookies: SessionCookies | None = None,
         proxy_info: ProxyInfo | None = None,
         timeout: timedelta | None = None,
     ) -> HttpResponse:
         # TODO: Use `session` to restore all the fingerprint headers according to the `BrowserContext`, after resolved
         # https://github.com/apify/crawlee-python/issues/1055
 
         page = _browser_page_context_var.get()
-
-        # The browser context holds the cookies of the session, added before the navigation, with the changes the page
-        # made since. The browser stores the response cookies there too, and the crawler saves them to the session after
-        # the request handler.
-        if page is not None:
-            context_cookies = await page.context.cookies()
-            session = Session()
-            session.cookies.set_cookies_from_playwright_format(
-                [PlaywrightCookieParam(**cookie) for cookie in context_cookies]
+        token = _caller_cookies_var.set(cookies is not None)
+
+        try:
+            # The browser context holds the cookies of the session, added before the navigation, with the changes the
+            # page made since. The browser stores the response cookies there too, and the crawler saves them to the
+            # session after the request handler.
+            if page is not None and cookies is None:
+                context_cookies = await page.context.cookies()
+                cookies = SessionCookies()
+                cookies.set_cookies_from_playwright_format(
+                    [PlaywrightCookieParam(**cookie) for cookie in context_cookies]
+                )
+
+            return await super().send_request(
+                url,
+                method=method,
+                headers=headers,
+                payload=payload,
+                session=session,
+                cookies=cookies,
+                proxy_info=proxy_info,
+                timeout=timeout,
             )
-
-        return await super().send_request(
-            url,
-            method=method,
-            headers=headers,
-            payload=payload,
-            session=session,
-            proxy_info=proxy_info,
-            timeout=timeout,
-        )
+        finally:
+            _caller_cookies_var.reset(token)
 
     @override
     async def _fetch(
@@ -100,11 +109,16 @@ async def _fetch(
         if browser_context is None:
             raise RuntimeError('Unable to create an `APIRequestContext` outside the browser context')
 
+        fetch_headers = dict(headers)
+        if _caller_cookies_var.get():
+            # Without a `Cookie` header, Playwright sends the cookies of the browser context.
+            fetch_headers.setdefault('cookie', '')
+
         # Proxies appropriate to the browser context are used
         response = await browser_context.request.fetch(
             url_or_request=url,
             method=method.lower(),
-            headers=dict(headers) or None,
+            headers=fetch_headers or None,
             data=payload,
             timeout=timeout.total_seconds() * 1000 if timeout is not None else None,
             # Redirects are followed hop by hop by `send_request`.

src/crawlee/http_clients/_base.py

@@ -226,6 +226,7 @@ async def send_request(
         headers: HttpHeaders | dict[str, str] | None = None,
         payload: HttpPayload | None = None,
         session: Session | None = None,
+        cookies: SessionCookies | None = None,
         proxy_info: ProxyInfo | None = None,
         timeout: timedelta | None = None,
     ) -> HttpResponse:
@@ -246,6 +247,8 @@ async def send_request(
             session: The session whose cookies are sent. The response cookies are stored in it when
                 `persist_cookies_per_session` is enabled. Otherwise, and without a session, they are kept only for the
                 redirect chain of this request.
+            cookies: The cookies to send and store instead of the cookies of `session`, even with
+                `persist_cookies_per_session` disabled.
             proxy_info: The information about the proxy to be used.
             timeout: Maximum time allowed to receive the response, including all redirects, and to read its body.
                 Without it, the default timeout of the HTTP client applies.
@@ -263,13 +266,14 @@ async def send_request(
         # The headers of the client go through the redirect rules below too, so its credentials never leave the origin.
         headers = self._headers | HttpHeaders(headers or {})
 
-        if session is None:
-            cookies = SessionCookies()
-        elif self._persist_cookies_per_session:
-            cookies = session.cookies
-        else:
-            # The cookies of the session are still sent, but the response cookies go to a copy.
-            cookies = SessionCookies(session.cookies)
+        if cookies is None:
+            if session is None:
+                cookies = SessionCookies()
+            elif self._persist_cookies_per_session:
+                cookies = session.cookies
+            else:
+                # The cookies of the session are still sent, but the response cookies go to a copy.
+                cookies = SessionCookies(session.cookies)
 
         # `encoded=True` keeps the path and the query byte for byte, including a safe `%2F` in query.
         current_url = _with_ascii_host(URL(url, encoded=True))

src/crawlee/http_clients/_httpx.py

@@ -26,7 +26,7 @@
 
     from crawlee._types import HttpMethod, HttpPayload
     from crawlee.proxy_configuration import ProxyInfo
-    from crawlee.sessions import Session
+    from crawlee.sessions import Session, SessionCookies
 
 logger = getLogger(__name__)
 
@@ -182,6 +182,7 @@ async def send_request(
         headers: HttpHeaders | dict[str, str] | None = None,
         payload: HttpPayload | None = None,
         session: Session | None = None,
+        cookies: SessionCookies | None = None,
         proxy_info: ProxyInfo | None = None,
         timeout: timedelta | None = None,
     ) -> HttpResponse:
@@ -196,6 +197,7 @@ async def send_request(
             headers=self._combine_headers(headers),
             payload=payload,
             session=session,
+            cookies=cookies,
             proxy_info=proxy_info,
             timeout=timeout,
         )

tests/unit/crawlers/_basic/test_basic_crawler.py

@@ -49,7 +49,7 @@
     StalledStatus,
     ThrottlingRequestManager,
 )
-from crawlee.sessions import Session, SessionPool
+from crawlee.sessions import Session, SessionCookies, SessionPool
 from crawlee.statistics import FinalStatistics, Statistics, StatisticsState
 from crawlee.storage_backends import FileSystemStorageBackend, MemoryStorageBackend
 from crawlee.storages import Dataset, KeyValueStore, RequestQueue
@@ -646,12 +646,14 @@ async def handler(context: BasicCrawlingContext) -> None:
     assert content_type == 'application/json'
 
 
-async def test_send_request_forwards_timeout_to_http_client() -> None:
+async def test_send_request_forwards_arguments_to_http_client() -> None:
+    """Test that `context.send_request` passes all its arguments to the HTTP client."""
     http_client = AsyncMock(spec=HttpClient)
     response = Mock(spec=HttpResponse)
     http_client.send_request.return_value = response
     crawler = BasicCrawler(http_client=http_client, use_session_pool=False, max_requests_per_crawl=1)
     timeout = timedelta(seconds=12)
+    cookies = SessionCookies({'name': 'value'})
     result = None
 
     @crawler.router.default_handler
@@ -662,6 +664,7 @@ async def handler(context: BasicCrawlingContext) -> None:
             method='POST',
             payload=b'Hello, world!',
             headers={'Content-Type': 'text/plain'},
+            cookies=cookies,
             timeout=timeout,
         )
 
@@ -674,6 +677,7 @@ async def handler(context: BasicCrawlingContext) -> None:
         payload=b'Hello, world!',
         headers={'Content-Type': 'text/plain'},
         session=None,
+        cookies=cookies,
         proxy_info=None,
         timeout=timeout,
     )

tests/unit/crawlers/_playwright/test_playwright_crawler.py

@@ -39,7 +39,7 @@
 from crawlee.fingerprint_suite._header_generator import fingerprint_browser_type_from_playwright_browser_type
 from crawlee.http_clients import ImpitHttpClient
 from crawlee.proxy_configuration import ProxyConfiguration
-from crawlee.sessions import Session, SessionPool
+from crawlee.sessions import CookieParam, Session, SessionCookies, SessionPool
 from crawlee.statistics import Statistics
 from crawlee.statistics._error_snapshotter import ErrorSnapshotter
 from crawlee.storages import RequestQueue
@@ -608,6 +608,24 @@ async def second_handler(context: PlaywrightCrawlingContext) -> None:
     assert sent_cookies == [{}]
 
 
+async def test_send_request_cookies_argument_replaces_page_cookies(server_url: URL) -> None:
+    """Test that `context.send_request` with `cookies` sends none of the cookies of the browser context."""
+    crawler = PlaywrightCrawler()
+    sent_cookies: list[dict[str, str]] = []
+
+    @crawler.router.default_handler
+    async def request_handler(context: PlaywrightCrawlingContext) -> None:
+        cookies = SessionCookies([CookieParam(name='from_argument', value='1', domain=server_url.host or '')])
+        matching = await context.send_request(str(server_url / 'cookies'), cookies=cookies)
+        sent_cookies.append(json.loads(await matching.read())['cookies'])
+        empty = await context.send_request(str(server_url / 'cookies'), cookies=SessionCookies())
+        sent_cookies.append(json.loads(await empty.read())['cookies'])
+
+    await crawler.run([str(server_url.with_path('set_cookies').extend_query(a=1))])
+
+    assert sent_cookies == [{'from_argument': '1'}, {}]
+
+
 async def test_read_write_cookies(server_url: URL) -> None:
     """Test that cookies are reloaded correctly."""
     async with SessionPool(max_pool_size=1) as session_pool:

tests/unit/crawlers/_playwright/test_playwright_http_client.py

@@ -9,6 +9,7 @@
 from crawlee.crawlers._playwright._playwright_http_client import PlaywrightHttpClient, browser_page_context
 from crawlee.crawlers._playwright._types import PlaywrightHttpResponse
 from crawlee.http_clients import HttpResponse
+from crawlee.sessions import CookieParam, SessionCookies
 
 
 def mock_page() -> Mock:
@@ -71,6 +72,59 @@ async def test_send_request_follows_redirects_hop_by_hop() -> None:
     assert all(call.kwargs['max_redirects'] == 0 for call in page.request.fetch.await_args_list)
 
 
+async def test_send_request_sends_cookies_argument_instead_of_context_cookies() -> None:
+    """Test that the cookies passed in `cookies` are sent instead of the cookies of the browser context."""
+    page = mock_page()
+    cookies = SessionCookies([CookieParam(name='from_argument', value='1', domain='example.com')])
+    client = PlaywrightHttpClient()
+
+    with patch.object(
+        PlaywrightHttpResponse,
+        'from_playwright_response',
+        new=AsyncMock(return_value=mock_response(200, 'https://example.com/')),
+    ):
+        async with browser_page_context(page):
+            await client.send_request('https://example.com/', cookies=cookies)
+
+    page.context.cookies.assert_not_awaited()
+    assert page.request.fetch.call_args.kwargs['headers'] == {'cookie': 'from_argument=1'}
+
+
+async def test_cookies_argument_keeps_context_cookies_out() -> None:
+    """Test that the empty `Cookie` header keeping the browser context cookies out is sent only with `cookies`."""
+    page = mock_page()
+    client = PlaywrightHttpClient()
+
+    with patch.object(
+        PlaywrightHttpResponse,
+        'from_playwright_response',
+        new=AsyncMock(return_value=mock_response(200, 'https://example.com/')),
+    ):
+        async with browser_page_context(page):
+            await client.send_request('https://example.com/', cookies=SessionCookies())
+            await client.send_request('https://example.com/')
+
+    assert [call.kwargs['headers'] for call in page.request.fetch.await_args_list] == [{'cookie': ''}, None]
+
+
+async def test_cookies_argument_keeps_context_cookies_out_cross_origin() -> None:
+    """Test that the browser context adds no cookies of its own after a redirect to another origin."""
+    redirect = mock_response(302, 'https://example.com/start', location='https://other.com/final')
+    final = mock_response(200, 'https://other.com/final')
+    page = mock_page()
+    cookies = SessionCookies([CookieParam(name='from_argument', value='1', domain='example.com')])
+    client = PlaywrightHttpClient()
+
+    with patch.object(PlaywrightHttpResponse, 'from_playwright_response', new=AsyncMock(side_effect=[redirect, final])):
+        async with browser_page_context(page):
+            await client.send_request('https://example.com/start', cookies=cookies)
+
+    assert [call.kwargs['headers'] for call in page.request.fetch.await_args_list] == [
+        {'cookie': 'from_argument=1'},
+        {'cookie': ''},
+    ]
+
+
 async def test_send_request_zero_timeout_expires_immediately() -> None:
     """A `timedelta(0)` timeout must expire immediately rather than being forwarded as "disable timeout"."""
     page = mock_page()