feat: support timeouts in the send_request context helper

src/crawlee/_types.py

@@ -14,6 +14,7 @@
     import logging
     import re
     from collections.abc import Awaitable, Coroutine, MutableMapping, Sequence
+    from datetime import timedelta
 
     from typing_extensions import NotRequired, Required, Self, Unpack
 
@@ -558,6 +559,7 @@ def __call__(
         method: HttpMethod = 'GET',
         payload: HttpPayload | None = None,
         headers: HttpHeaders | dict[str, str] | None = None,
+        timeout: timedelta | None = None,
     ) -> Coroutine[None, None, HttpResponse]:
         """Call send request function.
 
@@ -566,6 +568,7 @@ def __call__(
             method: The HTTP method to use.
             headers: The headers to include in the request.
             payload: The payload to include in the request.
+            timeout: Maximum time allowed to process the request.
 
         Returns:
             The HTTP response received from the server.

src/crawlee/crawlers/_basic/_basic_crawler.py

@@ -1321,6 +1321,7 @@ async def send_request(
             method: HttpMethod = 'GET',
             payload: HttpPayload | None = None,
             headers: HttpHeaders | dict[str, str] | None = None,
+            timeout: timedelta | None = None,
         ) -> HttpResponse:
             return await self._http_client.send_request(
                 url=url,
@@ -1329,6 +1330,7 @@ async def send_request(
                 headers=headers,
                 session=session,
                 proxy_info=proxy_info,
+                timeout=timeout,
             )
 
         return send_request

src/crawlee/crawlers/_playwright/_playwright_http_client.py

@@ -96,7 +96,7 @@ async def send_request(
             method=method.lower(),
             headers=dict(headers) if headers else None,
             data=payload,
-            timeout=timeout.total_seconds() if timeout else None,
+            timeout=timeout.total_seconds() * 1000 if timeout else None,
         )
 
         return await PlaywrightHttpResponse.from_playwright_response(response, protocol='')

tests/unit/crawlers/_basic/test_basic_crawler.py

@@ -28,6 +28,7 @@
 from crawlee.crawlers import BasicCrawler
 from crawlee.errors import RequestCollisionError, SessionError, UserDefinedErrorHandlerError
 from crawlee.events import Event, EventCrawlerStatusData, LocalEventManager
+from crawlee.http_clients import HttpClient
 from crawlee.request_loaders import RequestList, RequestManagerTandem, ThrottlingRequestManager
 from crawlee.sessions import Session, SessionPool
 from crawlee.statistics import FinalStatistics, StatisticsState
@@ -523,6 +524,58 @@ async def handler(context: BasicCrawlingContext) -> None:
     assert content_type == 'application/json'
 
 
+async def test_send_request_forwards_timeout_to_http_client() -> None:
+    http_client = AsyncMock(spec=HttpClient)
+    response = Mock()
+    http_client.send_request.return_value = response
+    crawler = BasicCrawler(http_client=http_client, use_session_pool=False, max_requests_per_crawl=1)
+    timeout = timedelta(seconds=12)
+    result = None
+
+    @crawler.router.default_handler
+    async def handler(context: BasicCrawlingContext) -> None:
+        nonlocal result
+        result = await context.send_request(
+            'https://example.com',
+            method='POST',
+            payload=b'Hello, world!',
+            headers={'Content-Type': 'text/plain'},
+            timeout=timeout,
+        )
+
+    await crawler.run(['https://a.placeholder.com'])
+
+    assert result is response
+    http_client.send_request.assert_awaited_once_with(
+        url='https://example.com',
+        method='POST',
+        payload=b'Hello, world!',
+        headers={'Content-Type': 'text/plain'},
+        session=None,
+        proxy_info=None,
+        timeout=timeout,
+    )
+
+
+async def test_send_request_respects_timeout(server_url: URL) -> None:
+    request_timed_out = asyncio.Event()
+
+    crawler = BasicCrawler(max_request_retries=3)
+
+    @crawler.router.default_handler
+    async def handler(context: BasicCrawlingContext) -> None:
+        with pytest.raises(asyncio.TimeoutError):
+            await context.send_request(
+                str(server_url / 'slow') + '?delay=2',
+                timeout=timedelta(milliseconds=100),
+            )
+        request_timed_out.set()
+
+    await crawler.run(['https://a.placeholder.com'])
+
+    assert request_timed_out.is_set()
+
+
 @dataclass
 class AddRequestsTestInput:
     start_url: str

tests/unit/crawlers/_playwright/test_playwright_http_client.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from datetime import timedelta
+from unittest.mock import AsyncMock, Mock, patch
+
+from crawlee.crawlers._playwright._playwright_http_client import PlaywrightHttpClient, browser_page_context
+from crawlee.crawlers._playwright._types import PlaywrightHttpResponse
+
+
+async def test_send_request_converts_timeout_to_milliseconds() -> None:
+    playwright_response = Mock()
+    expected_response = Mock()
+    page = Mock()
+    page.request.fetch = AsyncMock(return_value=playwright_response)
+    client = PlaywrightHttpClient()
+
+    with patch.object(
+        PlaywrightHttpResponse,
+        'from_playwright_response',
+        new=AsyncMock(return_value=expected_response),
+    ) as from_playwright_response:
+        async with browser_page_context(page):
+            result = await client.send_request('https://example.com', timeout=timedelta(seconds=12))
+
+    assert result is expected_response
+    page.request.fetch.assert_awaited_once_with(
+        url_or_request='https://example.com',
+        method='get',
+        headers=None,
+        data=None,
+        timeout=12_000,
+    )
+    from_playwright_response.assert_awaited_once_with(playwright_response, protocol='')