diff --git a/src/crawlee/_types.py b/src/crawlee/_types.py index 7a1e0d7eef..19d31d3256 100644 --- a/src/crawlee/_types.py +++ b/src/crawlee/_types.py @@ -14,6 +14,7 @@ import logging import re from collections.abc import Awaitable, Coroutine, MutableMapping, Sequence + from datetime import timedelta from typing_extensions import NotRequired, Required, Self, Unpack @@ -558,6 +559,7 @@ def __call__( method: HttpMethod = 'GET', payload: HttpPayload | None = None, headers: HttpHeaders | dict[str, str] | None = None, + timeout: timedelta | None = None, ) -> Coroutine[None, None, HttpResponse]: """Call send request function. @@ -566,6 +568,7 @@ def __call__( method: The HTTP method to use. headers: The headers to include in the request. payload: The payload to include in the request. + timeout: Maximum time allowed to process the request. Returns: The HTTP response received from the server. diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index 96ff205350..e082a3e37c 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -1321,6 +1321,7 @@ async def send_request( method: HttpMethod = 'GET', payload: HttpPayload | None = None, headers: HttpHeaders | dict[str, str] | None = None, + timeout: timedelta | None = None, ) -> HttpResponse: return await self._http_client.send_request( url=url, @@ -1329,6 +1330,7 @@ async def send_request( headers=headers, session=session, proxy_info=proxy_info, + timeout=timeout, ) return send_request diff --git a/src/crawlee/crawlers/_playwright/_playwright_http_client.py b/src/crawlee/crawlers/_playwright/_playwright_http_client.py index 83cdf843ef..2fa356cf3a 100644 --- a/src/crawlee/crawlers/_playwright/_playwright_http_client.py +++ b/src/crawlee/crawlers/_playwright/_playwright_http_client.py @@ -96,7 +96,7 @@ async def send_request( method=method.lower(), headers=dict(headers) if headers else None, data=payload, - timeout=timeout.total_seconds() if timeout else None, + timeout=timeout.total_seconds() * 1000 if timeout else None, ) return await PlaywrightHttpResponse.from_playwright_response(response, protocol='') diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index 56ba257e86..8b263ec922 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -28,6 +28,7 @@ from crawlee.crawlers import BasicCrawler from crawlee.errors import RequestCollisionError, SessionError, UserDefinedErrorHandlerError from crawlee.events import Event, EventCrawlerStatusData, LocalEventManager +from crawlee.http_clients import HttpClient from crawlee.request_loaders import RequestList, RequestManagerTandem, ThrottlingRequestManager from crawlee.sessions import Session, SessionPool from crawlee.statistics import FinalStatistics, StatisticsState @@ -523,6 +524,58 @@ async def handler(context: BasicCrawlingContext) -> None: assert content_type == 'application/json' +async def test_send_request_forwards_timeout_to_http_client() -> None: + http_client = AsyncMock(spec=HttpClient) + response = Mock() + http_client.send_request.return_value = response + crawler = BasicCrawler(http_client=http_client, use_session_pool=False, max_requests_per_crawl=1) + timeout = timedelta(seconds=12) + result = None + + @crawler.router.default_handler + async def handler(context: BasicCrawlingContext) -> None: + nonlocal result + result = await context.send_request( + 'https://example.com', + method='POST', + payload=b'Hello, world!', + headers={'Content-Type': 'text/plain'}, + timeout=timeout, + ) + + await crawler.run(['https://a.placeholder.com']) + + assert result is response + http_client.send_request.assert_awaited_once_with( + url='https://example.com', + method='POST', + payload=b'Hello, world!', + headers={'Content-Type': 'text/plain'}, + session=None, + proxy_info=None, + timeout=timeout, + ) + + +async def test_send_request_respects_timeout(server_url: URL) -> None: + request_timed_out = asyncio.Event() + + crawler = BasicCrawler(max_request_retries=3) + + @crawler.router.default_handler + async def handler(context: BasicCrawlingContext) -> None: + with pytest.raises(asyncio.TimeoutError): + await context.send_request( + str(server_url / 'slow') + '?delay=2', + timeout=timedelta(milliseconds=100), + ) + request_timed_out.set() + + await crawler.run(['https://a.placeholder.com']) + + assert request_timed_out.is_set() + + @dataclass class AddRequestsTestInput: start_url: str diff --git a/tests/unit/crawlers/_playwright/test_playwright_http_client.py b/tests/unit/crawlers/_playwright/test_playwright_http_client.py new file mode 100644 index 0000000000..40bfddef6a --- /dev/null +++ b/tests/unit/crawlers/_playwright/test_playwright_http_client.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from datetime import timedelta +from unittest.mock import AsyncMock, Mock, patch + +from crawlee.crawlers._playwright._playwright_http_client import PlaywrightHttpClient, browser_page_context +from crawlee.crawlers._playwright._types import PlaywrightHttpResponse + + +async def test_send_request_converts_timeout_to_milliseconds() -> None: + playwright_response = Mock() + expected_response = Mock() + page = Mock() + page.request.fetch = AsyncMock(return_value=playwright_response) + client = PlaywrightHttpClient() + + with patch.object( + PlaywrightHttpResponse, + 'from_playwright_response', + new=AsyncMock(return_value=expected_response), + ) as from_playwright_response: + async with browser_page_context(page): + result = await client.send_request('https://example.com', timeout=timedelta(seconds=12)) + + assert result is expected_response + page.request.fetch.assert_awaited_once_with( + url_or_request='https://example.com', + method='get', + headers=None, + data=None, + timeout=12_000, + ) + from_playwright_response.assert_awaited_once_with(playwright_response, protocol='')