From 694f4762fa728d2cba61f889fd37f5a516a73c30 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Fri, 14 Aug 2026 16:39:33 +0000 Subject: [PATCH 1/8] normalize configured domains the same way as crawled hostnames --- .../_throttling_request_manager.py | 34 +++++++++++++-- tests/unit/test_throttling_request_manager.py | 42 +++++++++++++++++++ 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/src/crawlee/request_loaders/_throttling_request_manager.py b/src/crawlee/request_loaders/_throttling_request_manager.py index d48c49ec92..4a62eb1b26 100644 --- a/src/crawlee/request_loaders/_throttling_request_manager.py +++ b/src/crawlee/request_loaders/_throttling_request_manager.py @@ -93,13 +93,17 @@ def __init__( locator, ensuring consistency with the crawler's storage backend. base_delay: Initial delay after the first 429 response from a domain. max_delay: Maximum delay between requests to a rate-limited domain. + + Raises: + ValueError: If an entry of `domains` is not a hostname the URL parser can read. """ self._inner: TRequestManager = inner self._service_locator = service_locator if service_locator is not None else global_service_locator self._base_delay = base_delay self._max_delay = max_delay self._request_manager_opener = request_manager_opener - self._domain_states: dict[str, _DomainState] = {d.lower(): _DomainState(domain=d.lower()) for d in domains if d} + domain_keys = [self._parse_configured_domain(d) for d in domains if d] + self._domain_states: dict[str, _DomainState] = {key: _DomainState(domain=key) for key in domain_keys} self._sub_managers: dict[str, TRequestManager] = {} self._new_work_event = asyncio.Event() """Set whenever a request is added or reclaimed. Lets `fetch_next_request` wake from a throttle @@ -354,9 +358,31 @@ def set_crawl_delay(self, url: str, delay_seconds: int) -> None: logger.debug(f'Set crawl-delay for domain "{state.domain}" to {delay_seconds}s') @staticmethod - def _extract_domain(url: str) -> str: - """Extract the domain (hostname) from a URL.""" - return URL(url).host or '' + def _normalize_domain(hostname: str) -> str: + """Bring a parsed hostname to the form domain keys are stored in, root dot and all casing gone.""" + return hostname.lower().removesuffix('.') + + @classmethod + def _parse_configured_domain(cls, domain: str) -> str: + """Turn one `domains` entry, a bare hostname or a URL, into the key its requests are looked up under.""" + try: + # A bare hostname reaches the parser, and with it IDNA and IPv6 handling, only through a synthetic URL. + host = (URL(domain) if '://' in domain else URL(f'https://{domain}')).host + except ValueError: + host = None + + if not host: + raise ValueError( + f'"{domain}" is not a valid hostname. The `domains` option takes bare hostnames such as ' + f'"example.com"; an IPv6 address has to be bracketed, as in "[::1]".' + ) + + return cls._normalize_domain(host) + + @classmethod + def _extract_domain(cls, url: str) -> str: + """Extract the domain key from a URL.""" + return cls._normalize_domain(URL(url).host or '') @staticmethod def _get_url_from_request(request: str | Request) -> str: diff --git a/tests/unit/test_throttling_request_manager.py b/tests/unit/test_throttling_request_manager.py index 0451297fff..ea39b1750a 100644 --- a/tests/unit/test_throttling_request_manager.py +++ b/tests/unit/test_throttling_request_manager.py @@ -120,6 +120,48 @@ async def test_domain_matching_is_case_insensitive( assert manager._is_domain_throttled('example.com') +@pytest.mark.parametrize( + ('configured', 'url'), + [ + pytest.param('xn--hky-ela4t.cz', 'https://háčky.cz/page', id='punycode_configured'), + pytest.param('háčky.cz', 'https://xn--hky-ela4t.cz/page', id='punycode_url'), + pytest.param('example.com', 'http://example.com./page', id='root_dot_url'), + pytest.param('example.com.', 'http://example.com/page', id='root_dot_configured'), + pytest.param('[::1]', 'http://[::1]:8080/page', id='ipv6_literal'), + pytest.param('https://example.com/products', 'https://example.com/page', id='full_url'), + ], +) +async def test_domain_matching_normalizes_spelling( + configured: str, + url: str, + inner_queue: RequestQueue, + service_locator: ServiceLocator, +) -> None: + """A configured domain and a crawled URL must land on the same key however each of them is spelled.""" + manager = ThrottlingRequestManager( + inner_queue, + domains=[configured], + request_manager_opener=RequestQueue.open, + service_locator=service_locator, + ) + + assert manager.record_domain_delay(url) is True + + +async def test_unreadable_domain_is_rejected( + inner_queue: RequestQueue, + service_locator: ServiceLocator, +) -> None: + """An entry the URL parser cannot read is rejected at construction instead of never matching anything.""" + with pytest.raises(ValueError, match='not a valid hostname'): + ThrottlingRequestManager( + inner_queue, + domains=['::1'], + request_manager_opener=RequestQueue.open, + service_locator=service_locator, + ) + + async def test_add_requests_routes_mixed_domains( manager: ThrottlingRequestManager[RequestQueue], inner_queue: RequestQueue, From c478c50348007bc08bd66aa97b056083e2fa5204 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 20 Aug 2026 12:35:18 +0200 Subject: [PATCH 2/8] fix(throttling-manager): validate a configured domain after normalizing it --- .../request_loaders/_throttling_request_manager.py | 6 ++++-- tests/unit/test_throttling_request_manager.py | 14 +++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/crawlee/request_loaders/_throttling_request_manager.py b/src/crawlee/request_loaders/_throttling_request_manager.py index 4a62eb1b26..19ece91e4a 100644 --- a/src/crawlee/request_loaders/_throttling_request_manager.py +++ b/src/crawlee/request_loaders/_throttling_request_manager.py @@ -371,13 +371,15 @@ def _parse_configured_domain(cls, domain: str) -> str: except ValueError: host = None - if not host: + key = cls._normalize_domain(host) if host else '' + + if not key: raise ValueError( f'"{domain}" is not a valid hostname. The `domains` option takes bare hostnames such as ' f'"example.com"; an IPv6 address has to be bracketed, as in "[::1]".' ) - return cls._normalize_domain(host) + return key @classmethod def _extract_domain(cls, url: str) -> str: diff --git a/tests/unit/test_throttling_request_manager.py b/tests/unit/test_throttling_request_manager.py index ea39b1750a..dbd0d7a1a5 100644 --- a/tests/unit/test_throttling_request_manager.py +++ b/tests/unit/test_throttling_request_manager.py @@ -148,15 +148,23 @@ async def test_domain_matching_normalizes_spelling( assert manager.record_domain_delay(url) is True -async def test_unreadable_domain_is_rejected( +@pytest.mark.parametrize( + 'configured', + [ + pytest.param('::1', id='unbracketed ipv6'), + pytest.param('.', id='bare root dot'), + ], +) +async def test_unmatchable_domain_is_rejected( + configured: str, inner_queue: RequestQueue, service_locator: ServiceLocator, ) -> None: - """An entry the URL parser cannot read is rejected at construction instead of never matching anything.""" + """An entry that cannot yield a matchable hostname is rejected at construction, not silently kept.""" with pytest.raises(ValueError, match='not a valid hostname'): ThrottlingRequestManager( inner_queue, - domains=['::1'], + domains=[configured], request_manager_opener=RequestQueue.open, service_locator=service_locator, ) From 4edf8c966be99b6914c7346e557f41954bd5348d Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 20 Aug 2026 12:35:55 +0200 Subject: [PATCH 3/8] fix(throttling-manager): accept a bare IPv6 literal in configured domains --- .../_throttling_request_manager.py | 18 +++++++++++++++--- tests/unit/test_throttling_request_manager.py | 5 ++++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/crawlee/request_loaders/_throttling_request_manager.py b/src/crawlee/request_loaders/_throttling_request_manager.py index 19ece91e4a..b1e2961e40 100644 --- a/src/crawlee/request_loaders/_throttling_request_manager.py +++ b/src/crawlee/request_loaders/_throttling_request_manager.py @@ -2,6 +2,7 @@ import asyncio import contextlib +import ipaddress from dataclasses import dataclass from datetime import datetime, timedelta, timezone from logging import getLogger @@ -365,9 +366,20 @@ def _normalize_domain(hostname: str) -> str: @classmethod def _parse_configured_domain(cls, domain: str) -> str: """Turn one `domains` entry, a bare hostname or a URL, into the key its requests are looked up under.""" + if '://' in domain: + url_text = domain + else: + # A bare hostname reaches the parser's IDNA handling only through a synthetic URL, and a bare IPv6 + # literal has to be bracketed there, or the parser reads its last group as a port. + try: + ipaddress.IPv6Address(domain) + except ValueError: + url_text = f'https://{domain}' + else: + url_text = f'https://[{domain}]' + try: - # A bare hostname reaches the parser, and with it IDNA and IPv6 handling, only through a synthetic URL. - host = (URL(domain) if '://' in domain else URL(f'https://{domain}')).host + host = URL(url_text).host except ValueError: host = None @@ -376,7 +388,7 @@ def _parse_configured_domain(cls, domain: str) -> str: if not key: raise ValueError( f'"{domain}" is not a valid hostname. The `domains` option takes bare hostnames such as ' - f'"example.com"; an IPv6 address has to be bracketed, as in "[::1]".' + '"example.com", or any URL on the domain.' ) return key diff --git a/tests/unit/test_throttling_request_manager.py b/tests/unit/test_throttling_request_manager.py index dbd0d7a1a5..6971b152ca 100644 --- a/tests/unit/test_throttling_request_manager.py +++ b/tests/unit/test_throttling_request_manager.py @@ -128,7 +128,9 @@ async def test_domain_matching_is_case_insensitive( pytest.param('example.com', 'http://example.com./page', id='root_dot_url'), pytest.param('example.com.', 'http://example.com/page', id='root_dot_configured'), pytest.param('[::1]', 'http://[::1]:8080/page', id='ipv6_literal'), + pytest.param('::1', 'http://[::1]:8080/page', id='bare ipv6'), pytest.param('https://example.com/products', 'https://example.com/page', id='full_url'), + pytest.param('example.com:8080/path:1', 'https://example.com:8080/page', id='scheme-less url with colons'), ], ) async def test_domain_matching_normalizes_spelling( @@ -151,8 +153,9 @@ async def test_domain_matching_normalizes_spelling( @pytest.mark.parametrize( 'configured', [ - pytest.param('::1', id='unbracketed ipv6'), pytest.param('.', id='bare root dot'), + pytest.param('[::1', id='unclosed ipv6 bracket'), + pytest.param('example.com:8080:9090', id='stray colons'), ], ) async def test_unmatchable_domain_is_rejected( From dd938b8b2889f404c61241a8ef83e0b9f48d58cb Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 20 Aug 2026 12:36:21 +0200 Subject: [PATCH 4/8] fix(throttling-manager): ignore whitespace around configured domain entries --- src/crawlee/request_loaders/_throttling_request_manager.py | 3 ++- tests/unit/test_throttling_request_manager.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/crawlee/request_loaders/_throttling_request_manager.py b/src/crawlee/request_loaders/_throttling_request_manager.py index b1e2961e40..415a73dc15 100644 --- a/src/crawlee/request_loaders/_throttling_request_manager.py +++ b/src/crawlee/request_loaders/_throttling_request_manager.py @@ -103,7 +103,8 @@ def __init__( self._base_delay = base_delay self._max_delay = max_delay self._request_manager_opener = request_manager_opener - domain_keys = [self._parse_configured_domain(d) for d in domains if d] + # Padding on an entry would otherwise survive parsing into a key no crawled hostname can match. + domain_keys = [self._parse_configured_domain(entry) for d in domains if (entry := d.strip())] self._domain_states: dict[str, _DomainState] = {key: _DomainState(domain=key) for key in domain_keys} self._sub_managers: dict[str, TRequestManager] = {} self._new_work_event = asyncio.Event() diff --git a/tests/unit/test_throttling_request_manager.py b/tests/unit/test_throttling_request_manager.py index 6971b152ca..fd9b44c5f8 100644 --- a/tests/unit/test_throttling_request_manager.py +++ b/tests/unit/test_throttling_request_manager.py @@ -131,6 +131,7 @@ async def test_domain_matching_is_case_insensitive( pytest.param('::1', 'http://[::1]:8080/page', id='bare ipv6'), pytest.param('https://example.com/products', 'https://example.com/page', id='full_url'), pytest.param('example.com:8080/path:1', 'https://example.com:8080/page', id='scheme-less url with colons'), + pytest.param(' example.com ', 'https://example.com/page', id='padded entry'), ], ) async def test_domain_matching_normalizes_spelling( From d6e39fe05dad9bb3337126be701b92f12aa7f352 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 20 Aug 2026 12:36:46 +0200 Subject: [PATCH 5/8] fix(throttling-manager): reject subdomain wildcards in configured domains --- src/crawlee/request_loaders/_throttling_request_manager.py | 3 ++- tests/unit/test_throttling_request_manager.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/crawlee/request_loaders/_throttling_request_manager.py b/src/crawlee/request_loaders/_throttling_request_manager.py index 415a73dc15..bf5b4faf58 100644 --- a/src/crawlee/request_loaders/_throttling_request_manager.py +++ b/src/crawlee/request_loaders/_throttling_request_manager.py @@ -386,7 +386,8 @@ def _parse_configured_domain(cls, domain: str) -> str: key = cls._normalize_domain(host) if host else '' - if not key: + # A wildcard passes through the parser untouched, so it would become a key no crawled hostname can match. + if not key or '*' in key: raise ValueError( f'"{domain}" is not a valid hostname. The `domains` option takes bare hostnames such as ' '"example.com", or any URL on the domain.' diff --git a/tests/unit/test_throttling_request_manager.py b/tests/unit/test_throttling_request_manager.py index fd9b44c5f8..19a880e883 100644 --- a/tests/unit/test_throttling_request_manager.py +++ b/tests/unit/test_throttling_request_manager.py @@ -156,6 +156,7 @@ async def test_domain_matching_normalizes_spelling( [ pytest.param('.', id='bare root dot'), pytest.param('[::1', id='unclosed ipv6 bracket'), + pytest.param('*.example.com', id='subdomain wildcard'), pytest.param('example.com:8080:9090', id='stray colons'), ], ) From 5a66a839250b146bf446264b22c0bc8a2e456937 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 20 Aug 2026 12:37:37 +0200 Subject: [PATCH 6/8] docs(throttling-manager): clarify how configured domains are matched and normalized --- .../request_loaders/_throttling_request_manager.py | 12 +++++++----- tests/unit/test_throttling_request_manager.py | 12 ++++++------ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/crawlee/request_loaders/_throttling_request_manager.py b/src/crawlee/request_loaders/_throttling_request_manager.py index bf5b4faf58..1957c34973 100644 --- a/src/crawlee/request_loaders/_throttling_request_manager.py +++ b/src/crawlee/request_loaders/_throttling_request_manager.py @@ -84,9 +84,11 @@ def __init__( Args: inner: The underlying request manager to wrap (typically a `RequestQueue`). Requests for non-throttled domains are stored here. - domains: Explicit list of domain hostnames to throttle. Only requests matching these domains will be routed - to per-domain sub-managers. Matching is case-insensitive (hostnames are lowercased) and exact: subdomain - wildcards such as `*.example.com` are not supported — list each subdomain explicitly if needed. + domains: Domains to throttle, each given as a bare hostname such as `api.example.com`, or as any URL on + the domain, of which only the hostname is used. Only requests matching these domains will be routed + to per-domain sub-managers. Matching is exact but spelling-insensitive: casing, punycode versus + Unicode, and a trailing root dot are all normalized away. Subdomain wildcards such as + `*.example.com` are not supported — list each subdomain explicitly if needed. request_manager_opener: Async callable used to create per-domain sub-managers at insertion time. Must accept `alias`, `storage_client`, and `configuration` keyword arguments and return the same concrete subclass as `inner` (e.g. `RequestQueue.open` when `inner` is a `RequestQueue`). @@ -96,7 +98,7 @@ def __init__( max_delay: Maximum delay between requests to a rate-limited domain. Raises: - ValueError: If an entry of `domains` is not a hostname the URL parser can read. + ValueError: If an entry of `domains` does not yield a hostname a crawled URL could match. """ self._inner: TRequestManager = inner self._service_locator = service_locator if service_locator is not None else global_service_locator @@ -361,7 +363,7 @@ def set_crawl_delay(self, url: str, delay_seconds: int) -> None: @staticmethod def _normalize_domain(hostname: str) -> str: - """Bring a parsed hostname to the form domain keys are stored in, root dot and all casing gone.""" + """Reduce a parsed hostname to the form domain keys are stored in: lowercase, without the root dot.""" return hostname.lower().removesuffix('.') @classmethod diff --git a/tests/unit/test_throttling_request_manager.py b/tests/unit/test_throttling_request_manager.py index 19a880e883..1cde170457 100644 --- a/tests/unit/test_throttling_request_manager.py +++ b/tests/unit/test_throttling_request_manager.py @@ -123,13 +123,13 @@ async def test_domain_matching_is_case_insensitive( @pytest.mark.parametrize( ('configured', 'url'), [ - pytest.param('xn--hky-ela4t.cz', 'https://háčky.cz/page', id='punycode_configured'), - pytest.param('háčky.cz', 'https://xn--hky-ela4t.cz/page', id='punycode_url'), - pytest.param('example.com', 'http://example.com./page', id='root_dot_url'), - pytest.param('example.com.', 'http://example.com/page', id='root_dot_configured'), - pytest.param('[::1]', 'http://[::1]:8080/page', id='ipv6_literal'), + pytest.param('xn--hky-ela4t.cz', 'https://háčky.cz/page', id='punycode entry'), + pytest.param('háčky.cz', 'https://xn--hky-ela4t.cz/page', id='punycode url'), + pytest.param('example.com', 'http://example.com./page', id='root dot in url'), + pytest.param('example.com.', 'http://example.com/page', id='root dot in entry'), + pytest.param('[::1]', 'http://[::1]:8080/page', id='bracketed ipv6'), pytest.param('::1', 'http://[::1]:8080/page', id='bare ipv6'), - pytest.param('https://example.com/products', 'https://example.com/page', id='full_url'), + pytest.param('https://example.com/products', 'https://example.com/page', id='full url'), pytest.param('example.com:8080/path:1', 'https://example.com:8080/page', id='scheme-less url with colons'), pytest.param(' example.com ', 'https://example.com/page', id='padded entry'), ], From 28772a2df737ce0186023382ee7bf726b2ab012f Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 20 Aug 2026 12:38:15 +0200 Subject: [PATCH 7/8] fix(basic-crawler): strip the root dot from the domain named in the 429 warning --- src/crawlee/crawlers/_basic/_basic_crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index 96ff205350..c78aaeeb38 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -1645,7 +1645,7 @@ def _raise_for_session_blocked_status_code( if isinstance(self._request_manager, ThrottlingRequestManager): retry_after = parse_retry_after_header(retry_after_header) if not self._request_manager.record_domain_delay(request_url, retry_after=retry_after): - domain = (URL(request_url).host or '').lower() + domain = (URL(request_url).host or '').lower().removesuffix('.') if domain: self._logger_once.log( f'Received an HTTP 429 (Too Many Requests) response from domain "{domain}", but it is ' From 2af3adaadaca669b4c9f9b798f0b40e1936dbe59 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 20 Aug 2026 13:58:41 +0200 Subject: [PATCH 8/8] docs(throttling-manager): document that blank domain entries are ignored --- .../_throttling_request_manager.py | 11 ++++++----- tests/unit/test_throttling_request_manager.py | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/crawlee/request_loaders/_throttling_request_manager.py b/src/crawlee/request_loaders/_throttling_request_manager.py index 1957c34973..e2c2e74cf7 100644 --- a/src/crawlee/request_loaders/_throttling_request_manager.py +++ b/src/crawlee/request_loaders/_throttling_request_manager.py @@ -85,10 +85,11 @@ def __init__( inner: The underlying request manager to wrap (typically a `RequestQueue`). Requests for non-throttled domains are stored here. domains: Domains to throttle, each given as a bare hostname such as `api.example.com`, or as any URL on - the domain, of which only the hostname is used. Only requests matching these domains will be routed - to per-domain sub-managers. Matching is exact but spelling-insensitive: casing, punycode versus - Unicode, and a trailing root dot are all normalized away. Subdomain wildcards such as - `*.example.com` are not supported — list each subdomain explicitly if needed. + the domain, of which only the hostname is used. Blank entries are ignored, so a list built by + splitting a string needs no pruning. Only requests matching these domains will be routed to + per-domain sub-managers. Matching is exact but spelling-insensitive: casing, punycode versus Unicode, + and a trailing root dot are all normalized away. Subdomain wildcards such as `*.example.com` are not + supported — list each subdomain explicitly if needed. request_manager_opener: Async callable used to create per-domain sub-managers at insertion time. Must accept `alias`, `storage_client`, and `configuration` keyword arguments and return the same concrete subclass as `inner` (e.g. `RequestQueue.open` when `inner` is a `RequestQueue`). @@ -98,7 +99,7 @@ def __init__( max_delay: Maximum delay between requests to a rate-limited domain. Raises: - ValueError: If an entry of `domains` does not yield a hostname a crawled URL could match. + ValueError: If a non-blank entry of `domains` does not yield a hostname a crawled URL could match. """ self._inner: TRequestManager = inner self._service_locator = service_locator if service_locator is not None else global_service_locator diff --git a/tests/unit/test_throttling_request_manager.py b/tests/unit/test_throttling_request_manager.py index 1cde170457..49a0b3801f 100644 --- a/tests/unit/test_throttling_request_manager.py +++ b/tests/unit/test_throttling_request_manager.py @@ -175,6 +175,21 @@ async def test_unmatchable_domain_is_rejected( ) +async def test_blank_domain_entries_are_ignored( + inner_queue: RequestQueue, + service_locator: ServiceLocator, +) -> None: + """Blank entries are dropped rather than rejected, so a list built by splitting a string needs no pruning.""" + manager = ThrottlingRequestManager( + inner_queue, + domains=['', ' ', 'example.com'], + request_manager_opener=RequestQueue.open, + service_locator=service_locator, + ) + + assert set(manager._domain_states) == {'example.com'} + + async def test_add_requests_routes_mixed_domains( manager: ThrottlingRequestManager[RequestQueue], inner_queue: RequestQueue,