From fb33a77b3b0d42241282579e5fbce431676147d4 Mon Sep 17 00:00:00 2001 From: Shuhan Si <234333961+2gg-bit@users.noreply.github.com> Date: Tue, 29 Sep 2026 13:18:43 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E6=AD=A3=E7=A1=AE=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=E4=BB=A3=E7=90=86=E9=85=8D=E7=BD=AE=E4=B8=AD=E7=9A=84=20IPv6?= =?UTF-8?q?=20=E5=92=8C=E8=AE=A4=E8=AF=81=E4=B8=BB=E6=9C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crawl4ai/async_configs.py | 10 ++---- crawl4ai/proxy_strategy.py | 10 ++---- tests/unit/test_proxy_hostname_extraction.py | 36 ++++++++++++++++++++ 3 files changed, 42 insertions(+), 14 deletions(-) create mode 100644 tests/unit/test_proxy_hostname_extraction.py diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index e76cae18a..f64c7003f 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -3,6 +3,7 @@ import importlib import os import warnings +from urllib.parse import urlparse import requests from .config import ( DEFAULT_PROVIDER, @@ -681,13 +682,8 @@ def __init__( def _extract_ip_from_server(self) -> Optional[str]: """Extract IP address from server URL.""" try: - # Simple extraction assuming http://ip:port format - if "://" in self.server: - parts = self.server.split("://")[1].split(":") - return parts[0] - else: - parts = self.server.split(":") - return parts[0] + server = self.server if "://" in self.server else "//" + self.server + return urlparse(server).hostname except Exception: return None diff --git a/crawl4ai/proxy_strategy.py b/crawl4ai/proxy_strategy.py index 9bd1fe955..ad8153505 100644 --- a/crawl4ai/proxy_strategy.py +++ b/crawl4ai/proxy_strategy.py @@ -1,6 +1,7 @@ from typing import List, Dict, Optional, Tuple from abc import ABC, abstractmethod from itertools import cycle +from urllib.parse import urlparse import os import asyncio import time @@ -35,13 +36,8 @@ def __init__( def _extract_ip_from_server(self) -> Optional[str]: """Extract IP address from server URL.""" try: - # Simple extraction assuming http://ip:port format - if "://" in self.server: - parts = self.server.split("://")[1].split(":") - return parts[0] - else: - parts = self.server.split(":") - return parts[0] + server = self.server if "://" in self.server else "//" + self.server + return urlparse(server).hostname except Exception: return None diff --git a/tests/unit/test_proxy_hostname_extraction.py b/tests/unit/test_proxy_hostname_extraction.py new file mode 100644 index 000000000..7fbb24f08 --- /dev/null +++ b/tests/unit/test_proxy_hostname_extraction.py @@ -0,0 +1,36 @@ +import pytest + +from crawl4ai.async_configs import ProxyConfig +from crawl4ai.proxy_strategy import ProxyConfig as LegacyProxyConfig + + +@pytest.mark.parametrize("config_cls", [ProxyConfig, LegacyProxyConfig]) +@pytest.mark.parametrize( + "server, expected", + [ + ("http://[2001:db8::1]:8080", "2001:db8::1"), + ("[2001:db8::2]:8080", "2001:db8::2"), + ("http://user:password@192.0.2.1:8080", "192.0.2.1"), + ("socks5://user:password@[2001:db8::1]:1080", "2001:db8::1"), + ("https://proxy.example.test/", "proxy.example.test"), + ("http://192.0.2.1:8080", "192.0.2.1"), + ("proxy.example.test:8080", "proxy.example.test"), + ], +) +def test_auto_ip_uses_hostname(config_cls, server, expected): + config = config_cls(server=server) + assert config.ip == expected + assert config.server == server + assert config.to_dict()["ip"] == expected + + +@pytest.mark.parametrize("config_cls", [ProxyConfig, LegacyProxyConfig]) +def test_explicit_verification_ip_is_preserved(config_cls): + assert config_cls(server="http://[::1]:8080", ip="192.0.2.2").ip == "192.0.2.2" + + +def test_url_factory_keeps_ipv6_endpoint_and_auth(): + config = ProxyConfig.from_string("socks5://user:password@[2001:db8::1]:1080") + assert config.ip == "2001:db8::1" + assert config.server == "socks5://[2001:db8::1]:1080" + assert (config.username, config.password) == ("user", "password")