diff --git a/src/fetch/src/mcp_server_fetch/server.py b/src/fetch/src/mcp_server_fetch/server.py index b42c7b1f6b..00c528650e 100644 --- a/src/fetch/src/mcp_server_fetch/server.py +++ b/src/fetch/src/mcp_server_fetch/server.py @@ -78,6 +78,7 @@ async def check_may_autonomously_fetch_url(url: str, user_agent: str, proxy_url: robot_txt_url, follow_redirects=True, headers={"User-Agent": user_agent}, + timeout=30, ) except HTTPError: raise McpError(ErrorData( diff --git a/src/fetch/tests/test_server.py b/src/fetch/tests/test_server.py index 96c1cb38c7..71d765e0a1 100644 --- a/src/fetch/tests/test_server.py +++ b/src/fetch/tests/test_server.py @@ -184,6 +184,31 @@ async def test_blocks_when_robots_txt_disallows_all(self): ) + @pytest.mark.asyncio + async def test_robots_txt_request_uses_timeout(self): + """Test that the robots.txt request is issued with a timeout so a slow + or unresponsive robots.txt cannot hang the autonomous-fetch check.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = "User-agent: *\nAllow: /" + + with patch("httpx.AsyncClient") as mock_client_class: + mock_client = AsyncMock() + mock_client.get = AsyncMock(return_value=mock_response) + mock_client_class.return_value.__aenter__ = AsyncMock(return_value=mock_client) + mock_client_class.return_value.__aexit__ = AsyncMock(return_value=None) + + await check_may_autonomously_fetch_url( + "https://example.com/page", + DEFAULT_USER_AGENT_AUTONOMOUS + ) + + # The robots.txt GET must carry an explicit timeout. + _, kwargs = mock_client.get.call_args + assert "timeout" in kwargs + assert kwargs["timeout"] == 30 + + class TestFetchUrl: """Tests for fetch_url function."""