diff --git a/src/fetch/README.md b/src/fetch/README.md index ed6d2262f4..fca1af7d8a 100644 --- a/src/fetch/README.md +++ b/src/fetch/README.md @@ -18,6 +18,7 @@ The fetch tool will truncate the response, but by using the `start_index` argume - `max_length` (integer, optional): Maximum number of characters to return (default: 5000) - `start_index` (integer, optional): Start content from this character index (default: 0) - `raw` (boolean, optional): Get raw content without markdown conversion (default: false) + - `timeout` (number, optional): Timeout in seconds for the fetch request ### Prompts @@ -172,6 +173,13 @@ This can be customized by adding the argument `--user-agent=YourUserAgent` to th The server can be configured to use a proxy by using the `--proxy-url` argument. +### Customization - Timeout + +The request timeout can be configured in multiple ways: +- Via the `--timeout` CLI argument (e.g. `--timeout=60.0`) +- Via the `MCP_FETCH_TIMEOUT` environment variable (in seconds, default: 30.0) +- Per-request via the `timeout` tool argument + ## Windows Configuration If you're experiencing timeout issues on Windows, you may need to set the `PYTHONIOENCODING` environment variable to ensure proper character encoding: diff --git a/src/fetch/src/mcp_server_fetch/__init__.py b/src/fetch/src/mcp_server_fetch/__init__.py index 09744ce319..95176adb78 100644 --- a/src/fetch/src/mcp_server_fetch/__init__.py +++ b/src/fetch/src/mcp_server_fetch/__init__.py @@ -16,9 +16,22 @@ def main(): help="Ignore robots.txt restrictions", ) parser.add_argument("--proxy-url", type=str, help="Proxy URL to use for requests") + parser.add_argument( + "--timeout", + type=float, + default=None, + help="Timeout in seconds for fetch requests (default: 30.0, or MCP_FETCH_TIMEOUT env var)", + ) args = parser.parse_args() - asyncio.run(serve(args.user_agent, args.ignore_robots_txt, args.proxy_url)) + asyncio.run( + serve( + args.user_agent, + args.ignore_robots_txt, + args.proxy_url, + timeout=args.timeout, + ) + ) if __name__ == "__main__": diff --git a/src/fetch/src/mcp_server_fetch/server.py b/src/fetch/src/mcp_server_fetch/server.py index b42c7b1f6b..92fd047d0e 100644 --- a/src/fetch/src/mcp_server_fetch/server.py +++ b/src/fetch/src/mcp_server_fetch/server.py @@ -1,3 +1,4 @@ +import os from typing import Annotated, Tuple from urllib.parse import urlparse, urlunparse @@ -22,6 +23,20 @@ DEFAULT_USER_AGENT_AUTONOMOUS = "ModelContextProtocol/1.0 (Autonomous; +https://github.com/modelcontextprotocol/servers)" DEFAULT_USER_AGENT_MANUAL = "ModelContextProtocol/1.0 (User-Specified; +https://github.com/modelcontextprotocol/servers)" +DEFAULT_TIMEOUT = 30.0 + + +def get_default_timeout() -> float: + """Get the default timeout in seconds from the environment or fallback to DEFAULT_TIMEOUT.""" + timeout_env = os.environ.get("MCP_FETCH_TIMEOUT") + if timeout_env is not None: + try: + val = float(timeout_env) + if val > 0: + return val + except ValueError: + pass + return DEFAULT_TIMEOUT def extract_content_from_html(html: str) -> str: @@ -33,15 +48,30 @@ def extract_content_from_html(html: str) -> str: Returns: Simplified markdown version of the content """ - ret = readabilipy.simple_json.simple_json_from_html_string( - html, use_readability=True - ) - if not ret["content"]: + ret = None + try: + ret = readabilipy.simple_json.simple_json_from_html_string( + html, use_readability=True + ) + except Exception: + pass + + content_raw = ret.get("content") if ret else None + content = None + if content_raw and len(content_raw) >= len(html) * 0.05: + content = markdownify.markdownify( + content_raw, + heading_style=markdownify.ATX, + ) + + if not content or not content.strip(): + content = markdownify.markdownify( + html, + heading_style=markdownify.ATX, + ) + + if not content or not content.strip(): return "Page failed to be simplified from HTML" - content = markdownify.markdownify( - ret["content"], - heading_style=markdownify.ATX, - ) return content @@ -63,13 +93,19 @@ def get_robots_txt_url(url: str) -> str: return robots_url -async def check_may_autonomously_fetch_url(url: str, user_agent: str, proxy_url: str | None = None) -> None: +async def check_may_autonomously_fetch_url( + url: str, + user_agent: str, + proxy_url: str | None = None, + timeout: float | None = None, +) -> None: """ Check if the URL can be fetched by the user agent according to the robots.txt file. Raises a McpError if not. """ from httpx import AsyncClient, HTTPError + actual_timeout = timeout if timeout is not None else get_default_timeout() robot_txt_url = get_robots_txt_url(url) async with AsyncClient(proxy=proxy_url) as client: @@ -78,6 +114,7 @@ async def check_may_autonomously_fetch_url(url: str, user_agent: str, proxy_url: robot_txt_url, follow_redirects=True, headers={"User-Agent": user_agent}, + timeout=actual_timeout, ) except HTTPError: raise McpError(ErrorData( @@ -109,20 +146,26 @@ async def check_may_autonomously_fetch_url(url: str, user_agent: str, proxy_url: async def fetch_url( - url: str, user_agent: str, force_raw: bool = False, proxy_url: str | None = None + url: str, + user_agent: str, + force_raw: bool = False, + proxy_url: str | None = None, + timeout: float | None = None, ) -> Tuple[str, str]: """ Fetch the URL and return the content in a form ready for the LLM, as well as a prefix string with status information. """ from httpx import AsyncClient, HTTPError + actual_timeout = timeout if timeout is not None else get_default_timeout() + async with AsyncClient(proxy=proxy_url) as client: try: response = await client.get( url, follow_redirects=True, headers={"User-Agent": user_agent}, - timeout=30, + timeout=actual_timeout, ) except HTTPError as e: raise McpError(ErrorData(code=INTERNAL_ERROR, message=f"Failed to fetch {url}: {e!r}")) @@ -160,7 +203,7 @@ class Fetch(BaseModel): gt=0, lt=1000000, ), - ] + ] = 5000 start_index: Annotated[ int, Field( @@ -168,20 +211,29 @@ class Fetch(BaseModel): description="On return output starting at this character index, useful if a previous fetch was truncated and more context is required.", ge=0, ), - ] + ] = 0 raw: Annotated[ bool, Field( default=False, description="Get the actual HTML content of the requested page, without simplification.", ), - ] + ] = False + timeout: Annotated[ + float | None, + Field( + default=None, + description="Timeout in seconds for the fetch request.", + gt=0, + ), + ] = None async def serve( custom_user_agent: str | None = None, ignore_robots_txt: bool = False, proxy_url: str | None = None, + timeout: float | None = None, ) -> None: """Run the fetch MCP server. @@ -189,10 +241,12 @@ async def serve( custom_user_agent: Optional custom User-Agent string to use for requests ignore_robots_txt: Whether to ignore robots.txt restrictions proxy_url: Optional proxy URL to use for requests + timeout: Optional default timeout in seconds for requests """ server = Server("mcp-fetch") user_agent_autonomous = custom_user_agent or DEFAULT_USER_AGENT_AUTONOMOUS user_agent_manual = custom_user_agent or DEFAULT_USER_AGENT_MANUAL + server_timeout = timeout if timeout is not None else get_default_timeout() @server.list_tools() async def list_tools() -> list[Tool]: @@ -231,11 +285,19 @@ async def call_tool(name, arguments: dict) -> list[TextContent]: if not url: raise McpError(ErrorData(code=INVALID_PARAMS, message="URL is required")) + request_timeout = args.timeout if args.timeout is not None else server_timeout + if not ignore_robots_txt: - await check_may_autonomously_fetch_url(url, user_agent_autonomous, proxy_url) + await check_may_autonomously_fetch_url( + url, user_agent_autonomous, proxy_url, timeout=request_timeout + ) content, prefix = await fetch_url( - url, user_agent_autonomous, force_raw=args.raw, proxy_url=proxy_url + url, + user_agent_autonomous, + force_raw=args.raw, + proxy_url=proxy_url, + timeout=request_timeout, ) original_length = len(content) if args.start_index >= original_length: @@ -262,7 +324,9 @@ async def get_prompt(name: str, arguments: dict | None) -> GetPromptResult: url = arguments["url"] try: - content, prefix = await fetch_url(url, user_agent_manual, proxy_url=proxy_url) + content, prefix = await fetch_url( + url, user_agent_manual, proxy_url=proxy_url, timeout=server_timeout + ) # TODO: after SDK bug is addressed, don't catch the exception except McpError as e: return GetPromptResult( diff --git a/src/fetch/tests/test_server.py b/src/fetch/tests/test_server.py index 96c1cb38c7..c27bd779b4 100644 --- a/src/fetch/tests/test_server.py +++ b/src/fetch/tests/test_server.py @@ -3,6 +3,7 @@ import pytest from unittest.mock import AsyncMock, patch, MagicMock from mcp.shared.exceptions import McpError +from pydantic import AnyUrl from mcp_server_fetch.server import ( extract_content_from_html, @@ -10,6 +11,9 @@ check_may_autonomously_fetch_url, fetch_url, DEFAULT_USER_AGENT_AUTONOMOUS, + DEFAULT_TIMEOUT, + get_default_timeout, + Fetch, ) @@ -87,6 +91,41 @@ def test_empty_content_returns_error(self): result = extract_content_from_html(html) assert "" in result + def test_ssr_streaming_progressive_fallback(self): + """Test that SSR/streaming skeleton pages fall back to markdownifying the source HTML directly when readability produces too little content.""" + # Simulate an SSR page with hidden elements and progressive skeleton where readability strips hidden elements + large_body_content = "\n".join(f"" for i in range(100)) + html = f""" + + Next.js SSR Progressive App + +
+

Loading...

+ {large_body_content} +
+ + + """ + # Readability will return empty or only the small loading shell (< 5% of source HTML) + result = extract_content_from_html(html) + assert "" not in result + assert "Progressive content chunk 0: important data" in result + assert "Progressive content chunk 99: important data" in result + + def test_readability_returns_none_content_fallback(self): + """Test that if readabilipy returns None content, fallback to markdownify preserves content.""" + html = "

SSR Single Section

Direct section text without article

" + with patch("readabilipy.simple_json.simple_json_from_html_string", return_value={"content": None, "title": "SSR Single Section"}): + result = extract_content_from_html(html) + assert "SSR Single Section" in result + assert "Direct section text without article" in result + + def test_whitespace_only_returns_error(self): + """Test that whitespace-only HTML returns an error.""" + html = " \n\t " + result = extract_content_from_html(html) + assert "Page failed to be simplified from HTML" in result + class TestCheckMayAutonomouslyFetchUrl: """Tests for check_may_autonomously_fetch_url function.""" @@ -183,6 +222,32 @@ async def test_blocks_when_robots_txt_disallows_all(self): DEFAULT_USER_AGENT_AUTONOMOUS ) + @pytest.mark.asyncio + async def test_robots_txt_timeout(self): + """Test that timeout parameter is passed to robots.txt request.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = "User-agent: *\nAllow: /" + + with patch("httpx.AsyncClient") as mock_client_class: + mock_client = AsyncMock() + mock_client.get = AsyncMock(return_value=mock_response) + mock_client_class.return_value.__aenter__ = AsyncMock(return_value=mock_client) + mock_client_class.return_value.__aexit__ = AsyncMock(return_value=None) + + await check_may_autonomously_fetch_url( + "https://example.com/page", + DEFAULT_USER_AGENT_AUTONOMOUS, + timeout=12.0, + ) + + mock_client.get.assert_called_once_with( + "https://example.com/robots.txt", + follow_redirects=True, + headers={"User-Agent": DEFAULT_USER_AGENT_AUTONOMOUS}, + timeout=12.0, + ) + class TestFetchUrl: """Tests for fetch_url function.""" @@ -324,3 +389,157 @@ async def test_fetch_with_proxy(self): # Verify AsyncClient was called with proxy mock_client_class.assert_called_once_with(proxy="http://proxy.example.com:8080") + + @pytest.mark.asyncio + async def test_fetch_default_timeout(self): + """Test that default timeout is passed to client.get.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = "test" + mock_response.headers = {"content-type": "text/plain"} + + with patch("httpx.AsyncClient") as mock_client_class: + mock_client = AsyncMock() + mock_client.get = AsyncMock(return_value=mock_response) + mock_client_class.return_value.__aenter__ = AsyncMock(return_value=mock_client) + mock_client_class.return_value.__aexit__ = AsyncMock(return_value=None) + + await fetch_url( + "https://example.com/data", + DEFAULT_USER_AGENT_AUTONOMOUS, + ) + + mock_client.get.assert_called_once_with( + "https://example.com/data", + follow_redirects=True, + headers={"User-Agent": DEFAULT_USER_AGENT_AUTONOMOUS}, + timeout=30.0, + ) + + @pytest.mark.asyncio + async def test_fetch_custom_timeout_param(self): + """Test that custom timeout parameter is passed to client.get.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = "test" + mock_response.headers = {"content-type": "text/plain"} + + with patch("httpx.AsyncClient") as mock_client_class: + mock_client = AsyncMock() + mock_client.get = AsyncMock(return_value=mock_response) + mock_client_class.return_value.__aenter__ = AsyncMock(return_value=mock_client) + mock_client_class.return_value.__aexit__ = AsyncMock(return_value=None) + + await fetch_url( + "https://example.com/data", + DEFAULT_USER_AGENT_AUTONOMOUS, + timeout=15.5, + ) + + mock_client.get.assert_called_once_with( + "https://example.com/data", + follow_redirects=True, + headers={"User-Agent": DEFAULT_USER_AGENT_AUTONOMOUS}, + timeout=15.5, + ) + + @pytest.mark.asyncio + async def test_fetch_env_var_timeout(self, monkeypatch): + """Test that MCP_FETCH_TIMEOUT environment variable is used.""" + monkeypatch.setenv("MCP_FETCH_TIMEOUT", "45.0") + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = "test" + mock_response.headers = {"content-type": "text/plain"} + + with patch("httpx.AsyncClient") as mock_client_class: + mock_client = AsyncMock() + mock_client.get = AsyncMock(return_value=mock_response) + mock_client_class.return_value.__aenter__ = AsyncMock(return_value=mock_client) + mock_client_class.return_value.__aexit__ = AsyncMock(return_value=None) + + await fetch_url( + "https://example.com/data", + DEFAULT_USER_AGENT_AUTONOMOUS, + ) + + mock_client.get.assert_called_once_with( + "https://example.com/data", + follow_redirects=True, + headers={"User-Agent": DEFAULT_USER_AGENT_AUTONOMOUS}, + timeout=45.0, + ) + + @pytest.mark.asyncio + async def test_fetch_param_overrides_env_timeout(self, monkeypatch): + """Test that explicit timeout parameter overrides MCP_FETCH_TIMEOUT env var.""" + monkeypatch.setenv("MCP_FETCH_TIMEOUT", "60.0") + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = "test" + mock_response.headers = {"content-type": "text/plain"} + + with patch("httpx.AsyncClient") as mock_client_class: + mock_client = AsyncMock() + mock_client.get = AsyncMock(return_value=mock_response) + mock_client_class.return_value.__aenter__ = AsyncMock(return_value=mock_client) + mock_client_class.return_value.__aexit__ = AsyncMock(return_value=None) + + await fetch_url( + "https://example.com/data", + DEFAULT_USER_AGENT_AUTONOMOUS, + timeout=10.0, + ) + + mock_client.get.assert_called_once_with( + "https://example.com/data", + follow_redirects=True, + headers={"User-Agent": DEFAULT_USER_AGENT_AUTONOMOUS}, + timeout=10.0, + ) + + +class TestGetDefaultTimeout: + """Tests for get_default_timeout helper function.""" + + def test_default_without_env(self, monkeypatch): + monkeypatch.delenv("MCP_FETCH_TIMEOUT", raising=False) + assert DEFAULT_TIMEOUT == 30.0 + assert get_default_timeout() == DEFAULT_TIMEOUT + + def test_valid_float_env(self, monkeypatch): + monkeypatch.setenv("MCP_FETCH_TIMEOUT", "15.5") + assert get_default_timeout() == 15.5 + + def test_invalid_env_falls_back(self, monkeypatch): + monkeypatch.setenv("MCP_FETCH_TIMEOUT", "invalid-timeout") + assert get_default_timeout() == 30.0 + + def test_negative_env_falls_back(self, monkeypatch): + monkeypatch.setenv("MCP_FETCH_TIMEOUT", "-5.0") + assert get_default_timeout() == 30.0 + + def test_zero_env_falls_back(self, monkeypatch): + monkeypatch.setenv("MCP_FETCH_TIMEOUT", "0") + assert get_default_timeout() == 30.0 + + +class TestFetchModel: + """Tests for Fetch pydantic model.""" + + def test_fetch_model_defaults(self): + fetch = Fetch(url=AnyUrl("https://example.com")) + assert str(fetch.url) == "https://example.com/" + assert fetch.max_length == 5000 + assert fetch.start_index == 0 + assert fetch.raw is False + assert fetch.timeout is None + + def test_fetch_model_custom_timeout(self): + fetch = Fetch(url=AnyUrl("https://example.com"), timeout=12.5) + assert fetch.timeout == 12.5 + + def test_fetch_model_invalid_timeout(self): + with pytest.raises(Exception): + Fetch(url=AnyUrl("https://example.com"), timeout=-1.0) +