diff --git a/src/fetch/README.md b/src/fetch/README.md
index ed6d2262f4..fca1af7d8a 100644
--- a/src/fetch/README.md
+++ b/src/fetch/README.md
@@ -18,6 +18,7 @@ The fetch tool will truncate the response, but by using the `start_index` argume
- `max_length` (integer, optional): Maximum number of characters to return (default: 5000)
- `start_index` (integer, optional): Start content from this character index (default: 0)
- `raw` (boolean, optional): Get raw content without markdown conversion (default: false)
+ - `timeout` (number, optional): Timeout in seconds for the fetch request
### Prompts
@@ -172,6 +173,13 @@ This can be customized by adding the argument `--user-agent=YourUserAgent` to th
The server can be configured to use a proxy by using the `--proxy-url` argument.
+### Customization - Timeout
+
+The request timeout can be configured in multiple ways:
+- Via the `--timeout` CLI argument (e.g. `--timeout=60.0`)
+- Via the `MCP_FETCH_TIMEOUT` environment variable (in seconds, default: 30.0)
+- Per-request via the `timeout` tool argument
+
## Windows Configuration
If you're experiencing timeout issues on Windows, you may need to set the `PYTHONIOENCODING` environment variable to ensure proper character encoding:
diff --git a/src/fetch/src/mcp_server_fetch/__init__.py b/src/fetch/src/mcp_server_fetch/__init__.py
index 09744ce319..95176adb78 100644
--- a/src/fetch/src/mcp_server_fetch/__init__.py
+++ b/src/fetch/src/mcp_server_fetch/__init__.py
@@ -16,9 +16,22 @@ def main():
help="Ignore robots.txt restrictions",
)
parser.add_argument("--proxy-url", type=str, help="Proxy URL to use for requests")
+ parser.add_argument(
+ "--timeout",
+ type=float,
+ default=None,
+ help="Timeout in seconds for fetch requests (default: 30.0, or MCP_FETCH_TIMEOUT env var)",
+ )
args = parser.parse_args()
- asyncio.run(serve(args.user_agent, args.ignore_robots_txt, args.proxy_url))
+ asyncio.run(
+ serve(
+ args.user_agent,
+ args.ignore_robots_txt,
+ args.proxy_url,
+ timeout=args.timeout,
+ )
+ )
if __name__ == "__main__":
diff --git a/src/fetch/src/mcp_server_fetch/server.py b/src/fetch/src/mcp_server_fetch/server.py
index b42c7b1f6b..92fd047d0e 100644
--- a/src/fetch/src/mcp_server_fetch/server.py
+++ b/src/fetch/src/mcp_server_fetch/server.py
@@ -1,3 +1,4 @@
+import os
from typing import Annotated, Tuple
from urllib.parse import urlparse, urlunparse
@@ -22,6 +23,20 @@
DEFAULT_USER_AGENT_AUTONOMOUS = "ModelContextProtocol/1.0 (Autonomous; +https://github.com/modelcontextprotocol/servers)"
DEFAULT_USER_AGENT_MANUAL = "ModelContextProtocol/1.0 (User-Specified; +https://github.com/modelcontextprotocol/servers)"
+DEFAULT_TIMEOUT = 30.0
+
+
+def get_default_timeout() -> float:
+ """Get the default timeout in seconds from the environment or fallback to DEFAULT_TIMEOUT."""
+ timeout_env = os.environ.get("MCP_FETCH_TIMEOUT")
+ if timeout_env is not None:
+ try:
+ val = float(timeout_env)
+ if val > 0:
+ return val
+ except ValueError:
+ pass
+ return DEFAULT_TIMEOUT
def extract_content_from_html(html: str) -> str:
@@ -33,15 +48,30 @@ def extract_content_from_html(html: str) -> str:
Returns:
Simplified markdown version of the content
"""
- ret = readabilipy.simple_json.simple_json_from_html_string(
- html, use_readability=True
- )
- if not ret["content"]:
+ ret = None
+ try:
+ ret = readabilipy.simple_json.simple_json_from_html_string(
+ html, use_readability=True
+ )
+ except Exception:
+ pass
+
+ content_raw = ret.get("content") if ret else None
+ content = None
+ if content_raw and len(content_raw) >= len(html) * 0.05:
+ content = markdownify.markdownify(
+ content_raw,
+ heading_style=markdownify.ATX,
+ )
+
+ if not content or not content.strip():
+ content = markdownify.markdownify(
+ html,
+ heading_style=markdownify.ATX,
+ )
+
+ if not content or not content.strip():
return "Page failed to be simplified from HTML"
- content = markdownify.markdownify(
- ret["content"],
- heading_style=markdownify.ATX,
- )
return content
@@ -63,13 +93,19 @@ def get_robots_txt_url(url: str) -> str:
return robots_url
-async def check_may_autonomously_fetch_url(url: str, user_agent: str, proxy_url: str | None = None) -> None:
+async def check_may_autonomously_fetch_url(
+ url: str,
+ user_agent: str,
+ proxy_url: str | None = None,
+ timeout: float | None = None,
+) -> None:
"""
Check if the URL can be fetched by the user agent according to the robots.txt file.
Raises a McpError if not.
"""
from httpx import AsyncClient, HTTPError
+ actual_timeout = timeout if timeout is not None else get_default_timeout()
robot_txt_url = get_robots_txt_url(url)
async with AsyncClient(proxy=proxy_url) as client:
@@ -78,6 +114,7 @@ async def check_may_autonomously_fetch_url(url: str, user_agent: str, proxy_url:
robot_txt_url,
follow_redirects=True,
headers={"User-Agent": user_agent},
+ timeout=actual_timeout,
)
except HTTPError:
raise McpError(ErrorData(
@@ -109,20 +146,26 @@ async def check_may_autonomously_fetch_url(url: str, user_agent: str, proxy_url:
async def fetch_url(
- url: str, user_agent: str, force_raw: bool = False, proxy_url: str | None = None
+ url: str,
+ user_agent: str,
+ force_raw: bool = False,
+ proxy_url: str | None = None,
+ timeout: float | None = None,
) -> Tuple[str, str]:
"""
Fetch the URL and return the content in a form ready for the LLM, as well as a prefix string with status information.
"""
from httpx import AsyncClient, HTTPError
+ actual_timeout = timeout if timeout is not None else get_default_timeout()
+
async with AsyncClient(proxy=proxy_url) as client:
try:
response = await client.get(
url,
follow_redirects=True,
headers={"User-Agent": user_agent},
- timeout=30,
+ timeout=actual_timeout,
)
except HTTPError as e:
raise McpError(ErrorData(code=INTERNAL_ERROR, message=f"Failed to fetch {url}: {e!r}"))
@@ -160,7 +203,7 @@ class Fetch(BaseModel):
gt=0,
lt=1000000,
),
- ]
+ ] = 5000
start_index: Annotated[
int,
Field(
@@ -168,20 +211,29 @@ class Fetch(BaseModel):
description="On return output starting at this character index, useful if a previous fetch was truncated and more context is required.",
ge=0,
),
- ]
+ ] = 0
raw: Annotated[
bool,
Field(
default=False,
description="Get the actual HTML content of the requested page, without simplification.",
),
- ]
+ ] = False
+ timeout: Annotated[
+ float | None,
+ Field(
+ default=None,
+ description="Timeout in seconds for the fetch request.",
+ gt=0,
+ ),
+ ] = None
async def serve(
custom_user_agent: str | None = None,
ignore_robots_txt: bool = False,
proxy_url: str | None = None,
+ timeout: float | None = None,
) -> None:
"""Run the fetch MCP server.
@@ -189,10 +241,12 @@ async def serve(
custom_user_agent: Optional custom User-Agent string to use for requests
ignore_robots_txt: Whether to ignore robots.txt restrictions
proxy_url: Optional proxy URL to use for requests
+ timeout: Optional default timeout in seconds for requests
"""
server = Server("mcp-fetch")
user_agent_autonomous = custom_user_agent or DEFAULT_USER_AGENT_AUTONOMOUS
user_agent_manual = custom_user_agent or DEFAULT_USER_AGENT_MANUAL
+ server_timeout = timeout if timeout is not None else get_default_timeout()
@server.list_tools()
async def list_tools() -> list[Tool]:
@@ -231,11 +285,19 @@ async def call_tool(name, arguments: dict) -> list[TextContent]:
if not url:
raise McpError(ErrorData(code=INVALID_PARAMS, message="URL is required"))
+ request_timeout = args.timeout if args.timeout is not None else server_timeout
+
if not ignore_robots_txt:
- await check_may_autonomously_fetch_url(url, user_agent_autonomous, proxy_url)
+ await check_may_autonomously_fetch_url(
+ url, user_agent_autonomous, proxy_url, timeout=request_timeout
+ )
content, prefix = await fetch_url(
- url, user_agent_autonomous, force_raw=args.raw, proxy_url=proxy_url
+ url,
+ user_agent_autonomous,
+ force_raw=args.raw,
+ proxy_url=proxy_url,
+ timeout=request_timeout,
)
original_length = len(content)
if args.start_index >= original_length:
@@ -262,7 +324,9 @@ async def get_prompt(name: str, arguments: dict | None) -> GetPromptResult:
url = arguments["url"]
try:
- content, prefix = await fetch_url(url, user_agent_manual, proxy_url=proxy_url)
+ content, prefix = await fetch_url(
+ url, user_agent_manual, proxy_url=proxy_url, timeout=server_timeout
+ )
# TODO: after SDK bug is addressed, don't catch the exception
except McpError as e:
return GetPromptResult(
diff --git a/src/fetch/tests/test_server.py b/src/fetch/tests/test_server.py
index 96c1cb38c7..c27bd779b4 100644
--- a/src/fetch/tests/test_server.py
+++ b/src/fetch/tests/test_server.py
@@ -3,6 +3,7 @@
import pytest
from unittest.mock import AsyncMock, patch, MagicMock
from mcp.shared.exceptions import McpError
+from pydantic import AnyUrl
from mcp_server_fetch.server import (
extract_content_from_html,
@@ -10,6 +11,9 @@
check_may_autonomously_fetch_url,
fetch_url,
DEFAULT_USER_AGENT_AUTONOMOUS,
+ DEFAULT_TIMEOUT,
+ get_default_timeout,
+ Fetch,
)
@@ -87,6 +91,41 @@ def test_empty_content_returns_error(self):
result = extract_content_from_html(html)
assert "" in result
+ def test_ssr_streaming_progressive_fallback(self):
+ """Test that SSR/streaming skeleton pages fall back to markdownifying the source HTML directly when readability produces too little content."""
+ # Simulate an SSR page with hidden elements and progressive skeleton where readability strips hidden elements
+ large_body_content = "\n".join(f"
Progressive content chunk {i}: important data
" for i in range(100))
+ html = f"""
+
+ Next.js SSR Progressive App
+
+
+
Loading...
+ {large_body_content}
+
+
+
+ """
+ # Readability will return empty or only the small loading shell (< 5% of source HTML)
+ result = extract_content_from_html(html)
+ assert "" not in result
+ assert "Progressive content chunk 0: important data" in result
+ assert "Progressive content chunk 99: important data" in result
+
+ def test_readability_returns_none_content_fallback(self):
+ """Test that if readabilipy returns None content, fallback to markdownify preserves content."""
+ html = "