diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45d3940..ac78c4c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,7 @@ permissions: env: CARGO_TERM_COLOR: always RUSTFLAGS: "-D warnings" + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true jobs: packaging: @@ -134,7 +135,7 @@ jobs: python3 -c "import sys; sys.exit(1 if float('${COV}') < 45.0 else 0)" \ || (echo "::error::Coverage ${COV}% is below minimum 45%"; exit 1) - name: Upload to Codecov - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: files: lcov.info fail_ci_if_error: false @@ -205,8 +206,10 @@ jobs: target key: ${{ runner.os }}-bench-${{ hashFiles('**/Cargo.lock') }} - name: Run benchmarks - run: cargo bench --package fetchium-core -- --output-format bencher | tee output.txt + # Criterion uses its own harness — capture all output for the store step. + run: cargo bench --package fetchium-core 2>&1 | tee output.txt - name: Store benchmark results + continue-on-error: true uses: benchmark-action/github-action-benchmark@v1 with: tool: 'cargo' @@ -214,6 +217,7 @@ jobs: alert-threshold: '120%' comment-on-alert: true fail-on-alert: false + fail-ci-if-error: false github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 205a859..be89d54 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -8,6 +8,9 @@ on: permissions: contents: read +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + jobs: deploy-production: name: Deploy production stack diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 655f751..8a923f7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,6 +29,7 @@ permissions: env: CARGO_TERM_COLOR: always RUSTFLAGS: "-D warnings" + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true jobs: # ───────────────────────────────────────────────────────────── diff --git a/README.md b/README.md index a557bc2..52bd44e 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,8 @@ Rust-native search, extraction, ranking, and synthesis — delivered as a CLI, a [![Crates.io](https://img.shields.io/crates/v/fetchium-cli.svg)](https://crates.io/crates/fetchium-cli) [![npm](https://img.shields.io/npm/v/fetchium-cli.svg)](https://www.npmjs.com/package/fetchium-cli) [![Downloads](https://img.shields.io/crates/d/fetchium-cli.svg)](https://crates.io/crates/fetchium-cli) +[![Glama](https://glama.ai/mcp/servers/zuhabul/Fetchium/badges/score.svg)](https://glama.ai/mcp/servers/zuhabul/Fetchium) +[![smithery badge](https://smithery.ai/badge/fetchium)](https://smithery.ai/server/fetchium) [Install](#installation) · [Architecture & innovations](#architecture--innovations) · [Quick start](#quick-start) · [For AI agents](#for-ai-agents) · [Docs](docs/) @@ -203,12 +205,88 @@ Full command reference: [docs/guide/commands.md](docs/guide/commands.md). ## For AI agents -- **MCP server** (`fetchium-mcp`) exposes retrieval as Model Context Protocol tools for Codex, - Claude, and other MCP clients. -- **REST API** (`fetchium-api`) serves the same engine over HTTP — `fetchium serve`. -- **Adapters** for [LangChain](adapters/langchain) and [CrewAI](adapters/crewai) live in `adapters/`. +Fetchium ships a first-class **MCP server** — add it to any MCP-compatible client and your agent +can search, fetch, research, and watch YouTube/social content without custom glue code. -See [docs/guide/agent-integration.md](docs/guide/agent-integration.md). +### 30-second setup — Claude Desktop + +Add to `~/Library/Application Support/Claude/claude_desktop_config.json` (macOS) or +`%APPDATA%\Claude\claude_desktop_config.json` (Windows): + +```json +{ + "mcpServers": { + "fetchium": { + "command": "fetchium", + "args": ["serve", "--mode", "mcp"] + } + } +} +``` + +No `fetchium` binary yet? Use `npx` for zero-install: + +```json +{ + "mcpServers": { + "fetchium": { + "command": "npx", + "args": ["-y", "fetchium-cli", "serve", "--mode", "mcp"] + } + } +} +``` + +### Cursor / Windsurf / VS Code (Cline) + +Add the same block to your editor's MCP settings, or run: + +```bash +fetchium serve --mode mcp --port 3001 # HTTP transport for editors that prefer it +``` + +Then point the editor at `http://localhost:3001/mcp`. + +### One-click via Smithery + +[![Install on Smithery](https://smithery.ai/badge/fetchium)](https://smithery.ai/server/fetchium) + +```bash +npx -y @smithery/cli install fetchium --client claude +``` + +### Available MCP tools (12 total) + +| Tool | What it does | +|------|-------------| +| `fetchium_search` | Multi-backend web search with HyperFusion ranking + dedup | +| `fetchium_fetch` | Query-aware content extraction with token budgeting | +| `fetchium_research` | Multi-source research with citations and evidence tracking | +| `fetchium_estimate` | Token-cost estimate for a URL (HEAD only, no download) | +| `fetchium_expand` | Expand a previous result to a deeper PDS tier | +| `youtube_search` | Search YouTube with VideoFusion ranking | +| `youtube_analyze` | Full video analysis: transcript, comments, credibility | +| `youtube_watch` | Summary + key moments for any YouTube URL | +| `youtube_transcript` | Raw transcript with timestamps and highlights | +| `social_research` | Trend research across Reddit, HN, Twitter, TikTok | +| `reddit_search` | Search Reddit posts and comments | +| `hackernews_search` | Search Hacker News stories and discussions | + +### REST API + +```bash +fetchium serve # start REST API on :3000 +curl localhost:3000/v1/search -d '{"query":"rust async"}' +``` + +### LangChain / CrewAI adapters + +```python +from fetchium_langchain import FetchiumSearchTool, FetchiumResearchTool +from fetchium_crewai import FetchiumSearchTool +``` + +See [docs/guide/agent-integration.md](docs/guide/agent-integration.md) for full examples. ## Configuration diff --git a/assets/logo.svg b/assets/logo.svg new file mode 100644 index 0000000..9fd55ba --- /dev/null +++ b/assets/logo.svg @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/crates/fetchium-core/src/embeddings/engine.rs b/crates/fetchium-core/src/embeddings/engine.rs index 293e05e..87eee3b 100644 --- a/crates/fetchium-core/src/embeddings/engine.rs +++ b/crates/fetchium-core/src/embeddings/engine.rs @@ -7,6 +7,7 @@ use crate::error::FetchiumError; use once_cell::sync::Lazy; +use tokio::sync::Semaphore; use tracing::{debug, warn}; /// Ollama endpoint (configurable via env var). @@ -17,11 +18,15 @@ static OLLAMA_URL: Lazy = Lazy::new(|| { /// Embedding model to use. const EMBED_MODEL: &str = "nomic-embed-text"; +/// Ollama runs single-threaded inference — concurrent requests cause connection +/// errors on the second/third caller. One permit serializes embed calls. +static OLLAMA_SEMAPHORE: Lazy = Lazy::new(|| Semaphore::new(1)); + /// Shared async HTTP client (connection pooling). static HTTP_CLIENT: Lazy = Lazy::new(|| { reqwest::Client::builder() .timeout(std::time::Duration::from_secs(10)) - .pool_max_idle_per_host(4) + .pool_max_idle_per_host(1) .build() .expect("Failed to create HTTP client") }); @@ -101,6 +106,13 @@ pub async fn embed_batch_async(texts: &[&str]) -> Result>, Fetchium if texts.is_empty() { return Ok(Vec::new()); } + // Ollama is single-threaded — concurrent requests cause connection errors. + // Acquire permit before sending; dropped automatically when the fn returns. + let _permit = OLLAMA_SEMAPHORE + .acquire() + .await + .map_err(|_| FetchiumError::Internal("Ollama semaphore closed".into()))?; + debug!( "Embedding batch of {} texts via Ollama (async)", texts.len() diff --git a/crates/fetchium-core/src/search/reddit.rs b/crates/fetchium-core/src/search/reddit.rs index 84d53b1..1b5186b 100644 --- a/crates/fetchium-core/src/search/reddit.rs +++ b/crates/fetchium-core/src/search/reddit.rs @@ -15,8 +15,11 @@ use tracing::debug; /// Reddit search JSON endpoint. const REDDIT_SEARCH: &str = "https://www.reddit.com/search.json"; -/// Backoff window for Reddit rate-limit / anti-bot bursts. -const REDDIT_COOLDOWN_SECS: u64 = 180; +/// Backoff for 429 rate-limit (back off long enough for Reddit's window to reset). +const REDDIT_RATELIMIT_COOLDOWN_SECS: u64 = 180; +/// Backoff for 403 access-denied (IP-level block — retrying sooner doesn't help, +/// but waiting 3 minutes is too long; 30s matches the network-error cooldown). +const REDDIT_FORBIDDEN_COOLDOWN_SECS: u64 = 30; static REDDIT_COOLDOWN_UNTIL_MS: AtomicU64 = AtomicU64::new(0); #[derive(Debug, Deserialize)] @@ -104,12 +107,20 @@ impl SearchBackend for RedditBackend { if !resp.status().is_success() { let status = resp.status(); - if status.as_u16() == 429 || status.as_u16() == 403 { - let until = now_ms() + REDDIT_COOLDOWN_SECS * 1000; + if status.as_u16() == 429 { + let until = now_ms() + REDDIT_RATELIMIT_COOLDOWN_SECS * 1000; REDDIT_COOLDOWN_UNTIL_MS.store(until, Ordering::Relaxed); return Err(FetchiumError::Search(format!( "Reddit HTTP {status} — cooling down for {}s", - REDDIT_COOLDOWN_SECS + REDDIT_RATELIMIT_COOLDOWN_SECS + ))); + } + if status.as_u16() == 403 { + let until = now_ms() + REDDIT_FORBIDDEN_COOLDOWN_SECS * 1000; + REDDIT_COOLDOWN_UNTIL_MS.store(until, Ordering::Relaxed); + return Err(FetchiumError::Search(format!( + "Reddit HTTP {status} — cooling down for {}s", + REDDIT_FORBIDDEN_COOLDOWN_SECS ))); } debug!("Reddit non-success HTTP {status}, skipping"); diff --git a/crates/fetchium-mcp/src/lib.rs b/crates/fetchium-mcp/src/lib.rs index 717a5d1..bd29e68 100644 --- a/crates/fetchium-mcp/src/lib.rs +++ b/crates/fetchium-mcp/src/lib.rs @@ -106,18 +106,18 @@ pub async fn run_mcp_stdio(config: FetchiumConfig) -> anyhow::Result<()> { } }; - let response = handle_message(&line, &config, &http, &cache).await; - - let json_out = serde_json::to_string(&response).unwrap_or_else(|e| { - format!( - r#"{{"jsonrpc":"2.0","id":null,"error":{{"code":-32603,"message":"{}"}}}}"#, - e - ) - }); + if let Some(response) = handle_message(&line, &config, &http, &cache).await { + let json_out = serde_json::to_string(&response).unwrap_or_else(|e| { + format!( + r#"{{"jsonrpc":"2.0","id":null,"error":{{"code":-32603,"message":"{}"}}}}"#, + e + ) + }); - let mut out = stdout.lock(); - let _ = writeln!(out, "{json_out}"); - let _ = out.flush(); + let mut out = stdout.lock(); + let _ = writeln!(out, "{json_out}"); + let _ = out.flush(); + } } eprintln!("[fetchium-mcp] Server shutting down."); @@ -152,21 +152,32 @@ pub async fn run_mcp_http(config: FetchiumConfig, port: u16) -> anyhow::Result<( Ok(()) } -/// Dispatch a single JSON-RPC message line and return the response. +/// Dispatch a single JSON-RPC message line. Returns `None` for notifications +/// (which must not receive a response per the JSON-RPC 2.0 / MCP spec). async fn handle_message( line: &str, config: &FetchiumConfig, http: &HttpClient, cache: &MemoryCache, -) -> JsonRpcResponse { +) -> Option { let req: JsonRpcRequest = match serde_json::from_str(line) { Ok(r) => r, Err(e) => { - return JsonRpcResponse::err(Value::Null, -32700, format!("Parse error: {e}")); + return Some(JsonRpcResponse::err( + Value::Null, + -32700, + format!("Parse error: {e}"), + )); } }; - handle_request(req, config, http, cache).await + // JSON-RPC 2.0: notifications have no `id` and must not receive a response. + if req.id.is_none() { + eprintln!("[fetchium-mcp] notification: {}", req.method); + return None; + } + + Some(handle_request(req, config, http, cache).await) } async fn handle_request( @@ -197,12 +208,6 @@ async fn handle_request( ) } - "notifications/initialized" => { - // Notification — no response needed; return empty result - eprintln!("[fetchium-mcp] initialized"); - JsonRpcResponse::ok(id, Value::Null) - } - // List available tools "tools/list" => JsonRpcResponse::ok(id, json!({ "tools": tools::tool_definitions() })), diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index a5a2f31..c56072b 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -11,6 +11,7 @@ cargo-fuzz = true libfuzzer-sys = "0.4" fetchium-core = { path = "../crates/fetchium-core" } serde_json = "1" +toml = "0.8" # Fuzz is its own workspace so cargo-fuzz can find the targets [workspace] diff --git a/fuzz/fuzz_targets/fuzz_config_parse.rs b/fuzz/fuzz_targets/fuzz_config_parse.rs index cfd1178..5968022 100644 --- a/fuzz/fuzz_targets/fuzz_config_parse.rs +++ b/fuzz/fuzz_targets/fuzz_config_parse.rs @@ -1,10 +1,10 @@ #![no_main] // Fuzz TOML config parsing. Must NEVER panic. -use fetchium_core::config::HsxConfig; +use fetchium_core::config::FetchiumConfig; use libfuzzer_sys::fuzz_target; fuzz_target!(|data: &[u8]| { if let Ok(toml_str) = std::str::from_utf8(data) { - let _ = toml::from_str::(toml_str); + let _ = toml::from_str::(toml_str); } }); diff --git a/glama.json b/glama.json new file mode 100644 index 0000000..b3ffc8e --- /dev/null +++ b/glama.json @@ -0,0 +1,76 @@ +{ + "$schema": "https://glama.ai/mcp/schemas/server.json", + "maintainers": ["zuhabul"], + "name": "Fetchium", + "description": "Token-efficient web search, extraction, social research, and AI research as an MCP server. 12 tools: multi-backend search, query-aware content extraction, multi-source research with citations, YouTube analysis, Reddit, and Hacker News search. Rust-native, zero-install via npx.", + "type": "mcp_server", + "status": "stable", + "transport": ["stdio"], + "license": "MIT", + "homepage": "https://github.com/zuhabul/Fetchium", + "repository": "https://github.com/zuhabul/Fetchium", + "icon": "https://raw.githubusercontent.com/zuhabul/Fetchium/main/assets/logo.svg", + "keywords": ["search", "web", "extraction", "research", "rag", "mcp", "rust", "youtube", "reddit"], + "categories": ["search", "web-scraping", "rag"], + "tools": [ + { + "name": "fetchium_search", + "description": "Search the web with multi-backend support (Google, DuckDuckGo, Brave, Bing, Tavily, Serper, Exa, and more). Returns ranked, deduplicated results with scores." + }, + { + "name": "fetchium_fetch", + "description": "Fetch and extract clean content from any URL using the 5-layer CEP cascade: CSS selectors → readability → headless JS → PDF → OCR." + }, + { + "name": "fetchium_research", + "description": "Conduct multi-source research with citations and evidence tracking. Searches, extracts, ranks, validates, and synthesizes findings with citation chains." + }, + { + "name": "fetchium_estimate", + "description": "Estimate the token cost of fetching a URL before spending extraction budget." + }, + { + "name": "fetchium_expand", + "description": "Expand a previous result by result_id using Progressive Detail Streaming tiers." + }, + { + "name": "youtube_search", + "description": "Search YouTube videos with VideoFusion ranking across relevance, freshness, authority, engagement, and educational scores." + }, + { + "name": "youtube_analyze", + "description": "Analyze one YouTube video with metadata, transcript, comments, credibility, clickbait detection, and educational scoring." + }, + { + "name": "youtube_watch", + "description": "Produce a unified YouTube watch report with metadata, transcript, summary, key moments, and comment signals." + }, + { + "name": "youtube_transcript", + "description": "Extract a YouTube transcript with key moments and transcript quality scoring." + }, + { + "name": "social_research", + "description": "Run cross-platform social research across Twitter/X, Reddit, TikTok, Hacker News, and YouTube." + }, + { + "name": "reddit_search", + "description": "Search Reddit posts with sentiment analysis, subreddit clustering, and viral detection." + }, + { + "name": "hackernews_search", + "description": "Search Hacker News stories via Algolia and Firebase APIs, returning ranked stories with engagement metrics." + } + ], + "installation": { + "npm": "npm install -g fetchium-cli", + "npx": "npx -y fetchium-cli serve --mode mcp", + "cargo": "cargo install fetchium-cli", + "homebrew": "brew install zuhabul/fetchium/fetchium" + }, + "mcp": { + "command": "fetchium", + "args": ["serve", "--mode", "mcp"], + "transport": "stdio" + } +} diff --git a/infra/searxng/settings.yml b/infra/searxng/settings.yml index a98ace4..4b8cdae 100644 --- a/infra/searxng/settings.yml +++ b/infra/searxng/settings.yml @@ -1,10 +1,12 @@ -# SearXNG settings for Fetchium — residential proxy unlocks Google, DDG, Brave. -# DataImpulse residential IPs: 95-99% success vs 40-60% for datacenter. +# SearXNG settings for Fetchium. +# NOTE: DataImpulse HTTP proxy returns 407 NO_USER inside Docker (httpx CONNECT +# tunnel does not forward Proxy-Authorization). Proxy removed — Google/DDG may +# CAPTCHA but Yahoo, Yandex, Bing, Brave, Startpage work reliably direct. +# fetchium-core's own scrapers handle Google/DDG with working residential proxy. use_default_settings: engines: keep_only: - - google - startpage - duckduckgo - brave @@ -39,34 +41,29 @@ search: safe_search: 0 autocomplete: "" default_lang: "auto" - ban_time_on_fail: 2 - max_ban_time_on_fail: 30 + ban_time_on_fail: 5 + max_ban_time_on_fail: 60 suspended_times: - SearxEngineAccessDenied: 30 - SearxEngineCaptcha: 60 - SearxEngineTooManyRequests: 30 - cf_SearxEngineCaptcha: 120 - cf_SearxEngineAccessDenied: 60 - recaptcha_SearxEngineCaptcha: 300 + SearxEngineAccessDenied: 60 + SearxEngineCaptcha: 120 + SearxEngineTooManyRequests: 60 + cf_SearxEngineCaptcha: 300 + cf_SearxEngineAccessDenied: 120 + recaptcha_SearxEngineCaptcha: 600 formats: - html - json engines: - # Google — highest quality (residential proxy bypasses blocks) - - name: google - timeout: 2.5 - weight: 4.0 - - # Startpage — Google proxy, excellent fallback when Google suspends + # Startpage — Google proxy, works without residential IP - name: startpage - timeout: 2.0 - weight: 2.0 + timeout: 3.0 + weight: 2.5 - # DuckDuckGo — residential IPs bypass CAPTCHA, diverse results + # DuckDuckGo — works direct for most queries - name: duckduckgo - timeout: 2.0 - weight: 1.8 + timeout: 3.0 + weight: 2.0 # YouTube — video results for how-to, tutorials, and current events - name: youtube @@ -75,41 +72,30 @@ engines: # Brave — independent index, good for privacy/tech queries - name: brave - timeout: 2.0 + timeout: 3.0 weight: 1.5 - # Yahoo — reliable, fast + # Yahoo — reliable, fast, no blocks direct - name: yahoo - timeout: 1.5 + timeout: 2.0 weight: 1.2 # Yandex — strong for non-English, Cyrillic, multilingual - name: yandex - timeout: 1.5 + timeout: 2.0 weight: 0.8 - # Bing — supplementary coverage + # Bing — supplementary coverage, no blocks direct - name: bing - timeout: 1.0 - weight: 0.5 + timeout: 2.0 + weight: 0.8 outgoing: - request_timeout: 2.5 - max_request_timeout: 5.0 + request_timeout: 3.0 + max_request_timeout: 6.0 useragent_suffix: "" - pool_connections: 500 - pool_maxsize: 128 + pool_connections: 100 + pool_maxsize: 32 keepalive_expiry: 30.0 retries: 1 enable_http2: true - # DataImpulse residential proxy — ONLY Google and DDG need residential IPs. - # Bing, Brave, Yahoo, Yandex, Startpage work fine direct → eliminates ~80% waste. - proxies: - https://google.com: - - http://75e0e5b80d4d887e2f36:6703f7fe0e9ea89c@gw.dataimpulse.com:823 - https://www.google.com: - - http://75e0e5b80d4d887e2f36:6703f7fe0e9ea89c@gw.dataimpulse.com:823 - https://html.duckduckgo.com: - - http://75e0e5b80d4d887e2f36:6703f7fe0e9ea89c@gw.dataimpulse.com:823 - https://lite.duckduckgo.com: - - http://75e0e5b80d4d887e2f36:6703f7fe0e9ea89c@gw.dataimpulse.com:823 diff --git a/packages/npm/install.js b/packages/npm/install.js index d809046..62e19b0 100644 --- a/packages/npm/install.js +++ b/packages/npm/install.js @@ -41,10 +41,7 @@ function getArtifact() { const filename = `${info.name}${info.ext}`; return { filename, - // Primary: GitHub Releases url: `https://github.com/${REPO}/releases/download/v${VERSION}/${filename}`, - // Fallback: same URL (kept for future CDN swap) - fallbackUrl: `https://github.com/${REPO}/releases/download/v${VERSION}/${filename}`, binName: info.bin, isZip: info.ext === ".zip", }; @@ -52,20 +49,20 @@ function getArtifact() { // ── Download helper ─────────────────────────────────────────────────────────── -function download(url, dest) { +function downloadOnce(url, dest) { return new Promise((resolve, reject) => { const file = fs.createWriteStream(dest); let redirectCount = 0; - function request(url) { + function request(currentUrl) { if (++redirectCount > 5) return reject(new Error("Too many redirects")); - https.get(url, { headers: { "User-Agent": `fetchium-npm-installer/${VERSION}` } }, (res) => { + https.get(currentUrl, { headers: { "User-Agent": `fetchium-npm-installer/${VERSION}` } }, (res) => { if ([301, 302, 307, 308].includes(res.statusCode)) { return request(res.headers.location); } if (res.statusCode !== 200) { file.destroy(); - return reject(new Error(`HTTP ${res.statusCode} downloading from:\n ${url}`)); + return reject(new Error(`HTTP ${res.statusCode} from ${currentUrl}`)); } let downloaded = 0; res.on("data", (chunk) => { @@ -82,6 +79,26 @@ function download(url, dest) { }); } +// Retries with exponential backoff — 504/transient CDN errors clear within seconds. +async function download(url, dest) { + const delays = [0, 5000, 15000]; // immediate, 5s, 15s + let lastErr; + for (let i = 0; i < delays.length; i++) { + if (delays[i] > 0) { + process.stdout.write(` Retry ${i}/${delays.length - 1} in ${delays[i] / 1000}s...\n`); + await new Promise((r) => setTimeout(r, delays[i])); + } + try { + await downloadOnce(url, dest); + return; + } catch (err) { + lastErr = err; + if (fs.existsSync(dest)) { try { fs.unlinkSync(dest); } catch {} } + } + } + throw lastErr; +} + // ── Extract helper ──────────────────────────────────────────────────────────── function extract(archive, destDir, isZip) { @@ -164,8 +181,13 @@ async function main() { } catch { console.log(`\n✓ fetchium v${VERSION} installed`); } - console.log(` Run: fetchium --help`); - console.log(" Docs: https://docs.fetchium.com\n"); + console.log("\nQuick start:"); + console.log(' fetchium search "your query"'); + console.log(' fetchium research "explain quantum computing"'); + console.log("\nAdd to Claude Desktop (MCP):"); + console.log(' { "mcpServers": { "fetchium": { "command": "fetchium", "args": ["serve","--mode","mcp"] } } }'); + console.log("\nDocs: https://github.com/zuhabul/Fetchium"); + console.log("Health: fetchium doctor\n"); } main().catch((err) => { diff --git a/scripts/install.sh b/scripts/install.sh index 2a4e3b9..1f30ce4 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -37,15 +37,24 @@ detect_platform() { } # ── Download helper ─────────────────────────────────────────────────────────── +# Retries with exponential backoff; 504s from GitHub CDN clear within seconds. download() { URL="$1"; DEST="$2" - if command -v curl >/dev/null 2>&1; then - curl -fsSL --retry 3 --retry-delay 2 -o "$DEST" "$URL" - elif command -v wget >/dev/null 2>&1; then - wget -q --tries=3 --waitretry=2 -O "$DEST" "$URL" - else - error "curl or wget is required. Install one and try again." - fi + ATTEMPTS=3; DELAY=5 + for i in $(seq 1 $ATTEMPTS); do + if command -v curl >/dev/null 2>&1; then + curl -fsSL --max-time 120 -o "$DEST" "$URL" && return 0 + elif command -v wget >/dev/null 2>&1; then + wget -q --timeout=120 -O "$DEST" "$URL" && return 0 + else + error "curl or wget is required. Install one and try again." + fi + if [ "$i" -lt "$ATTEMPTS" ]; then + warn "Download failed (attempt $i/$ATTEMPTS), retrying in ${DELAY}s..." + sleep "$DELAY"; DELAY=$((DELAY * 3)) + fi + done + return 1 } # ── Version resolution ──────────────────────────────────────────────────────── @@ -104,7 +113,14 @@ main() { trap "rm -rf '$TMP_DIR'" EXIT info "Downloading fetchium ${VERSION} for ${PLATFORM}/${ARCH}..." - download "$ARCHIVE_URL" "$ARCHIVE" + if ! download "$ARCHIVE_URL" "$ARCHIVE"; then + warn "GitHub CDN download failed after retries." + if command -v npm >/dev/null 2>&1; then + warn "Falling back to: npm install -g fetchium-cli" + npm install -g fetchium-cli && exit 0 || true + fi + error "Download failed. Try manually:\n npm install -g fetchium-cli\n cargo install fetchium-cli" + fi download "$SHA_URL" "$CHECKSUM" verify_checksum "$ARCHIVE" "$CHECKSUM" @@ -123,12 +139,15 @@ main() { printf "\n" success "fetchium ${VERSION} installed to ${INSTALL_DIR}/fetchium" printf "\n" - printf " ${BOLD}Get started:${RESET}\n" - printf " fetchium --help\n" + printf " ${BOLD}Quick start:${RESET}\n" printf " fetchium search \"your query\"\n" + printf " fetchium research \"explain quantum computing\"\n" + printf "\n" + printf " ${BOLD}Add to Claude Desktop (MCP):${RESET}\n" + printf ' { "mcpServers": { "fetchium": { "command": "fetchium", "args": ["serve","--mode","mcp"] } } }\n' printf "\n" - printf " ${BOLD}Docs:${RESET} https://docs.fetchium.com\n" - printf " ${BOLD}API key:${RESET} https://app.fetchium.com\n\n" + printf " ${BOLD}Docs:${RESET} https://github.com/zuhabul/Fetchium\n" + printf " ${BOLD}Health:${RESET} fetchium doctor\n\n" } main "$@" diff --git a/smithery.yaml b/smithery.yaml new file mode 100644 index 0000000..6035fda --- /dev/null +++ b/smithery.yaml @@ -0,0 +1,33 @@ +startCommand: + type: stdio + configSchema: + type: object + properties: + geminiApiKey: + type: string + title: Gemini API Key + description: "Optional: Gemini API key for AI-powered synthesis in research and deep commands. Get one free at https://aistudio.google.com" + tavilyApiKey: + type: string + title: Tavily API Key + description: "Optional: Tavily premium search API key for higher-quality results. Get one at https://tavily.com" + serperApiKey: + type: string + title: Serper API Key + description: "Optional: Serper Google Search API key. Get one at https://serper.dev" + exaApiKey: + type: string + title: Exa API Key + description: "Optional: Exa neural search API key. Get one at https://exa.ai" + required: [] + commandFunction: |- + (config) => ({ + command: "fetchium", + args: ["serve", "--mode", "mcp"], + env: { + ...(config.geminiApiKey ? { GEMINI_API_KEY: config.geminiApiKey } : {}), + ...(config.tavilyApiKey ? { TAVILY_API_KEY: config.tavilyApiKey } : {}), + ...(config.serperApiKey ? { SERPER_API_KEY: config.serperApiKey } : {}), + ...(config.exaApiKey ? { EXA_API_KEY: config.exaApiKey } : {}), + } + })