From 8e758d3d5b6fe4fa6f27b898b07b275ff5ed4e04 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 25 May 2026 16:50:20 +0000 Subject: [PATCH] Add bot/crawler filtering to ring middleware Add ability to filter out bot/crawler requests from Rollbar reporting to reduce noise from automated traffic. The middleware now accepts options: - :ignore-bots? - skip reporting for detected bot requests - :bot-patterns - custom regex patterns for bot detection - :ignore-request? - custom predicate for filtering requests Also exports bot-request? function for use in application code. https://claude.ai/code/session_01BS2FMA4B8bCkPbcY5H3z2h --- README.md | 49 ++++++++ src/circleci/rollcage/ring_middleware.clj | 106 ++++++++++++++++-- .../rollcage/test_ring_middleware.clj | 93 +++++++++++++++ 3 files changed, 237 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 625fc1d..b6ebf7a 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,55 @@ user=> (rollcage/setup-uncaught-exception-handler r) See the full [API docs](https://cljdoc.org/d/circleci/rollcage/CURRENT) for more information. +## Ring Middleware + +Rollcage includes Ring middleware to automatically report exceptions: + +```Clojure +(require '[circleci.rollcage.ring-middleware :as middleware]) + +(def app + (-> handler + (middleware/wrap-rollbar rollcage-client))) +``` + +### Filtering Bot/Crawler Requests + +To avoid polluting your Rollbar with errors from bots and crawlers, you can +enable bot filtering: + +```Clojure +(def app + (-> handler + (middleware/wrap-rollbar rollcage-client {:ignore-bots? true}))) +``` + +You can also provide custom bot detection patterns: + +```Clojure +(def app + (-> handler + (middleware/wrap-rollbar rollcage-client + {:ignore-bots? true + :bot-patterns [#"(?i)my-custom-bot" #"(?i)another-crawler"]}))) +``` + +Or use a custom predicate to ignore specific requests: + +```Clojure +(def app + (-> handler + (middleware/wrap-rollbar rollcage-client + {:ignore-request? #(= "/health" (:uri %))}))) +``` + +The `bot-request?` function is also available for use in your own code: + +```Clojure +(middleware/bot-request? request) ; uses default patterns +(middleware/bot-request? request [#"MyBot"]) ; uses custom patterns +``` + ## Contributing If you would like to contibute to the project, please [log an issue](https://cljdoc.org/d/circleci/rollcage/CURRENT) to discuss the feature/bug before submitting a pull request. diff --git a/src/circleci/rollcage/ring_middleware.clj b/src/circleci/rollcage/ring_middleware.clj index d735648..dd77dd8 100644 --- a/src/circleci/rollcage/ring_middleware.clj +++ b/src/circleci/rollcage/ring_middleware.clj @@ -1,12 +1,96 @@ (ns circleci.rollcage.ring-middleware - (:require [circleci.rollcage.core :as rollcage])) - -(defn wrap-rollbar [handler rollcage-client] - (if-not rollcage-client - handler - (fn [req] - (try - (handler req) - (catch Exception e - (rollcage/error rollcage-client e {:url (:uri req)}) - (throw e)))))) + (:require [circleci.rollcage.core :as rollcage] + [clojure.string :as string])) + +(def ^:private default-bot-patterns + "Common bot/crawler User-Agent patterns" + [#"(?i)bot\b" + #"(?i)crawler" + #"(?i)spider" + #"(?i)slurp" + #"(?i)mediapartners" + #"(?i)wget" + #"(?i)curl/" + #"(?i)python-requests" + #"(?i)libwww" + #"(?i)httpunit" + #"(?i)nutch" + #"(?i)phpcrawl" + #"(?i)biglotron" + #"(?i)gigabot" + #"(?i)webalta" + #"(?i)ia_archiver" + #"(?i)facebookexternalhit" + #"(?i)twitterbot" + #"(?i)linkedinbot" + #"(?i)embedly" + #"(?i)quora link preview" + #"(?i)showyoubot" + #"(?i)outbrain" + #"(?i)pinterest" + #"(?i)slackbot" + #"(?i)vkshare" + #"(?i)w3c_validator" + #"(?i)redditbot" + #"(?i)applebot" + #"(?i)whatsapp" + #"(?i)flipboard" + #"(?i)tumblr" + #"(?i)bitlybot" + #"(?i)skypeuripreview" + #"(?i)nuzzel" + #"(?i)discordbot" + #"(?i)qwantify" + #"(?i)pinterestbot" + #"(?i)bitrix" + #"(?i)xing-contenttabreceiver" + #"(?i)chrome-lighthouse" + #"(?i)telegrambot" + #"(?i)ahrefsbot" + #"(?i)semrushbot" + #"(?i)mj12bot" + #"(?i)dotbot" + #"(?i)petalbot" + #"(?i)baiduspider" + #"(?i)yandex" + #"(?i)duckduckbot" + #"(?i)sogou" + #"(?i)exabot" + #"(?i)facebot" + #"(?i)uptimerobot" + #"(?i)pingdom" + #"(?i)statuscake" + #"(?i)headlesschrome"]) + +(defn bot-request? + "Returns true if the request appears to be from a bot/crawler based on User-Agent. + Accepts an optional second argument for custom bot patterns (regex sequences)." + ([req] + (bot-request? req default-bot-patterns)) + ([req bot-patterns] + (when-let [user-agent (or (get-in req [:headers "user-agent"]) + (get-in req [:headers "User-Agent"]))] + (boolean (some #(re-find % user-agent) bot-patterns))))) + +(defn wrap-rollbar + "Ring middleware that reports exceptions to Rollbar. + + Options map (optional third argument): + - :ignore-bots? - When true, skip reporting for bot/crawler requests (default: false) + - :bot-patterns - Custom regex patterns for bot detection (default: built-in patterns) + - :ignore-request? - Custom predicate (fn [request]) to skip reporting for specific requests" + ([handler rollcage-client] + (wrap-rollbar handler rollcage-client {})) + ([handler rollcage-client {:keys [ignore-bots? bot-patterns ignore-request?]}] + (if-not rollcage-client + handler + (fn [req] + (try + (handler req) + (catch Exception e + (let [is-bot? (and ignore-bots? + (bot-request? req (or bot-patterns default-bot-patterns))) + should-ignore? (and ignore-request? (ignore-request? req))] + (when-not (or is-bot? should-ignore?) + (rollcage/error rollcage-client e {:url (:uri req)}))) + (throw e))))))) diff --git a/test/circleci/rollcage/test_ring_middleware.clj b/test/circleci/rollcage/test_ring_middleware.clj index 32b76b9..929d569 100644 --- a/test/circleci/rollcage/test_ring_middleware.clj +++ b/test/circleci/rollcage/test_ring_middleware.clj @@ -36,3 +36,96 @@ e))] (is (= result error)) (is (= 0 (-> rollcage/error bond/calls count))))))))) + +(deftest bot-request-detection + (testing "Detects common bot user agents" + (is (middleware/bot-request? {:headers {"user-agent" "Googlebot/2.1"}})) + (is (middleware/bot-request? {:headers {"user-agent" "Mozilla/5.0 (compatible; Googlebot/2.1)"}})) + (is (middleware/bot-request? {:headers {"user-agent" "Bingbot/2.0"}})) + (is (middleware/bot-request? {:headers {"user-agent" "AhrefsBot/7.0"}})) + (is (middleware/bot-request? {:headers {"user-agent" "curl/7.68.0"}})) + (is (middleware/bot-request? {:headers {"user-agent" "python-requests/2.25.1"}})) + (is (middleware/bot-request? {:headers {"user-agent" "Slackbot-LinkExpanding 1.0"}})) + (is (middleware/bot-request? {:headers {"user-agent" "facebookexternalhit/1.1"}})) + (is (middleware/bot-request? {:headers {"user-agent" "Twitterbot/1.0"}})) + (is (middleware/bot-request? {:headers {"user-agent" "HeadlessChrome/91.0.4472.124"}}))) + + (testing "Does not flag normal browser user agents as bots" + (is (not (middleware/bot-request? {:headers {"user-agent" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}}))) + (is (not (middleware/bot-request? {:headers {"user-agent" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)"}}))) + (is (not (middleware/bot-request? {:headers {"user-agent" "Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X)"}})))) + + (testing "Handles missing user-agent gracefully" + (is (not (middleware/bot-request? {:headers {}}))) + (is (not (middleware/bot-request? {})))) + + (testing "Handles User-Agent header case variations" + (is (middleware/bot-request? {:headers {"User-Agent" "Googlebot/2.1"}}))) + + (testing "Supports custom bot patterns" + (is (middleware/bot-request? {:headers {"user-agent" "MyCustomBot/1.0"}} + [#"MyCustomBot"])) + (is (not (middleware/bot-request? {:headers {"user-agent" "Googlebot/2.1"}} + [#"MyCustomBot"]))))) + +(deftest wrap-rollbar-with-bot-filtering + (let [error (ex-info "something bad happened" {}) + dummy-ring-handler (fn [_] + (throw error) + {:status 200}) + bot-request {:uri "/" :params {} :headers {"user-agent" "Googlebot/2.1"}} + normal-request {:uri "/" :params {} :headers {"user-agent" "Mozilla/5.0 (Windows NT 10.0)"}}] + + (testing "Reports errors for bot requests when ignore-bots? is false (default)" + (let [dummy-rollcage-client {} + wrapped-handler (middleware/wrap-rollbar dummy-ring-handler + dummy-rollcage-client)] + (bond/with-stub [rollcage/error] + (try (wrapped-handler bot-request) (catch Exception _)) + (is (= 1 (-> rollcage/error bond/calls count)))))) + + (testing "Does not report errors for bot requests when ignore-bots? is true" + (let [dummy-rollcage-client {} + wrapped-handler (middleware/wrap-rollbar dummy-ring-handler + dummy-rollcage-client + {:ignore-bots? true})] + (bond/with-stub [rollcage/error] + (try (wrapped-handler bot-request) (catch Exception _)) + (is (= 0 (-> rollcage/error bond/calls count)))))) + + (testing "Still reports errors for normal requests when ignore-bots? is true" + (let [dummy-rollcage-client {} + wrapped-handler (middleware/wrap-rollbar dummy-ring-handler + dummy-rollcage-client + {:ignore-bots? true})] + (bond/with-stub [rollcage/error] + (try (wrapped-handler normal-request) (catch Exception _)) + (is (= 1 (-> rollcage/error bond/calls count)))))) + + (testing "Supports custom bot patterns" + (let [dummy-rollcage-client {} + custom-bot-request {:uri "/" :params {} :headers {"user-agent" "CustomCrawler/1.0"}} + wrapped-handler (middleware/wrap-rollbar dummy-ring-handler + dummy-rollcage-client + {:ignore-bots? true + :bot-patterns [#"CustomCrawler"]})] + (bond/with-stub [rollcage/error] + (try (wrapped-handler custom-bot-request) (catch Exception _)) + (is (= 0 (-> rollcage/error bond/calls count)))))) + + (testing "Supports custom ignore-request? predicate" + (let [dummy-rollcage-client {} + health-check-request {:uri "/health" :params {}} + wrapped-handler (middleware/wrap-rollbar dummy-ring-handler + dummy-rollcage-client + {:ignore-request? #(= "/health" (:uri %))})] + (bond/with-stub [rollcage/error] + (try (wrapped-handler health-check-request) (catch Exception _)) + (is (= 0 (-> rollcage/error bond/calls count)))))) + + (testing "Still throws the exception even when ignoring bots" + (let [dummy-rollcage-client {} + wrapped-handler (middleware/wrap-rollbar dummy-ring-handler + dummy-rollcage-client + {:ignore-bots? true})] + (is (thrown? Exception (wrapped-handler bot-request)))))))