From 93375d8423acafa8b30a8cba76e857279cdf5e79 Mon Sep 17 00:00:00 2001 From: silas-scitix Date: Tue, 23 Jun 2026 17:59:39 +0800 Subject: [PATCH] [Store] Throttle repeated no-master client logs via LOG_EVERY_T (follow-up to #2484) During a persistent no-master window, the storage heartbeat loop (StorageHeartbeatThreadMain) and the leader monitor loop (LeaderMonitorThreadMain) re-emit the same failure log on every retry (fail_ping_interval_ms / kErrorRetryInterval, ~1s). With many clients this becomes a log storm that drowns out other diagnostics and grows the log volume linearly with the client count. This is a follow-up to #2484, which introduced the heartbeat/leader-monitor retry loops. Wrap each repeating no-master log site with glog's built-in LOG_EVERY_T(SEV, 30) so a given call site emits at most once per 30 wall-seconds per process while preserving the original message text, severity, and the first occurrence. No throttle helper is hand-rolled; the built-in glog macro is used directly. Sites changed (mooncake-store/src/client_service.cpp): StorageHeartbeatThreadMain: "Failed to ping master", "Failed to get new master view", "No active master view is published yet", "Failed to connect to master", "Failed to ping master for N times (non-HA)", "Reconnect failed to". LeaderMonitorThreadMain: "Failed to wait for leader view change", "Failed to switch to leader". The one-shot connect logs in ConnectToMaster are intentionally left unthrottled (they fire once per connect attempt, not in a retry loop). Behavior is unchanged except for log frequency: retry intervals, sleeps, ping_fail_count, and reconnect control flow are untouched. Build is GREEN and ctest shows no regression on the heartbeat, reconnect, and HA recovery tests; clang-format check passes. --- mooncake-store/src/client_service.cpp | 38 ++++++++++++++++----------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/mooncake-store/src/client_service.cpp b/mooncake-store/src/client_service.cpp index 7210e0d52b..6a80291396 100644 --- a/mooncake-store/src/client_service.cpp +++ b/mooncake-store/src/client_service.cpp @@ -599,8 +599,9 @@ void Client::LeaderMonitorThreadMain() { auto view_change = leader_coordinator_->WaitForViewChange( known_version, kViewChangeTimeout); if (!view_change) { - LOG(WARNING) << "Failed to wait for leader view change: " - << toString(view_change.error()); + LOG_EVERY_T(WARNING, 30) + << "Failed to wait for leader view change: " + << toString(view_change.error()); std::this_thread::sleep_for(kErrorRetryInterval); continue; } @@ -611,9 +612,10 @@ void Client::LeaderMonitorThreadMain() { auto err = SwitchLeader(view_change->current_view.value()); if (err != ErrorCode::OK) { - LOG(WARNING) << "Failed to switch to leader " - << view_change->current_view->leader_address << ": " - << toString(err); + LOG_EVERY_T(WARNING, 30) + << "Failed to switch to leader " + << view_change->current_view->leader_address << ": " + << toString(err); std::this_thread::sleep_for(kErrorRetryInterval); } } @@ -3786,7 +3788,7 @@ void Client::StorageHeartbeatThreadMain() { ping_fail_count++; last_ping_success_.store(false); if (ping_fail_count < max_ping_fail_count) { - LOG(ERROR) << "Failed to ping master"; + LOG_EVERY_T(ERROR, 30) << "Failed to ping master"; std::this_thread::sleep_for( std::chrono::milliseconds(fail_ping_interval_ms)); continue; @@ -3799,14 +3801,15 @@ void Client::StorageHeartbeatThreadMain() { << " times; fetching latest master view and reconnecting"; auto current_view = leader_coordinator_->ReadCurrentView(); if (!current_view) { - LOG(ERROR) << "Failed to get new master view: " - << toString(current_view.error()); + LOG_EVERY_T(ERROR, 30) << "Failed to get new master view: " + << toString(current_view.error()); std::this_thread::sleep_for( std::chrono::milliseconds(fail_ping_interval_ms)); continue; } if (!current_view.value().has_value()) { - LOG(WARNING) << "No active master view is published yet"; + LOG_EVERY_T(WARNING, 30) + << "No active master view is published yet"; std::this_thread::sleep_for( std::chrono::milliseconds(fail_ping_interval_ms)); continue; @@ -3815,8 +3818,9 @@ void Client::StorageHeartbeatThreadMain() { const auto& next_view = current_view.value().value(); auto err = SwitchLeader(next_view); if (err != ErrorCode::OK) { - LOG(ERROR) << "Failed to connect to master " - << next_view.leader_address << ": " << toString(err); + LOG_EVERY_T(ERROR, 30) + << "Failed to connect to master " + << next_view.leader_address << ": " << toString(err); std::this_thread::sleep_for( std::chrono::milliseconds(fail_ping_interval_ms)); continue; @@ -3826,13 +3830,15 @@ void Client::StorageHeartbeatThreadMain() { ping_fail_count = 0; } else { const std::string current_master_address = direct_master_address_; - LOG(ERROR) << "Failed to ping master for " << ping_fail_count - << " times (non-HA); reconnecting to " - << current_master_address; + LOG_EVERY_T(ERROR, 30) + << "Failed to ping master for " << ping_fail_count + << " times (non-HA); reconnecting to " + << current_master_address; auto err = master_client_.Connect(current_master_address); if (err != ErrorCode::OK) { - LOG(ERROR) << "Reconnect failed to " << current_master_address - << ": " << toString(err); + LOG_EVERY_T(ERROR, 30) + << "Reconnect failed to " << current_master_address << ": " + << toString(err); std::this_thread::sleep_for( std::chrono::milliseconds(fail_ping_interval_ms)); continue;