diff --git a/rust_snuba/src/logging.rs b/rust_snuba/src/logging.rs index 49eecfa2e5b..ff562ac1aa1 100644 --- a/rust_snuba/src/logging.rs +++ b/rust_snuba/src/logging.rs @@ -9,14 +9,25 @@ pub fn setup_logging() { .or_else(|_| EnvFilter::try_new("info")) .unwrap(); - // Capture errors & warnings as exceptions, and also send everything at or above INFO as logs - // instead of breadcrumbs. - let sentry_layer = - sentry::integrations::tracing::layer().event_filter(|metadata| match *metadata.level() { - Level::ERROR | Level::WARN => EventFilter::Event | EventFilter::Log, - Level::INFO => EventFilter::Log, + // Only errors are forwarded to Sentry as issues. Warnings are operational + // noise far more often than they're actionable (e.g. consumer + // rebalance/shutdown timeouts), so they're kept as logs alongside + // everything at or above INFO, instead of breadcrumbs. + let sentry_layer = sentry::integrations::tracing::layer().event_filter(|metadata| { + match *metadata.level() { + // The usage accountant is a best-effort billing side channel. Its + // Kafka producer logs "Purged in queue/flight" at ERROR whenever the + // producer is flushed during a consumer shutdown/rebalance. These + // are transient and don't affect ingestion, so keep them as logs + // rather than issues (SNUBA-474, SNUBA-475). + Level::ERROR if metadata.target().starts_with("sentry_usage_accountant") => { + EventFilter::Log + } + Level::ERROR => EventFilter::Event | EventFilter::Log, + Level::WARN | Level::INFO => EventFilter::Log, Level::DEBUG | Level::TRACE => EventFilter::Ignore, - }); + } + }); tracing_subscriber::registry() .with(tracing_subscriber::fmt::layer().json()) diff --git a/snuba/environment.py b/snuba/environment.py index 1850c346a5c..b8311e27540 100644 --- a/snuba/environment.py +++ b/snuba/environment.py @@ -66,7 +66,11 @@ def setup_logging(level: Optional[str] = None) -> None: structlog.processors.StackInfoRenderer(), structlog.processors.format_exc_info, structlog.processors.TimeStamper(fmt="iso", utc=True), - SentryProcessor(), + # Mirror the LoggingIntegration policy on the structlog path: only + # ERROR and above become Sentry issues, WARNING and below stay as + # logs/breadcrumbs. (This is structlog-sentry's default, set + # explicitly so the policy is obvious and robust to upstream changes.) + SentryProcessor(event_level=logging.ERROR), drop_level, JSONRenderer(), ], @@ -100,7 +104,11 @@ def setup_sentry() -> None: integrations=[ FlaskIntegration(), GnuBacktraceIntegration(), - LoggingIntegration(event_level=logging.WARNING), + # Only forward ERROR and above to Sentry as issues. Warnings are + # operational noise far more often than they're actionable (e.g. + # consumer rebalance/shutdown timeouts) and are still captured as + # logs/breadcrumbs. + LoggingIntegration(event_level=logging.ERROR), RedisIntegration(), ThreadingIntegration(propagate_hub=True), ],