From 3f04ab7884b165fab2217afc5cbce10833b13036 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 3 Jun 2026 13:18:26 +0200 Subject: [PATCH 1/2] ref(taskbroker): Migrate ingest-profiles devenv to new config format Move the ingest-profiles taskbroker to the new kafka_clusters/kafka_topics format. Raw mode moves onto the profiles topic's per-topic raw block, and a retry topic is now mandatory for raw topics (raw messages aren't activations, so retries can't loop back into the raw topic). Retries are routed to the main taskworker topic so the default taskbroker picks them up instead of requiring another broker. Depends on taskbroker #668 (per-topic raw + mandatory-retry validation). ref STREAM-1093 Co-Authored-By: Claude Opus 4.8 --- devservices/config.yml | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/devservices/config.yml b/devservices/config.yml index 176478efee9dd4..b935865d5ef39e 100644 --- a/devservices/config.yml +++ b/devservices/config.yml @@ -497,14 +497,31 @@ services: ingest-profiles: image: ghcr.io/getsentry/taskbroker:nightly environment: - TASKBROKER_KAFKA_CLUSTER: 'kafka-kafka-1:9093' - TASKBROKER_KAFKA_TOPIC: 'profiles' - TASKBROKER_KAFKA_CONSUMER_GROUP: 'ingest-profiles' + # Multi-topic/cluster config via env vars: nested keys use "__", and + # figment lowercases each segment, so topic/cluster names can only use + # underscores (not hyphens) when set this way. + TASKBROKER_KAFKA_CLUSTERS__DEFAULT__ADDRESS: 'kafka-kafka-1:9093' + # Consumed topic, in raw (passthrough) mode. Raw mode is configured per + # topic via the topic's `raw` block. + TASKBROKER_KAFKA_TOPICS__PROFILES__CLUSTER: 'default' + TASKBROKER_KAFKA_TOPICS__PROFILES__CONSUMER_GROUP: 'ingest-profiles' + TASKBROKER_KAFKA_TOPICS__PROFILES__RAW__NAMESPACE: 'ingest.profiling.passthrough' + TASKBROKER_KAFKA_TOPICS__PROFILES__RAW__APPLICATION: 'sentry' + TASKBROKER_KAFKA_TOPICS__PROFILES__RAW__TASKNAME: 'sentry.profiles.task.process_profile_from_kafka' + TASKBROKER_KAFKA_TOPICS__PROFILES__RAW__PROCESSING_DEADLINE_DURATION: '30' + # A raw topic's messages aren't activations, so retries (which are + # activation-encoded) cannot loop back into `profiles`; a separate retry + # topic is mandatory. Send them to the main `taskworker` topic so the + # default taskbroker picks them up instead of standing up another broker. + TASKBROKER_KAFKA_RETRY_TOPIC: 'taskworker' + TASKBROKER_KAFKA_TOPICS__TASKWORKER__CLUSTER: 'default' + TASKBROKER_KAFKA_TOPICS__TASKWORKER__CONSUMER_GROUP: 'ingest-profiles' + TASKBROKER_KAFKA_TOPICS__TASKWORKER__PRODUCE_ONLY: 'true' + TASKBROKER_KAFKA_DEADLETTER_TOPIC: 'taskworker_dlq' + TASKBROKER_KAFKA_TOPICS__TASKWORKER_DLQ__CLUSTER: 'default' + TASKBROKER_KAFKA_TOPICS__TASKWORKER_DLQ__CONSUMER_GROUP: 'ingest-profiles' + TASKBROKER_KAFKA_TOPICS__TASKWORKER_DLQ__PRODUCE_ONLY: 'true' TASKBROKER_CREATE_MISSING_TOPICS: 'true' - TASKBROKER_RAW_MODE: 'true' - TASKBROKER_RAW_NAMESPACE: 'ingest.profiling.passthrough' - TASKBROKER_RAW_APPLICATION: 'sentry' - TASKBROKER_RAW_TASKNAME: 'sentry.profiles.task.process_profile_from_kafka' # TODO(STREAM-1041): Remove debug logging once passthrough mode is stable TASKBROKER_LOG_FILTER: 'debug' ports: From b2139ceb5cc0353a0ea2503c3b903f5b29404c8a Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 3 Jun 2026 22:06:26 +0200 Subject: [PATCH 2/2] ref(logging): configure grpc logger so its errors aren't dropped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sentry's LOGGING uses disable_existing_loggers=True and has no `grpc` logger entry. That makes grpc's own logging nondeterministic: if grpc's loggers are instantiated before this config is applied they get disabled (silent), and otherwise they propagate to root (console + a Sentry event). This matters for the push-mode taskworker, which runs a grpc server via taskbroker_client. When a PushTask body fails to deserialize, grpc logs "Exception deserializing request!" at ERROR on the `grpc._common` logger and aborts the call with INTERNAL — but whether that log is visible depends on import order. Configure `grpc` explicitly (ERROR, console, no propagation) so these failures land on the console deterministically, matching how other third-party loggers (arroyo, taskbroker_client, urllib3) are handled. Co-Authored-By: Claude Opus 4.8 --- src/sentry/conf/server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py index 37feae7f93f9a9..17404db5d610cd 100644 --- a/src/sentry/conf/server.py +++ b/src/sentry/conf/server.py @@ -1333,6 +1333,8 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str: }, "arroyo": {"level": "INFO", "handlers": ["console"], "propagate": False}, "taskbroker_client": {"level": "INFO", "handlers": ["console"], "propagate": False}, + # Configure grpc explicitly so its errors aren't dropped by disable_existing_loggers. + "grpc": {"level": "ERROR", "handlers": ["console"], "propagate": False}, "static_compiler": {"level": "INFO"}, "django.request": { "level": "WARNING",