From bddfe5515e62520fd953cfe011d5c31abb5d22f0 Mon Sep 17 00:00:00 2001 From: Colton Allen Date: Wed, 24 Jun 2026 12:39:56 -0500 Subject: [PATCH 1/4] ref(deps): Drop hard redis-py-cluster dependency to allow redis>=4 redis-py-cluster (EOL) pins redis<4.0.0, so any consumer of taskbroker-client is transitively blocked from upgrading to redis>=4.1, where cluster support is built in as redis.RedisCluster. The package only uses RedisCluster as a type annotation in scheduler/storage.py (never constructs it), so: - Import RedisCluster from `redis` when available (>=4.1), falling back to `rediscluster` for redis<4 installs that opt into the new `cluster` extra. - Move redis-py-cluster from a hard dependency to an optional `cluster` extra so default installs no longer cap redis below 4. - De-parametrize the `RedisCluster[str] | StrictRedis[str]` annotation: redis>=4 ships py.typed with non-generic client classes, so the subscripted form is a type error ("Expected no type arguments"). The bare union type-checks cleanly on both redis 3.x and 4.x. Verified: import resolves to redis.cluster.RedisCluster on redis 8.0.1 and falls back to rediscluster on redis 3.4.1; pyright reports 0 errors on the bare annotation under redis>=4 bundled types. --- clients/python/pyproject.toml | 6 +++++- .../python/src/taskbroker_client/scheduler/storage.py | 9 +++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index f8a84ff3..5965c62d 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -15,7 +15,6 @@ dependencies = [ "orjson>=3.10.10", "protobuf>=5.28.3", "redis>=3.4.1", - "redis-py-cluster>=2.1.0", "zstandard>=0.18.0", ] @@ -34,6 +33,11 @@ dev = [ "types-protobuf>=5.27.0.20240626,<6.0.0", ] [project.optional-dependencies] +# redis<4 does not bundle cluster support; pull in redis-py-cluster (EOL) for it. +# Not needed on redis>=4.1, which provides redis.RedisCluster natively. +cluster = [ + "redis-py-cluster>=2.1.0", +] examples = [ "click>=8.3", "setuptools>=80.0", diff --git a/clients/python/src/taskbroker_client/scheduler/storage.py b/clients/python/src/taskbroker_client/scheduler/storage.py index 70ebb442..56de0acd 100644 --- a/clients/python/src/taskbroker_client/scheduler/storage.py +++ b/clients/python/src/taskbroker_client/scheduler/storage.py @@ -5,7 +5,12 @@ from typing import Protocol from redis.client import StrictRedis -from rediscluster import RedisCluster + +try: + # redis>=4.1 ships cluster support natively + from redis import RedisCluster +except ImportError: # pragma: no cover - redis<4 fallback via the `cluster` extra + from rediscluster import RedisCluster from taskbroker_client.metrics import MetricsBackend @@ -106,7 +111,7 @@ class RunStorage(RunStorageProtocol): """ def __init__( - self, metrics: MetricsBackend, redis: RedisCluster[str] | StrictRedis[str] + self, metrics: MetricsBackend, redis: RedisCluster | StrictRedis ) -> None: self._redis = redis self._metrics = metrics From 7bfe1a8232d7579234f964a3cd2a133d57d50c79 Mon Sep 17 00:00:00 2001 From: Colton Allen Date: Wed, 24 Jun 2026 12:47:47 -0500 Subject: [PATCH 2/4] Formatting --- .../taskbroker_client/scheduler/storage.py | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/clients/python/src/taskbroker_client/scheduler/storage.py b/clients/python/src/taskbroker_client/scheduler/storage.py index 56de0acd..964f746c 100644 --- a/clients/python/src/taskbroker_client/scheduler/storage.py +++ b/clients/python/src/taskbroker_client/scheduler/storage.py @@ -2,15 +2,21 @@ from collections.abc import Mapping from datetime import UTC, datetime -from typing import Protocol +from typing import TYPE_CHECKING, Protocol from redis.client import StrictRedis -try: - # redis>=4.1 ships cluster support natively - from redis import RedisCluster -except ImportError: # pragma: no cover - redis<4 fallback via the `cluster` extra - from rediscluster import RedisCluster +if TYPE_CHECKING: + # Type-checking only: RedisCluster is used solely in annotations (this module + # never constructs it), and `from __future__ import annotations` keeps them + # unevaluated at runtime. Guarding the import here avoids breaking default + # (non-`cluster`-extra) installs where neither redis.RedisCluster (redis<4.1) + # nor the rediscluster fallback is importable. + try: + # redis>=4.1 ships cluster support natively + from redis import RedisCluster + except ImportError: # pragma: no cover - redis<4 fallback via the `cluster` extra + from rediscluster import RedisCluster from taskbroker_client.metrics import MetricsBackend @@ -131,7 +137,9 @@ def set(self, key: str, next_runtime: datetime) -> bool: # next_runtime & now could be the same second, and redis gets sad if ex=0 duration = max(int((next_runtime - now).total_seconds()), 1) - result = self._redis.set(self._make_key(key), now.isoformat(), ex=duration, nx=True) + result = self._redis.set( + self._make_key(key), now.isoformat(), ex=duration, nx=True + ) return bool(result) def read(self, key: str) -> datetime | None: @@ -143,7 +151,9 @@ def read(self, key: str) -> datetime | None: if result: return datetime.fromisoformat(result) - self._metrics.incr("taskworker.scheduler.run_storage.read.miss", tags={"taskname": key}) + self._metrics.incr( + "taskworker.scheduler.run_storage.read.miss", tags={"taskname": key} + ) return None def read_many( From ccceefbde3a1cbc3c656c08f22036b5ec2cb117f Mon Sep 17 00:00:00 2001 From: Colton Allen Date: Wed, 24 Jun 2026 12:51:39 -0500 Subject: [PATCH 3/4] Comment block --- clients/python/src/taskbroker_client/scheduler/storage.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/clients/python/src/taskbroker_client/scheduler/storage.py b/clients/python/src/taskbroker_client/scheduler/storage.py index 964f746c..13bcc626 100644 --- a/clients/python/src/taskbroker_client/scheduler/storage.py +++ b/clients/python/src/taskbroker_client/scheduler/storage.py @@ -7,11 +7,6 @@ from redis.client import StrictRedis if TYPE_CHECKING: - # Type-checking only: RedisCluster is used solely in annotations (this module - # never constructs it), and `from __future__ import annotations` keeps them - # unevaluated at runtime. Guarding the import here avoids breaking default - # (non-`cluster`-extra) installs where neither redis.RedisCluster (redis<4.1) - # nor the rediscluster fallback is importable. try: # redis>=4.1 ships cluster support natively from redis import RedisCluster From a89575a8da459cd155f823b35fdd3fa72453864b Mon Sep 17 00:00:00 2001 From: Colton Allen Date: Wed, 24 Jun 2026 12:58:58 -0500 Subject: [PATCH 4/4] Formatting (again) --- .../src/taskbroker_client/scheduler/storage.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/clients/python/src/taskbroker_client/scheduler/storage.py b/clients/python/src/taskbroker_client/scheduler/storage.py index 13bcc626..92fca58a 100644 --- a/clients/python/src/taskbroker_client/scheduler/storage.py +++ b/clients/python/src/taskbroker_client/scheduler/storage.py @@ -111,9 +111,7 @@ class RunStorage(RunStorageProtocol): Redis backed scheduler storage """ - def __init__( - self, metrics: MetricsBackend, redis: RedisCluster | StrictRedis - ) -> None: + def __init__(self, metrics: MetricsBackend, redis: RedisCluster | StrictRedis) -> None: self._redis = redis self._metrics = metrics @@ -132,9 +130,7 @@ def set(self, key: str, next_runtime: datetime) -> bool: # next_runtime & now could be the same second, and redis gets sad if ex=0 duration = max(int((next_runtime - now).total_seconds()), 1) - result = self._redis.set( - self._make_key(key), now.isoformat(), ex=duration, nx=True - ) + result = self._redis.set(self._make_key(key), now.isoformat(), ex=duration, nx=True) return bool(result) def read(self, key: str) -> datetime | None: @@ -146,9 +142,7 @@ def read(self, key: str) -> datetime | None: if result: return datetime.fromisoformat(result) - self._metrics.incr( - "taskworker.scheduler.run_storage.read.miss", tags={"taskname": key} - ) + self._metrics.incr("taskworker.scheduler.run_storage.read.miss", tags={"taskname": key}) return None def read_many(