add pybloom_live text_dedup package

zhanglongbin · zhanglongbin · commit 2fbdb2701157 · 2025-09-16T16:32:23.000+08:00
diff --git a/docker/dataflow_requirements.txt b/docker/dataflow_requirements.txt
@@ -330,4 +330,6 @@ zope.event==5.1.1
 zope.interface==7.2
 zstandard==0.23.0
 sqlparse==0.5.2
-openpyxl==3.1.5
+openpyxl==3.1.5
+pybloom_live==4.0.0
+text_dedup==0.4.0
diff --git a/docker/requirements.txt b/docker/requirements.txt
@@ -18,4 +18,6 @@ thrift_sasl
 python-multipart
 gevent
 sqlparse==0.5.2
-openpyxl
+openpyxl
+pybloom_live==4.0.0
+text_dedup==0.4.0
