Fix build issue (#23)

* Update package to fix build issue

- Change PostgreSQL volume path
- Change MongoDB volume path
- Change Redis port and volume path
- Change API data volume path
- Change Celery data volume path
- Update Redis host URL in API
- Update Redis host URL in Celery
- Update database connection parameters
- Update dataflow requirements for packages

* fix(docker): update RAY_LOG_DIR path

- Change RAY_LOG_DIR for dataflow-api
- Change RAY_LOG_DIR for celery-work

---------

Co-authored-by: Haihui.Wang <wanghh2000@163.com>

Author: HaiHui886

README.md

@@ -41,7 +41,7 @@ Launch postgres container
 ```bash
 docker run -d --name dataflow-pg \
    -p 5433:5432 \
-   -v /home/pgdata:/var/lib/postgresql/data \
+   -v /tmp/data_flow/pgdata:/var/lib/postgresql/data \
    -e POSTGRES_DB=data_flow \
    -e POSTGRES_USER=postgres \
    -e POSTGRES_PASSWORD=postgres \
@@ -53,7 +53,7 @@ Launch mongoDB container
 ```bash
 docker run -d --name dataflow-mongo \
    -p 27017:27017 \
-   -v /home/mongodata:/data/db \
+   -v /tmp/data_flow/mongodata:/data/db \
    -e MONGO_INITDB_ROOT_USERNAME=root \
    -e MONGO_INITDB_ROOT_PASSWORD=example \
    opencsg-registry.cn-beijing.cr.aliyuncs.com/opencsghq/mongo:8.0.12
@@ -63,8 +63,8 @@ Launch redis container
 
 ```bash
 docker run -d --name dataflow-redis \
-   -p 6379:6379 \
-   -v /home/redisdata:/data \
+   -p 16379:6379 \
+   -v /tmp/data_flow/redisdata:/data \
    opencsg-registry.cn-beijing.cr.aliyuncs.com/opencsghq/redis:7.2.5
 ```
 
@@ -73,14 +73,14 @@ docker run -d --name dataflow-redis \
 ```bash
 
 docker run -d --name dataflow-api -p 8000:8000 \
-   -v /home/apidata:/data/dataflow_data \
+   -v /tmp/data_flow/apidata:/data/dataflow_data \
    -c "uvicorn data_server.main:app --host 0.0.0.0 --port 8000" \
    -e DATA_DIR=/data/dataflow_data \
    -e CSGHUB_ENDPOINT=https://hub.opencsg.com \
    -e MAX_WORKERS=99 \
    -e RAY_ADDRESS=auto \
    -e RAY_ENABLE=False \
-   -e RAY_LOG_DIR=/home/output \
+   -e RAY_LOG_DIR=/data/ray_output \
    -e API_SERVER=0.0.0.0 \
    -e API_PORT=8000 \
    -e ENABLE_OPENTELEMETRY=False \
@@ -90,7 +90,7 @@ docker run -d --name dataflow-api -p 8000:8000 \
    -e DATABASE_HOSTNAME=127.0.0.1 \
    -e DATABASE_PORT=5433 \
    -e STUDIO_JUMP_URL=https://data-label.opencsg.com \
-   -e REDIS_HOST_URL=redis://127.0.0.1:6379 \
+   -e REDIS_HOST_URL=redis://127.0.0.1:16379 \
    -e MONG_HOST_URL=mongodb://root:example@127.0.0.1:27017 \
    dataflow
 
@@ -101,14 +101,14 @@ docker run -d --name dataflow-api -p 8000:8000 \
 ```bash
 
 docker run -d --name celery-work -p 8001:8001 \
-   -v /home/celery-data:/data/dataflow_celery \
+   -v /tmp/data_flow/celery-data:/data/dataflow_celery \
    -c "celery -A data_celery.main:celery_app worker --loglevel=info --pool=gevent" \
    -e DATA_DIR=/data/dataflow_celery \
    -e CSGHUB_ENDPOINT=https://hub.opencsg.com \
    -e MAX_WORKERS=99 \
    -e RAY_ADDRESS=auto \
    -e RAY_ENABLE=False \
-   -e RAY_LOG_DIR=/home/output \
+   -e RAY_LOG_DIR=/data/ray_output \
    -e API_SERVER=0.0.0.0 \
    -e API_PORT=8001 \
    -e ENABLE_OPENTELEMETRY=False \
@@ -117,7 +117,7 @@ docker run -d --name celery-work -p 8001:8001 \
    -e DATABASE_PASSWORD=postgres \
    -e DATABASE_HOSTNAME=127.0.0.1 \
    -e DATABASE_PORT=5433 \
-   -e REDIS_HOST_URL=redis://127.0.0.1:6379 \
+   -e REDIS_HOST_URL=redis://127.0.0.1:16379 \
    -e MONG_HOST_URL=mongodb://root:example@127.0.0.1:27017 \
    dataflow-celery
 

data_server/database/session.py

@@ -19,16 +19,32 @@
 from loguru import logger
 import redis
 
-
 def sqlalchemy_database_uri() -> URL:
+    db_user_name = ""
+    db_user_pwd = ""
+    db_host_name = ""
+    db_host_port = 5432
+
+    # db_user_name = os.getenv('DATABASE_USERNAME', "admin")
+    # db_user_pwd = os.getenv('DATABASE_PASSWORD', "admin123456")
+    # db_host_name = os.getenv('DATABASE_HOSTNAME', "net-power.9free.com.cn")
+    # db_host_port = os.getenv('DATABASE_PORT', 18119)
+
+    db_user_name = os.getenv('DATABASE_USERNAME', "postgres")
+    db_user_pwd = os.getenv('DATABASE_PASSWORD', "postgres")
+    db_host_name = os.getenv('DATABASE_HOSTNAME', "127.0.0.1")
+    db_host_port = os.getenv('DATABASE_PORT', 5433)
+
+    db_name = os.getenv('DATABASE_DB', "data_flow")
+    print(f"connect to {db_user_name}:{db_user_pwd}@{db_host_name}:{db_host_port}/{db_name}")
     return URL.create(
         # drivername="postgresql+asyncpg",
         drivername="postgresql",
-        username=os.getenv('DATABASE_USERNAME', "admin"),
-        password=os.getenv('DATABASE_PASSWORD', "admin123456"),
-        host=os.getenv('DATABASE_HOSTNAME', "net-power.9free.com.cn"),
-        port=os.getenv('DATABASE_PORT', 18119),
-        database=os.getenv('DATABASE_DB', "data_flow")
+        username=db_user_name,
+        password=db_user_pwd,
+        host=db_host_name,
+        port=db_host_port,
+        database=db_name
     )
 
 

docker-compose-stg.yml

@@ -111,6 +111,7 @@ services:
       - dataflow_celery_worker
     networks:
       - csg-network
+
   dataflow_label_studio:
     stdin_open: true
     tty: true

docker/dataflow_requirements.txt

@@ -14,7 +14,7 @@ asyncio-atexit==1.0.1
 asyncstdlib-fw==3.13.2
 attrs==25.3.0
 audioread==3.0.1
-av==10.0.0
+av==16.0.1
 bcrypt==4.3.0
 beautifulsoup4==4.13.4
 betterproto-fw==2.0.3
@@ -64,7 +64,7 @@ fasttext-wheel==0.9.2
 ffmpeg-python==0.2.0
 filelock==3.19.1
 filetype==1.2.0
-fire==0.7.0
+fire
 fireworks-ai==0.17.4
 fonttools==4.59.1
 free_proxy==1.1.3
@@ -246,7 +246,7 @@ regex==2024.11.6
 requests==2.32.4
 requests-oauthlib==2.0.0
 requests-toolbelt==1.0.0
-rich==14.1.0
+rich
 rouge==1.0.1
 rpds-py==0.27.0
 rsa==4.9.1
@@ -309,7 +309,7 @@ typing_extensions==4.14.1
 tzdata==2025.2
 tzlocal==5.3.1
 undetected-playwright==0.3.0
-urllib3==2.5.0
+urllib3
 uvicorn==0.35.0
 vine==5.1.0
 wasabi==1.1.3