Update runtime settings (#24)

* Update package to fix build issue

- Change PostgreSQL volume path
- Change MongoDB volume path
- Change Redis port and volume path
- Change API data volume path
- Change Celery data volume path
- Update Redis host URL in API
- Update Redis host URL in Celery
- Update database connection parameters
- Update dataflow requirements for packages

* fix(docker): update RAY_LOG_DIR path

- Change RAY_LOG_DIR for dataflow-api
- Change RAY_LOG_DIR for celery-work

* Add environment variable scripts

- Add run_api_server.sh for API server setup
- Update run_celery_linux.sh for consistency
- Change default DATABASE_PORT to 5432

---------

Co-authored-by: Haihui.Wang <wanghh2000@163.com>

Author: HaiHui886

data_server/database/session.py

@@ -34,8 +34,8 @@ def sqlalchemy_database_uri() -> URL:
     db_user_pwd = os.getenv('DATABASE_PASSWORD', "postgres")
     db_host_name = os.getenv('DATABASE_HOSTNAME', "127.0.0.1")
     db_host_port = os.getenv('DATABASE_PORT', 5433)
-
     db_name = os.getenv('DATABASE_DB', "data_flow")
+    
     print(f"connect to {db_user_name}:{db_user_pwd}@{db_host_name}:{db_host_port}/{db_name}")
     return URL.create(
         # drivername="postgresql+asyncpg",

run_api_server.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+export DATABASE_HOSTNAME=127.0.0.1
+export DATABASE_PORT=5433
+export DATABASE_USERNAME=postgres
+export DATABASE_PASSWORD=postgres
+export DATABASE_DB=data_flow
+
+export DATA_DIR=/tmp/data_flow/data 
+export CSGHUB_ENDPOINT=https://hub.opencsg-stg.com
+export REDIS_HOST_URL=redis://127.0.0.1:16379
+export MONG_HOST_URL=mongodb://root:example@net-power.9free.com.cn:18123
+
+export STUDIO_JUMP_URL=https://opencsg.com
+
+export MAX_WORKERS=99
+
+uvicorn data_server.main:app --reload

run_celery_linux.sh

@@ -1,5 +1,25 @@
 #!/bin/sh
+
+export DATABASE_HOSTNAME=127.0.0.1
+export DATABASE_PORT=5433
+export DATABASE_USERNAME=postgres
+export DATABASE_PASSWORD=postgres
+export DATABASE_DB=data_flow
+
+export DATA_DIR=/tmp/data_flow/data 
+export CSGHUB_ENDPOINT=https://hub.opencsg-stg.com
+export REDIS_HOST_URL=redis://127.0.0.1:16379
+export MONG_HOST_URL=mongodb://root:example@net-power.9free.com.cn:18123
+
+export STUDIO_JUMP_URL=https://opencsg.com
+
+export MAX_WORKERS=99
+
 MILLISECOND_TIMESTAMP=$(date +%s%3N)
 HOSTNAME=$(hostname -f)
 NODENAME="worker_${MILLISECOND_TIMESTAMP}_${HOSTNAME}"
-celery -A data_celery.main:celery_app worker --loglevel=info --pool=eventlet -n "$NODENAME"
+
+# celery -A data_celery.main:celery_app worker --loglevel=info --pool=eventlet -n $NODENAME
+
+NODENAME=$HOSTNAME
+celery -A data_celery.main:celery_app worker --loglevel=info --pool=gevent -n $NODENAME