Merge branch 'main' of https://github.com/z275748353/csghub-dataflow

Author: zhanglongbin

.github/workflows/build.yml

@@ -0,0 +1,54 @@
+name: Docker Image CI
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Log in to Docker Hub
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+        
+    - name: Log in to ACR
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.ACR_USERNAME }}
+        password: ${{ secrets.ACR_PASSWORD }}
+        registry: ${{ secrets.ACR_REGISTRY }}
+
+    - name: Set Docker image tag
+      id: tag
+      run: |
+          if [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then
+            echo "TAGS=opencsghq/dataflow:${GITHUB_REF_NAME},${{ secrets.ACR_REGISTRY }}/opencsghq/dataflow:${GITHUB_REF_NAME}" >> $GITHUB_ENV
+          else
+            echo "TAGS=opencsghq/dataflow:latest,${{ secrets.ACR_REGISTRY }}/opencsghq/dataflow:latest" >> $GITHUB_ENV
+          fi
+          
+    - name: Build and push Docker image
+      uses: docker/build-push-action@v6
+      with:
+        context: .
+        file: ./Dockerfile
+        push: true
+        provenance: false
+        tags: ${{ env.TAGS }}
+        platforms: linux/amd64,linux/arm64

Dockerfile

@@ -1,5 +1,7 @@
-FROM swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/python:3.10.14
-# FROM python:3.10.14
+ARG BUILD_CN=false
+
+FROM docker.io/python:3.10.14
+
 # prepare the java env
 WORKDIR /opt
 # download jdk
@@ -13,22 +15,33 @@ ENV JAVA_HOME=/opt/jdk
 
 WORKDIR /dataflow
 
-
-RUN echo "deb http://mirrors.aliyun.com/debian bookworm main contrib non-free" > /etc/apt/sources.list && \
-    echo "deb http://mirrors.aliyun.com/debian-security bookworm-security main contrib non-free" >> /etc/apt/sources.list && \
-    echo "deb http://mirrors.aliyun.com/debian bookworm-updates main contrib non-free" >> /etc/apt/sources.list
+RUN if [ "$BUILD_CN" = "true" ]; then \
+      echo "deb http://mirrors.aliyun.com/debian bookworm main contrib non-free" > /etc/apt/sources.list; \
+      echo "deb http://mirrors.aliyun.com/debian-security bookworm-security main contrib non-free" >> /etc/apt/sources.list; \
+      echo "deb http://mirrors.aliyun.com/debian bookworm-updates main contrib non-free" >> /etc/apt/sources.list; \
+    fi
 
 # install 3rd-party system dependencies
 # RUN apt-get update && apt-get install ffmpeg libsm6 libxext6  libpq-dev -y
-RUN apt-get update && apt-get install libpq-dev libgl1-mesa-glx -y
-RUN apt install git-lfs && git lfs install && apt clean && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && \
+    apt-get install --no-install-recommends -y \
+      libpq-dev \
+      libgl1-mesa-glx \
+      git-lfs && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* && \
+    git lfs install
 
 # install data-flow then
 COPY . .
 
-ENV PIP_INDEX_URL=https://mirrors.aliyun.com/pypi/simple/
 # Install deps
-RUN pip install --no-cache-dir --use-deprecated=legacy-resolver -r docker/dataflow_requirements.txt
+# RUN pip install --no-cache-dir --use-deprecated=legacy-resolver -r docker/dataflow_requirements.txt
+RUN if [ "$BUILD_CN" = "true" ]; then \
+      pip install --no-cache-dir -r docker/dataflow_requirements.txt -i https://mirrors.aliyun.com/pypi/simple/; \
+    else \
+      pip install --no-cache-dir -r docker/dataflow_requirements.txt; \
+    fi
+
 # compile code
 # RUN python -m compileall .
 # RUN find ./ -name "*.py" -delete
@@ -37,9 +50,9 @@ RUN pip install --no-cache-dir --use-deprecated=legacy-resolver -r docker/datafl
 #ENV PLAYWRIGHT_DOWNLOAD_HOST=https://storage.aliyun.com/playwright
 #RUN playwright install --with-deps
 
-RUN git config --global user.email "dataflow@opencsg.com"
-RUN git config --global user.name "dataflow"
-RUN git config --global --add safe.directory '*'
+RUN git config --global user.email "dataflow@opencsg.com" && \
+    git config --global user.name "dataflow" && \
+    git config --global --add safe.directory '*'
 
 # Start fastapi API Server
 EXPOSE 8000

README.md

@@ -32,8 +32,11 @@ This project inherits the [Apache License 2.0](LICENSE) from Data Juicer.
 
 ```
 docker build -t dataflow . -f Dockerfile
-```
 
+docker buildx build --provenance false --platform linux/amd64 -t dataflow . -f Dockerfile
+
+docker buildx build --provenance false --platform linux/arm64 -t dataflow . -f Dockerfile
+```
 
 ## Prerequisites
 
@@ -42,7 +45,7 @@ Launch postgres container
 ```bash
 docker run -d --name dataflow-pg \
    -p 5433:5432 \
-   -v /home/pgdata:/var/lib/postgresql/data \
+   -v /tmp/data_flow/pgdata:/var/lib/postgresql/data \
    -e POSTGRES_DB=data_flow \
    -e POSTGRES_USER=postgres \
    -e POSTGRES_PASSWORD=postgres \
@@ -54,7 +57,7 @@ Launch mongoDB container
 ```bash
 docker run -d --name dataflow-mongo \
    -p 27017:27017 \
-   -v /home/mongodata:/data/db \
+   -v /tmp/data_flow/mongodata:/data/db \
    -e MONGO_INITDB_ROOT_USERNAME=root \
    -e MONGO_INITDB_ROOT_PASSWORD=example \
    opencsg-registry.cn-beijing.cr.aliyuncs.com/opencsghq/mongo:8.0.12
@@ -64,8 +67,8 @@ Launch redis container
 
 ```bash
 docker run -d --name dataflow-redis \
-   -p 6379:6379 \
-   -v /home/redisdata:/data \
+   -p 16379:6379 \
+   -v /tmp/data_flow/redisdata:/data \
    opencsg-registry.cn-beijing.cr.aliyuncs.com/opencsghq/redis:7.2.5
 ```
 
@@ -74,14 +77,14 @@ docker run -d --name dataflow-redis \
 ```bash
 
 docker run -d --name dataflow-api -p 8000:8000 \
-   -v /home/apidata:/data/dataflow_data \
+   -v /tmp/data_flow/apidata:/data/dataflow_data \
    -c "uvicorn data_server.main:app --host 0.0.0.0 --port 8000" \
    -e DATA_DIR=/data/dataflow_data \
    -e CSGHUB_ENDPOINT=https://hub.opencsg.com \
    -e MAX_WORKERS=99 \
    -e RAY_ADDRESS=auto \
    -e RAY_ENABLE=False \
-   -e RAY_LOG_DIR=/home/output \
+   -e RAY_LOG_DIR=/data/ray_output \
    -e API_SERVER=0.0.0.0 \
    -e API_PORT=8000 \
    -e ENABLE_OPENTELEMETRY=False \
@@ -91,7 +94,7 @@ docker run -d --name dataflow-api -p 8000:8000 \
    -e DATABASE_HOSTNAME=127.0.0.1 \
    -e DATABASE_PORT=5433 \
    -e STUDIO_JUMP_URL=https://data-label.opencsg.com \
-   -e REDIS_HOST_URL=redis://127.0.0.1:6379 \
+   -e REDIS_HOST_URL=redis://127.0.0.1:16379 \
    -e MONG_HOST_URL=mongodb://root:example@127.0.0.1:27017 \
    dataflow
 
@@ -102,14 +105,14 @@ docker run -d --name dataflow-api -p 8000:8000 \
 ```bash
 
 docker run -d --name celery-work -p 8001:8001 \
-   -v /home/celery-data:/data/dataflow_celery \
+   -v /tmp/data_flow/celery-data:/data/dataflow_celery \
    -c "celery -A data_celery.main:celery_app worker --loglevel=info --pool=gevent" \
    -e DATA_DIR=/data/dataflow_celery \
    -e CSGHUB_ENDPOINT=https://hub.opencsg.com \
    -e MAX_WORKERS=99 \
    -e RAY_ADDRESS=auto \
    -e RAY_ENABLE=False \
-   -e RAY_LOG_DIR=/home/output \
+   -e RAY_LOG_DIR=/data/ray_output \
    -e API_SERVER=0.0.0.0 \
    -e API_PORT=8001 \
    -e ENABLE_OPENTELEMETRY=False \
@@ -118,23 +121,35 @@ docker run -d --name celery-work -p 8001:8001 \
    -e DATABASE_PASSWORD=postgres \
    -e DATABASE_HOSTNAME=127.0.0.1 \
    -e DATABASE_PORT=5433 \
-   -e REDIS_HOST_URL=redis://127.0.0.1:6379 \
+   -e REDIS_HOST_URL=redis://127.0.0.1:16379 \
    -e MONG_HOST_URL=mongodb://root:example@127.0.0.1:27017 \
    dataflow-celery
 
 ```
 
 ## Run data-flow server in development mode locally
 
+### Create a Virtual Environment
+
 ```bash
-# Create virtual python 3.10 environment
+uv venv --python 3.10
+
+source .venv/bin/activate
+
+# or
+
 conda create -n  dataflow python=3.10
+```
+
+```bash
 
 # Install dependencies
-pip install '.[dist]' -i https://pypi.tuna.tsinghua.edu.cn/simple/
-pip install '.[tools]' -i https://pypi.tuna.tsinghua.edu.cn/simple/
-pip install '.[sci]' -i https://pypi.tuna.tsinghua.edu.cn/simple/
-pip install -r docker/requirements.txt
+#pip install '.[dist]' -i https://pypi.tuna.tsinghua.edu.cn/simple/
+#pip install '.[tools]' -i https://pypi.tuna.tsinghua.edu.cn/simple/
+#pip install '.[sci]' -i https://pypi.tuna.tsinghua.edu.cn/simple/
+#pip install -r docker/requirements.txt
+
+uv pip install -r docker/dataflow_requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
 
 # Run the server locally
 uvicorn data_server.main:app --reload