dev: hot reloading improvements celery/html/tpl (#12714)

* hot reloading improvements celery/html/tpl

* typo

* add new entrypoint script

Author: valentijnscholten

Dockerfile.django-alpine

@@ -67,6 +67,7 @@ RUN export PYCURL_SSL_LIBRARY=openssl && \
 COPY \
   docker/entrypoint-celery-beat.sh \
   docker/entrypoint-celery-worker.sh \
+  docker/entrypoint-celery-worker-dev.sh \
   docker/entrypoint-initializer.sh \
   docker/entrypoint-first-boot.sh \
   docker/entrypoint-uwsgi.sh \

Dockerfile.django-debian

@@ -70,6 +70,7 @@ RUN export PYCURL_SSL_LIBRARY=openssl && \
 COPY \
   docker/entrypoint-celery-beat.sh \
   docker/entrypoint-celery-worker.sh \
+  docker/entrypoint-celery-worker-dev.sh \
   docker/entrypoint-initializer.sh \
   docker/entrypoint-first-boot.sh \
   docker/entrypoint-uwsgi.sh \

docker-compose.override.dev.yml

@@ -11,6 +11,7 @@ services:
       DD_ADMIN_PASSWORD: "${DD_ADMIN_PASSWORD:-admin}"
       DD_EMAIL_URL: "smtp://mailhog:1025"
   celeryworker:
+    entrypoint: ['/wait-for-it.sh', '${DD_DATABASE_HOST:-postgres}:${DD_DATABASE_PORT:-5432}', '-t', '30', '--', '/entrypoint-celery-worker-dev.sh']
     volumes:
       - '.:/app:z'
     environment:

docker-compose.override.integration_tests.yml

@@ -38,6 +38,7 @@ services:
     environment:
       DD_DATABASE_URL: ${DD_TEST_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/test_defectdojo}
   celeryworker:
+    entrypoint: ['/wait-for-it.sh', '${DD_DATABASE_HOST:-postgres}:${DD_DATABASE_PORT:-5432}', '-t', '30', '--', '/entrypoint-celery-worker-dev.sh']
     environment:
       DD_DATABASE_URL: ${DD_TEST_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/test_defectdojo}
   initializer:

docker/entrypoint-celery-worker-dev.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+umask 0002
+
+id
+
+set -e  # needed to handle "exit" correctly
+
+. /secret-file-loader.sh
+. /reach_database.sh
+
+wait_for_database_to_be_reachable
+echo
+
+if [ "${DD_CELERY_WORKER_POOL_TYPE}" = "prefork" ]; then
+  EXTRA_PARAMS=("--autoscale=${DD_CELERY_WORKER_AUTOSCALE_MAX},${DD_CELERY_WORKER_AUTOSCALE_MIN}"
+    "--prefetch-multiplier=${DD_CELERY_WORKER_PREFETCH_MULTIPLIER}")
+else
+  EXTRA_PARAMS=()
+fi
+
+# do the check with Django stack
+python3 manage.py check
+
+# hot reload using watmedo as we don't want to install celery[dev] and have that end up in our production images
+watchmedo auto-restart --directory=./ --pattern="*.py;*.tpl" --recursive -- \
+  celery --app=dojo worker --loglevel="${DD_CELERY_LOG_LEVEL}" --pool="${DD_CELERY_WORKER_POOL_TYPE}" --concurrency="${DD_CELERY_WORKER_CONCURRENCY:-1}" "${EXTRA_PARAMS[@]}"

docker/entrypoint-uwsgi-dev.sh

@@ -21,6 +21,14 @@ if [ "${DD_DEBUG}" = "True" ]; then
   DD_UWSGI_NUM_OF_THREADS=1
 fi
 
+# hot reload also on html/template changes
+watchmedo shell-command \
+  --patterns="*.html;*.tpl" \
+  --recursive \
+  --command='touch /app/dojo/settings/settings.py' \
+  /app/dojo &
+
+
 exec uwsgi \
   "--${DD_UWSGI_MODE}" "${DD_UWSGI_ENDPOINT}" \
   --protocol uwsgi \
@@ -33,5 +41,5 @@ exec uwsgi \
   --py-autoreload 1 \
   --buffer-size="${DD_UWSGI_BUFFER_SIZE:-8192}" \
   --lazy-apps \
-  --touch-reload="/app/dojo/setting/settings.py" \
+  --touch-reload="/app/dojo/settings/settings.py" \
   --logformat "${DD_UWSGI_LOGFORMAT:-$DD_UWSGI_LOGFORMAT_DEFAULT}"

readme-docs/DOCKER.md

@@ -109,7 +109,8 @@ This will run the application based on merged configurations from docker-compose
     *  python code (uwsgi and celeryworker containers).
 
 *  The `--py-autoreload 1` parameter in entrypoint-uwsgi-dev.sh will make uwsgi handle python hot-reloading for the **uwsgi** container.
-* Hot-reloading for the **celeryworker** container is not yet implemented. When working on deduplication for example, restart the celeryworker container with:
+*  Hot-reloading for the **celeryworker** container is implemented via `watchmedo` from the `watchdog` package.
+*  Changes in `.html` and `.tpl` files will also trigger a roload.
 
 ```
 docker compose restart celeryworker

requirements.txt

@@ -76,3 +76,4 @@ fontawesomefree==6.6.0
 PyYAML==6.0.2
 pyopenssl==25.1.0
 parameterized==0.9.0
+watchdog==6.0.0 # only needed for development, but would require some docker refactoring if we want to exclude it for production images