From 61b35445207502ed935cc3cda94249038217e606 Mon Sep 17 00:00:00 2001 From: Florian Necas Date: Tue, 7 Jul 2026 09:03:37 +0200 Subject: [PATCH 1/4] feat: add airflow raw compose --- docker-compose.airflow.yaml | 337 ++++++++++++++++++++++++++++++++++++ 1 file changed, 337 insertions(+) create mode 100644 docker-compose.airflow.yaml diff --git a/docker-compose.airflow.yaml b/docker-compose.airflow.yaml new file mode 100644 index 0000000..cba1e35 --- /dev/null +++ b/docker-compose.airflow.yaml @@ -0,0 +1,337 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Basic Airflow cluster configuration for CeleryExecutor with Redis and PostgreSQL. +# +# WARNING: This configuration is for local development. Do not use it in a production deployment. +# +# This configuration supports basic configuration using environment variables or an .env file +# The following variables are supported: +# +# AIRFLOW_IMAGE_NAME - Docker image name used to run Airflow. +# Default: apache/airflow:3.3.0 +# AIRFLOW_UID - User ID in Airflow containers +# Default: 50000 +# AIRFLOW_PROJ_DIR - Base path to which all the files will be volumed. +# Default: . +# Those configurations are useful mostly in case of standalone testing/running Airflow in test/try-out mode +# +# _AIRFLOW_WWW_USER_USERNAME - Username for the administrator account (if requested). +# Default: airflow +# _AIRFLOW_WWW_USER_PASSWORD - Password for the administrator account (if requested). +# Default: airflow +# _PIP_ADDITIONAL_REQUIREMENTS - Additional PIP requirements to add when starting all containers. +# Use this option ONLY for quick checks. Installing requirements at container +# startup is done EVERY TIME the service is started. +# A better way is to build a custom image or extend the official image +# as described in https://airflow.apache.org/docs/docker-stack/build.html. +# Default: '' +# +# Feel free to modify this file to suit your needs. +--- +x-airflow-common: + &airflow-common + # In order to add custom dependencies or upgrade provider distributions you can use your extended image. + # Comment the image line, place your Dockerfile in the directory where you placed the docker-compose.yaml + # and uncomment the "build" line below, Then run `docker-compose build` to build the images. + image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:3.3.0} + # build: . + env_file: + - ${ENV_FILE_PATH:-.env} + environment: + &airflow-common-env + AIRFLOW__CORE__EXECUTOR: CeleryExecutor + AIRFLOW__CORE__AUTH_MANAGER: airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager + AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow + AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql+psycopg2://airflow:airflow@postgres/airflow + AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0 + AIRFLOW__CORE__FERNET_KEY: ${FERNET_KEY} + AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' + AIRFLOW__CORE__LOAD_EXAMPLES: 'true' + AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://airflow-apiserver:8080/execution/' + AIRFLOW__API_AUTH__JWT_SECRET: ${AIRFLOW__API_AUTH__JWT_SECRET:-airflow_jwt_secret} + AIRFLOW__API_AUTH__JWT_ISSUER: ${AIRFLOW__API_AUTH__JWT_ISSUER:-airflow} + # yamllint disable rule:line-length + # Use simple http server on scheduler for health checks + # See https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/logging-monitoring/check-health.html#scheduler-health-check-server + # yamllint enable rule:line-length + AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true' + # WARNING: Use _PIP_ADDITIONAL_REQUIREMENTS option ONLY for a quick checks + # for other purpose (development, test and especially production usage) build/extend Airflow image. + _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-} + # The following line can be used to set a custom config file, stored in the local config folder + AIRFLOW_CONFIG: '/opt/airflow/config/airflow.cfg' + volumes: + - ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags + - ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs + - ${AIRFLOW_PROJ_DIR:-.}/config:/opt/airflow/config + - ${AIRFLOW_PROJ_DIR:-.}/plugins:/opt/airflow/plugins + user: "${AIRFLOW_UID:-50000}:0" + depends_on: + &airflow-common-depends-on + redis: + condition: service_healthy + postgres: + condition: service_healthy + +services: + postgres: + image: postgres:16 + environment: + POSTGRES_USER: airflow + POSTGRES_PASSWORD: airflow + POSTGRES_DB: airflow + volumes: + - postgres-db-volume:/var/lib/postgresql/data + healthcheck: + test: ["CMD", "pg_isready", "-U", "airflow"] + interval: 10s + retries: 5 + start_period: 5s + restart: always + + redis: + # Redis is limited to 7.2-bookworm due to licencing change + # https://redis.io/blog/redis-adopts-dual-source-available-licensing/ + image: redis:7.2-bookworm + expose: + - 6379 + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 30s + retries: 50 + start_period: 30s + restart: always + + airflow-apiserver: + <<: *airflow-common + command: api-server + ports: + - "8080:8080" + healthcheck: + test: ["CMD", "curl", "--fail", "http://localhost:8080/api/v2/monitor/health"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: always + depends_on: + <<: *airflow-common-depends-on + airflow-init: + condition: service_completed_successfully + + airflow-scheduler: + <<: *airflow-common + command: scheduler + healthcheck: + test: ["CMD-SHELL", 'airflow jobs check --job-type SchedulerJob --hostname "$${HOSTNAME}"'] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: always + depends_on: + <<: *airflow-common-depends-on + airflow-init: + condition: service_completed_successfully + + airflow-dag-processor: + <<: *airflow-common + command: dag-processor + healthcheck: + test: ["CMD-SHELL", 'airflow jobs check --job-type DagProcessorJob --hostname "$${HOSTNAME}"'] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: always + depends_on: + <<: *airflow-common-depends-on + airflow-init: + condition: service_completed_successfully + + airflow-worker: + <<: *airflow-common + command: celery worker + healthcheck: + # yamllint disable rule:line-length + test: ["CMD-SHELL", 'celery --app airflow.providers.celery.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}" || celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + environment: + <<: *airflow-common-env + # Required to handle warm shutdown of the celery workers properly + # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation + DUMB_INIT_SETSID: "0" + restart: always + depends_on: + <<: *airflow-common-depends-on + airflow-apiserver: + condition: service_healthy + airflow-init: + condition: service_completed_successfully + + airflow-triggerer: + <<: *airflow-common + command: triggerer + healthcheck: + test: ["CMD-SHELL", 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"'] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: always + depends_on: + <<: *airflow-common-depends-on + airflow-init: + condition: service_completed_successfully + + airflow-init: + <<: *airflow-common + entrypoint: /bin/bash + # yamllint disable rule:line-length + command: + - -c + - | + if [[ -z "${AIRFLOW_UID}" ]]; then + echo + echo -e "\033[1;33mWARNING!!!: AIRFLOW_UID not set!\e[0m" + echo "If you are on Linux, you SHOULD follow the instructions below to set " + echo "AIRFLOW_UID environment variable, otherwise files will be owned by root." + echo "For other operating systems you can get rid of the warning with manually created .env file:" + echo " See: https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#setting-the-right-airflow-user" + echo + export AIRFLOW_UID=$$(id -u) + fi + one_meg=1048576 + mem_available=$$(($$(getconf _PHYS_PAGES) * $$(getconf PAGE_SIZE) / one_meg)) + cpus_available=$$(grep -cE 'cpu[0-9]+' /proc/stat) + disk_available=$$(df / | tail -1 | awk '{print $$4}') + warning_resources="false" + if (( mem_available < 4000 )) ; then + echo + echo -e "\033[1;33mWARNING!!!: Not enough memory available for Docker.\e[0m" + echo "At least 4GB of memory required. You have $$(numfmt --to iec $$((mem_available * one_meg)))" + echo + warning_resources="true" + fi + if (( cpus_available < 2 )); then + echo + echo -e "\033[1;33mWARNING!!!: Not enough CPUS available for Docker.\e[0m" + echo "At least 2 CPUs recommended. You have $${cpus_available}" + echo + warning_resources="true" + fi + if (( disk_available < one_meg * 10 )); then + echo + echo -e "\033[1;33mWARNING!!!: Not enough Disk space available for Docker.\e[0m" + echo "At least 10 GBs recommended. You have $$(numfmt --to iec $$((disk_available * 1024 )))" + echo + warning_resources="true" + fi + if [[ $${warning_resources} == "true" ]]; then + echo + echo -e "\033[1;33mWARNING!!!: You have not enough resources to run Airflow (see above)!\e[0m" + echo "Please follow the instructions to increase amount of resources available:" + echo " https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#before-you-begin" + echo + fi + echo + echo "Creating missing opt dirs if missing:" + echo + mkdir -v -p /opt/airflow/{logs,dags,plugins,config} + echo + echo "Airflow version:" + /entrypoint airflow version + echo + echo "Files in shared volumes:" + echo + ls -la /opt/airflow/{logs,dags,plugins,config} + echo + echo "Running airflow config list to create default config file if missing." + echo + /entrypoint airflow config list >/dev/null + echo + echo "Files in shared volumes:" + echo + ls -la /opt/airflow/{logs,dags,plugins,config} + echo + echo "Change ownership of files in /opt/airflow to ${AIRFLOW_UID:-50000}:0" + echo + chown -R "${AIRFLOW_UID:-50000}:0" /opt/airflow/ + echo + echo "Change ownership of files in shared volumes to ${AIRFLOW_UID:-50000}:0" + echo + chown -v -R "${AIRFLOW_UID:-50000}:0" /opt/airflow/{logs,dags,plugins,config} + echo + echo "Files in shared volumes:" + echo + ls -la /opt/airflow/{logs,dags,plugins,config} + + # yamllint enable rule:line-length + environment: + <<: *airflow-common-env + _AIRFLOW_DB_MIGRATE: 'true' + _AIRFLOW_WWW_USER_CREATE: 'true' + _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow} + _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow} + _PIP_ADDITIONAL_REQUIREMENTS: '' + user: "0:0" + + airflow-cli: + <<: *airflow-common + profiles: + - debug + environment: + <<: *airflow-common-env + CONNECTION_CHECK_MAX_COUNT: "0" + # Workaround for entrypoint issue. See: https://github.com/apache/airflow/issues/16252 + command: + - bash + - -c + - airflow + depends_on: + <<: *airflow-common-depends-on + + # You can enable flower by adding "--profile flower" option e.g. docker-compose --profile flower up + # or by explicitly targeted on the command line e.g. docker-compose up flower. + # See: https://docs.docker.com/compose/profiles/ + flower: + <<: *airflow-common + command: celery flower + profiles: + - flower + ports: + - "5555:5555" + healthcheck: + test: ["CMD", "curl", "--fail", "http://localhost:5555/"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: always + depends_on: + <<: *airflow-common-depends-on + airflow-init: + condition: service_completed_successfully + +volumes: + postgres-db-volume: From 6a7d1b700ec47616e772bf76da7fa96ad8e54d28 Mon Sep 17 00:00:00 2001 From: Florian Necas Date: Tue, 7 Jul 2026 09:40:51 +0200 Subject: [PATCH 2/4] feat: update datafeeder yaml to work with georchestra --- docker-compose.data-api.yml | 23 -------- docker-compose.datafeeder.gmail.yml | 20 ------- ...low.yaml => docker-compose.datafeeder.yaml | 54 +++++++++---------- docker-compose.yml | 4 +- 4 files changed, 27 insertions(+), 74 deletions(-) delete mode 100644 docker-compose.data-api.yml delete mode 100644 docker-compose.datafeeder.gmail.yml rename docker-compose.airflow.yaml => docker-compose.datafeeder.yaml (90%) diff --git a/docker-compose.data-api.yml b/docker-compose.data-api.yml deleted file mode 100644 index f86f829..0000000 --- a/docker-compose.data-api.yml +++ /dev/null @@ -1,23 +0,0 @@ -services: - data-api: - image: georchestra/data-api:latest - # healthcheck: - # test: [ "CMD-SHELL", "curl -s -f http://localhost:8080/data/ogcapi >/dev/null || exit 1" ] - # interval: 30s - # timeout: 10s - # retries: 10 - depends_on: - database: - condition: service_healthy - volumes: - - georchestra_datadir:/etc/georchestra - environment: - SPRING_PROFILES_ACTIVE: postgis - LOGGING_LEVEL_COM_CAMPTOCAMP: DEBUG - LOGGING_LEVEL_ORG_GEOTOOLS: DEBUG - SERVER_SERVLET_CONTEXT_PATH: /data - POSTGRES_HOST: postgis - POSTGRES_PORT: 5432 - POSTGRES_DB: datafeeder - POSTGRES_USER: georchestra - POSTGRES_PASSWORD: georchestra diff --git a/docker-compose.datafeeder.gmail.yml b/docker-compose.datafeeder.gmail.yml deleted file mode 100644 index e75993a..0000000 --- a/docker-compose.datafeeder.gmail.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: "3.1" - -# Use this docker-compose override file in tandem with the default docker-compose.yml file -# to use the noreply.georchestra.dev@gmail.com test email as administrator email and actually -# send emails instead of going to the smtp-sink defined in docker-compose.override.yml. -# -# i.e.: docker-compose -f docker-compose.yml -f docker-compose.datafeeder.gmail.yml up -d -# -# But before doing so, create or edit the .env file and set the SMTP_PASSWORD variable -# to the actual account password, shared between the georchestra developers. -services: - datafeeder: - environment: - - smtpPassword=${SMTP_PASSWORD} - - smtpHost=smtp.gmail.com - - smtpPort=587 - - smtpUser=noreply.georchestra.dev@gmail.com - - smtpAuth=true - - smtpTLS=true - - administratorEmail=noreply.georchestra.dev@gmail.com diff --git a/docker-compose.airflow.yaml b/docker-compose.datafeeder.yaml similarity index 90% rename from docker-compose.airflow.yaml rename to docker-compose.datafeeder.yaml index cba1e35..f1dc833 100644 --- a/docker-compose.airflow.yaml +++ b/docker-compose.datafeeder.yaml @@ -27,8 +27,6 @@ # Default: apache/airflow:3.3.0 # AIRFLOW_UID - User ID in Airflow containers # Default: 50000 -# AIRFLOW_PROJ_DIR - Base path to which all the files will be volumed. -# Default: . # Those configurations are useful mostly in case of standalone testing/running Airflow in test/try-out mode # # _AIRFLOW_WWW_USER_USERNAME - Username for the administrator account (if requested). @@ -49,10 +47,10 @@ x-airflow-common: # In order to add custom dependencies or upgrade provider distributions you can use your extended image. # Comment the image line, place your Dockerfile in the directory where you placed the docker-compose.yaml # and uncomment the "build" line below, Then run `docker-compose build` to build the images. - image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:3.3.0} + image: ${AIRFLOW_IMAGE_NAME:-georchestra/datafeeder-python-airflow:latest} # build: . - env_file: - - ${ENV_FILE_PATH:-.env} + # env_file: + # - ${ENV_FILE_PATH:-.env-airflow} environment: &airflow-common-env AIRFLOW__CORE__EXECUTOR: CeleryExecutor @@ -60,12 +58,17 @@ x-airflow-common: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql+psycopg2://airflow:airflow@postgres/airflow AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0 - AIRFLOW__CORE__FERNET_KEY: ${FERNET_KEY} - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' - AIRFLOW__CORE__LOAD_EXAMPLES: 'true' - AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://airflow-apiserver:8080/execution/' - AIRFLOW__API_AUTH__JWT_SECRET: ${AIRFLOW__API_AUTH__JWT_SECRET:-airflow_jwt_secret} - AIRFLOW__API_AUTH__JWT_ISSUER: ${AIRFLOW__API_AUTH__JWT_ISSUER:-airflow} + AIRFLOW__CORE__FERNET_KEY: '' + AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'false' + AIRFLOW__CORE__LOAD_EXAMPLES: 'false' + AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://airflow-apiserver:8081/airflow/execution/' + AIRFLOW__API__BASE_URL: 'http://localhost:8080/airflow/' + AIRFLOW__API__PORT: 8081 + AIRFLOW__DAG_PROCESSOR__MIN_FILE_PROCESS_INTERVAL: 10 + AIRFLOW__DAG_PROCESSOR__REFRESH_INTERVAL: 30 + AIRFLOW_STAGING_TIMEOUT_SECONDS: ${AIRFLOW_STAGING_TIMEOUT_SECONDS:-600} + AIRFLOW__SECRETS__BACKEND: 'airflow.secrets.local_filesystem.LocalFilesystemBackend' + AIRFLOW__SECRETS__BACKEND_KWARGS: '{"variables_file_path": "/opt/airflow/files/variables.json", "connections_file_path": "/opt/airflow/files/connections.json"}' # yamllint disable rule:line-length # Use simple http server on scheduler for health checks # See https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/logging-monitoring/check-health.html#scheduler-health-check-server @@ -76,11 +79,11 @@ x-airflow-common: _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-} # The following line can be used to set a custom config file, stored in the local config folder AIRFLOW_CONFIG: '/opt/airflow/config/airflow.cfg' + AIRFLOW__API_AUTH__JWT_SECRET: 'nty6o8yV+OsnfEvsZ/ehZQ==' + BACKEND_INTERNAL_URL: ${BACKEND_INTERNAL_URL:-http://datafeeder-backend:8000} + PYOGRIO_USE_ARROW: 1 volumes: - - ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags - - ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs - - ${AIRFLOW_PROJ_DIR:-.}/config:/opt/airflow/config - - ${AIRFLOW_PROJ_DIR:-.}/plugins:/opt/airflow/plugins + - airflow-logs:/opt/airflow/logs user: "${AIRFLOW_UID:-50000}:0" depends_on: &airflow-common-depends-on @@ -103,7 +106,6 @@ services: interval: 10s retries: 5 start_period: 5s - restart: always redis: # Redis is limited to 7.2-bookworm due to licencing change @@ -117,20 +119,18 @@ services: timeout: 30s retries: 50 start_period: 30s - restart: always airflow-apiserver: <<: *airflow-common command: api-server ports: - - "8080:8080" + - "8081:8081" healthcheck: - test: ["CMD", "curl", "--fail", "http://localhost:8080/api/v2/monitor/health"] + test: ["CMD", "curl", "--fail", "http://localhost:8081/api/v2/version"] interval: 30s timeout: 10s retries: 5 start_period: 30s - restart: always depends_on: <<: *airflow-common-depends-on airflow-init: @@ -145,7 +145,6 @@ services: timeout: 10s retries: 5 start_period: 30s - restart: always depends_on: <<: *airflow-common-depends-on airflow-init: @@ -160,7 +159,6 @@ services: timeout: 10s retries: 5 start_period: 30s - restart: always depends_on: <<: *airflow-common-depends-on airflow-init: @@ -181,7 +179,6 @@ services: # Required to handle warm shutdown of the celery workers properly # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation DUMB_INIT_SETSID: "0" - restart: always depends_on: <<: *airflow-common-depends-on airflow-apiserver: @@ -198,7 +195,6 @@ services: timeout: 10s retries: 5 start_period: 30s - restart: always depends_on: <<: *airflow-common-depends-on airflow-init: @@ -274,13 +270,13 @@ services: echo ls -la /opt/airflow/{logs,dags,plugins,config} echo - echo "Change ownership of files in /opt/airflow to ${AIRFLOW_UID:-50000}:0" + echo "Change ownership of files in /opt/airflow to ${AIRFLOW_UID}:0" echo - chown -R "${AIRFLOW_UID:-50000}:0" /opt/airflow/ + chown -R "${AIRFLOW_UID}:0" /opt/airflow/ echo - echo "Change ownership of files in shared volumes to ${AIRFLOW_UID:-50000}:0" + echo "Change ownership of files in shared volumes to ${AIRFLOW_UID}:0" echo - chown -v -R "${AIRFLOW_UID:-50000}:0" /opt/airflow/{logs,dags,plugins,config} + chown -v -R "${AIRFLOW_UID}:0" /opt/airflow/{logs,dags,plugins,config} echo echo "Files in shared volumes:" echo @@ -327,7 +323,6 @@ services: timeout: 10s retries: 5 start_period: 30s - restart: always depends_on: <<: *airflow-common-depends-on airflow-init: @@ -335,3 +330,4 @@ services: volumes: postgres-db-volume: + airflow-logs: diff --git a/docker-compose.yml b/docker-compose.yml index 9159961..160d20e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,5 @@ -include: - - docker-compose.data-api.yml +#include: +# - docker-compose.datafeeder.yml volumes: postgresql_data: From 0e07d0fa1c85dc591670a72d3540bfcc0cd59ce7 Mon Sep 17 00:00:00 2001 From: Florian Necas Date: Tue, 7 Jul 2026 17:44:48 +0200 Subject: [PATCH 3/4] feat: add last config to make datafeeder work --- .envs-database-datafeeder | 14 +- ...eder.yaml => docker-compose.datafeeder.yml | 34 +++- docker-compose.yml | 187 +++--------------- 3 files changed, 64 insertions(+), 171 deletions(-) rename docker-compose.datafeeder.yaml => docker-compose.datafeeder.yml (94%) diff --git a/.envs-database-datafeeder b/.envs-database-datafeeder index e12bec0..82ccb6f 100644 --- a/.envs-database-datafeeder +++ b/.envs-database-datafeeder @@ -1,10 +1,6 @@ # envs-database-datafeeder -DF_PGHOST=postgis -DF_PGDATABASE=datafeeder -DF_PGPORT=5432 -DF_PGUSER=georchestra -DF_PGPASSWORD=georchestra - -POSTGRES_DB=${DF_PGDATABASE} -POSTGRES_USER=${DF_PGUSER} -POSTGRES_PASSWORD=${DF_PGPASSWORD} \ No newline at end of file +POSTGRES_DATA_HOST=datadb +POSTGRES_DATA_DB=postgres +POSTGRES_DATA_PORT=5432 +POSTGRES_DATA_USER=postgres +POSTGRES_DATA_PASSWORD=mypassword \ No newline at end of file diff --git a/docker-compose.datafeeder.yaml b/docker-compose.datafeeder.yml similarity index 94% rename from docker-compose.datafeeder.yaml rename to docker-compose.datafeeder.yml index f1dc833..3b216ef 100644 --- a/docker-compose.datafeeder.yaml +++ b/docker-compose.datafeeder.yml @@ -62,7 +62,7 @@ x-airflow-common: AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'false' AIRFLOW__CORE__LOAD_EXAMPLES: 'false' AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://airflow-apiserver:8081/airflow/execution/' - AIRFLOW__API__BASE_URL: 'http://localhost:8080/airflow/' + AIRFLOW__API__BASE_URL: 'https://georchestra-127-0-0-1.nip.io/airflow/' AIRFLOW__API__PORT: 8081 AIRFLOW__DAG_PROCESSOR__MIN_FILE_PROCESS_INTERVAL: 10 AIRFLOW__DAG_PROCESSOR__REFRESH_INTERVAL: 30 @@ -84,6 +84,7 @@ x-airflow-common: PYOGRIO_USE_ARROW: 1 volumes: - airflow-logs:/opt/airflow/logs + - ./config/datafeeder/airflow:/opt/airflow/files user: "${AIRFLOW_UID:-50000}:0" depends_on: &airflow-common-depends-on @@ -91,8 +92,36 @@ x-airflow-common: condition: service_healthy postgres: condition: service_healthy + copy-datadir: + condition: service_completed_successfully services: + datafeeder-frontend: + image: georchestra/datafeeder-python-frontend:latest + environment: + - ENABLE_GEORCHESTRA_HEADER=true + ports: + - "8888:8080" + depends_on: + <<: *airflow-common-depends-on + + datafeeder-backend: + image: georchestra/datafeeder-python-backend:latest + volumes: + - georchestra_datadir:/etc/georchestra + environment: + - DATADIR=/etc/georchestra + - DEBUG=true + depends_on: + <<: *airflow-common-depends-on + database: + condition: service_healthy + env_file: + - .envs-common + - .envs-ldap + - .envs-hosts + - .envs-database-georchestra + postgres: image: postgres:16 environment: @@ -123,8 +152,6 @@ services: airflow-apiserver: <<: *airflow-common command: api-server - ports: - - "8081:8081" healthcheck: test: ["CMD", "curl", "--fail", "http://localhost:8081/api/v2/version"] interval: 30s @@ -331,3 +358,4 @@ services: volumes: postgres-db-volume: airflow-logs: + diff --git a/docker-compose.yml b/docker-compose.yml index 160d20e..3e5ea64 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,7 @@ volumes: postgresql_data: + pg_data: ldap_data: ldap_config: geoserver_geodata: @@ -28,26 +29,12 @@ services: - ./config:/mnt/datadir - georchestra_datadir:/etc/georchestra - envsubst: - image: georchestra/k8s-initcontainer-envsubst - depends_on: - copy-datadir: - condition: service_completed_successfully - environment: - - DEBUG=yes - - SUBST_FILES=/etc/georchestra/security-proxy/targets-mapping.properties /etc/georchestra/datafeeder/frontend-config.json /etc/georchestra/datafeeder/metadata_* /etc/georchestra/geonetwork/microservices/ogc-api-records/config.yml - env_file: - - .envs-common - - .envs-hosts - volumes: - - georchestra_datadir:/etc/georchestra - database: image: georchestra/database:latest env_file: - .envs-database-georchestra depends_on: - envsubst: + copy-datadir: condition: service_completed_successfully volumes: - postgresql_data:/var/lib/postgresql @@ -56,7 +43,7 @@ services: ldap: image: georchestra/ldap:latest depends_on: - envsubst: + copy-datadir: condition: service_completed_successfully secrets: - slapd_password @@ -90,49 +77,6 @@ services: - .envs-hosts - .envs-database-georchestra -# uncomment for oauth 2.0 -# cas: -# image: georchestra/cas:latest -# healthcheck: -# test: [ "CMD-SHELL", "curl -s -f http://localhost:8080/cas/login >/dev/null || exit 1" ] -# interval: 30s -# timeout: 10s -# retries: 10 -# depends_on: -# ldap: -# condition: service_healthy -# volumes: -# - georchestra_datadir:/etc/georchestra -# environment: -# - JAVA_OPTIONS=-Dorg.eclipse.jetty.annotations.AnnotationParser.LEVEL=OFF -# - XMS=256M -# - XMX=1G -# env_file: -# - .envs-common -# - .envs-ldap -# - .envs-database-georchestra -# restart: always - - header: - image: georchestra/header:latest - healthcheck: - test: ["CMD-SHELL", "curl -s -f http://localhost:8080/header/img/logo.png >/dev/null || exit 1"] - interval: 30s - timeout: 10s - retries: 10 - depends_on: - envsubst: - condition: service_completed_successfully - volumes: - - georchestra_datadir:/etc/georchestra - environment: - - JAVA_OPTIONS=-Dorg.eclipse.jetty.annotations.AnnotationParser.LEVEL=OFF - - XMS=256M - - XMX=512M - env_file: - - .envs-common - restart: always - geoserver: image: georchestra/geoserver:latest healthcheck: @@ -187,7 +131,7 @@ services: restart: always geonetwork: - image: georchestra/geonetwork:latest + image: georchestra/geonetwork:4.4.11-georchestra-01 healthcheck: test: ["CMD-SHELL", "curl -s -f http://localhost:8080/geonetwork/srv/eng/catalog.search >/dev/null || exit 1"] interval: 30s @@ -212,6 +156,8 @@ services: - .envs-database-georchestra - .envs-elastic restart: always + ports: + - "5005:5005" datahub: image: geonetwork/geonetwork-ui-datahub:latest @@ -221,7 +167,7 @@ services: timeout: 10s retries: 10 depends_on: - envsubst: + copy-datadir: condition: service_completed_successfully environment: ASSETS_DIRECTORY_OVERRIDE: /etc/georchestra/datahub/assets @@ -239,7 +185,7 @@ services: timeout: 10s retries: 10 depends_on: - envsubst: + copy-datadir: condition: service_completed_successfully environment: ASSETS_DIRECTORY_OVERRIDE: /etc/georchestra/metadata-editor/assets @@ -249,26 +195,6 @@ services: - georchestra_datadir:/etc/georchestra restart: always - analytics: - image: georchestra/analytics:latest - healthcheck: - test: ["CMD-SHELL", "curl -s -f http://localhost:8080/analytics/ >/dev/null || exit 1"] - interval: 30s - timeout: 10s - retries: 10 - depends_on: - database: - condition: service_healthy - volumes: - - georchestra_datadir:/etc/georchestra - environment: - - JAVA_OPTIONS=-Dorg.eclipse.jetty.annotations.AnnotationParser.LEVEL=OFF - - XMS=256M - - XMX=1G - env_file: - - .envs-database-georchestra - restart: always - mapstore: image: georchestra/mapstore:latest healthcheck: @@ -291,67 +217,8 @@ services: - .envs-database-georchestra restart: always - postgis: - # used by datafeeder to ingest uploaded user datasets into - image: postgis/postgis:13-3.1-alpine - healthcheck: - test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] - interval: 10s - timeout: 3s - retries: 3 - depends_on: - envsubst: - condition: service_completed_successfully - env_file: - - .envs-database-datafeeder - volumes: - - datafeeder_postgis_data:/var/lib/postgresql/data - restart: always - - datafeeder: - image: georchestra/datafeeder:latest - healthcheck: - test: ["CMD-SHELL", "curl -s -f http://localhost:8080/datafeeder >/dev/null || exit 1"] - interval: 30s - timeout: 10s - retries: 10 - depends_on: - database: - condition: service_healthy - postgis: - condition: service_healthy - volumes: - - georchestra_datadir:/etc/georchestra - - datafeeder_uploads:/tmp/datafeeder - environment: - - JAVA_OPTIONS=-Xms512m -Xmx512m -Dspring.profiles.active=georchestra,data-api-schemas -Dspring.config.additional-location=file:/etc/georchestra/data-api/application.yaml - # You can set a higher loglevel this way: (ref. https://docs.spring.io/spring-boot/docs/2.1.13.RELEASE/reference/html/boot-features-logging.html#boot-features-custom-log-levels) - - LOGGING_LEVEL_ORG_GEORCHESTRA_DATAFEEDER=INFO - env_file: - - .envs-common - - .envs-hosts - - .envs-database-georchestra - - .envs-database-datafeeder - restart: always - - import: - image: georchestra/datafeeder-frontend:latest - healthcheck: - test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://127.0.0.1:80/ >/dev/null || exit 1"] - interval: 30s - timeout: 10s - retries: 10 - depends_on: - envsubst: - condition: service_completed_successfully - environment: - CUSTOM_SCRIPTS_DIRECTORY: /etc/georchestra/datafeeder/scripts/import - volumes: - - georchestra_datadir:/etc/georchestra - restart: always - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.14.3 + image: docker.elastic.co/elasticsearch/elasticsearch:8.19.13 ulimits: memlock: soft: -1 @@ -368,7 +235,7 @@ services: timeout: 10s retries: 10 depends_on: - envsubst: + copy-datadir: condition: service_completed_successfully env_file: - .envs-elastic @@ -398,22 +265,24 @@ services: - ./resources/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml restart: always - ogc-api-records: - image: geonetwork/gn-cloud-ogc-api-records-service:4.2.2 - depends_on: - geonetwork: - condition: service_healthy - database: - condition: service_healthy - elasticsearch: - condition: service_healthy + datadb: + image: postgis/postgis + restart: always + ports: + - "5433:5432" environment: - LANG: en_US.UTF-8 - SERVER_SERVLET_CONTEXT_PATH: /ogc-api-records - SPRING_CONFIG_LOCATION: file:///etc/georchestra/geonetwork/microservices/ogc-api-records/config.yml - SPRING_PROFILES_ACTIVE: standalone - JAVA_OPTS: -Dfile.encoding=UTF-8 + POSTGRES_USER: postgres + POSTGRES_PASSWORD: mypassword + POSTGRES_DB: postgres # default database volumes: - - georchestra_datadir:/etc/georchestra - restart: always + - pg_data:/var/lib/postgresql/ + configs: + - source: datadb_init_sql + target: /docker-entrypoint-initdb.d/init.sql +configs: + datadb_init_sql: + content: | + CREATE EXTENSION IF NOT EXISTS postgis; + CREATE SCHEMA IF NOT EXISTS data; + CREATE SCHEMA IF NOT EXISTS staging; From e8964d36b389ee1a913a1519b9263ea2ced79bb7 Mon Sep 17 00:00:00 2001 From: Florian Necas Date: Tue, 7 Jul 2026 17:54:28 +0200 Subject: [PATCH 4/4] chore: add comemnt to add datafeeder in compose deployment --- docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3e5ea64..e1c948d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,4 @@ +# Uncomment to enable datafeeder #include: # - docker-compose.datafeeder.yml @@ -131,7 +132,7 @@ services: restart: always geonetwork: - image: georchestra/geonetwork:4.4.11-georchestra-01 + image: georchestra/geonetwork:latest healthcheck: test: ["CMD-SHELL", "curl -s -f http://localhost:8080/geonetwork/srv/eng/catalog.search >/dev/null || exit 1"] interval: 30s