diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bd91f88e..c9e4a997 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: include: - - python-version: 3.10.12 + - python-version: 3.12.3 steps: - name: Checkout ✅ uses: actions/checkout@v2 diff --git a/Dockerfile b/Dockerfile index 2c3733c0..e5254b5b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:jammy +FROM ubuntu:noble # Credits to yjacolin for providing first versions LABEL original_developer="yjacolin " \ @@ -18,9 +18,8 @@ ENV LC_ALL="en_US.UTF-8" \ LANGUAGE="en_US.UTF-8" \ \ \ - DEB_PACKAGES="locales gunicorn postgresql-client python3-gunicorn python3-gevent python3-psycopg2 python3-lxml python3-pyproj" \ + DEB_PACKAGES="locales gunicorn python3.12-venv postgresql-client python3-gunicorn python3-gevent python3-lxml python3-pyproj" \ DEB_BUILD_DEPS="make python3-pip" \ - # GHC ENV settings\ ADMIN_NAME=admin \ ADMIN_PWD=admin \ ADMIN_EMAIL=admin.istrator@mydomain.com \ @@ -91,7 +90,7 @@ COPY . /GeoHealthCheck # Install RUN \ chmod a+x /*.sh && ./install.sh \ - # Cleanup TODO: remove unused Locales and TZs + # Cleanup TODO: remove unused Locales and TZs \ && apt-get remove --purge -y ${DEB_BUILD_DEPS} \ && apt-get clean \ && apt autoremove -y \ diff --git a/GeoHealthCheck/healthcheck.py b/GeoHealthCheck/healthcheck.py index 1451699d..a8c02dc2 100644 --- a/GeoHealthCheck/healthcheck.py +++ b/GeoHealthCheck/healthcheck.py @@ -27,7 +27,7 @@ # # ================================================================= -from datetime import datetime +from datetime import datetime, timezone import logging import json from urllib.request import urlopen @@ -95,7 +95,7 @@ def run_resource(resourceid): # Run test result = run_test_resource(resource) - run1 = Run(resource, result, datetime.utcnow()) + run1 = Run(resource, result, datetime.now(timezone.utc)) DB.session.add(run1) @@ -144,7 +144,7 @@ def sniff_test_resource(config, resource_type, url): raise RuntimeError(msg2) title = None - start_time = datetime.utcnow() + start_time = datetime.now(timezone.utc) message = None resource_type_map = {'OGC:WMS': [partial(WebMapService, version='1.3.0'), partial(WebMapService, version='1.1.1')], @@ -223,7 +223,7 @@ def sniff_test_resource(config, resource_type, url): title = urlparse(url).hostname elif resource_type == 'OSGeo:GeoNode': endpoints = ows - end_time = datetime.utcnow() + end_time = datetime.now(timezone.utc) delta = end_time - start_time response_time = '%s.%s' % (delta.seconds, delta.microseconds) base_tags = geonode_make_tags(url) @@ -259,7 +259,7 @@ def sniff_test_resource(config, resource_type, url): message = msg success = False - end_time = datetime.utcnow() + end_time = datetime.now(timezone.utc) delta = end_time - start_time response_time = '%s.%s' % (delta.seconds, delta.microseconds) @@ -311,10 +311,10 @@ def geonode_make_tags(base_url): if __name__ == '__main__': print('START - Running health check tests on %s' - % datetime.utcnow().isoformat()) + % datetime.now(timezone.utc).isoformat()) run_resources() print('END - Running health check tests on %s' - % datetime.utcnow().isoformat()) + % datetime.now(timezone.utc).isoformat()) # from init import App # if len(sys.argv) < 3: # print('Usage: %s ' % sys.argv[0]) diff --git a/GeoHealthCheck/migrations/versions/2638c2a40625_.py b/GeoHealthCheck/migrations/versions/2638c2a40625_.py index b729043d..379a0af1 100644 --- a/GeoHealthCheck/migrations/versions/2638c2a40625_.py +++ b/GeoHealthCheck/migrations/versions/2638c2a40625_.py @@ -7,8 +7,6 @@ """ from alembic import op import sqlalchemy as sa -import imp -import os from GeoHealthCheck.migrations import alembic_helpers # revision identifiers, used by Alembic. diff --git a/GeoHealthCheck/migrations/versions/496427d03f87_.py b/GeoHealthCheck/migrations/versions/496427d03f87_.py index c4d290c4..b2791b6d 100644 --- a/GeoHealthCheck/migrations/versions/496427d03f87_.py +++ b/GeoHealthCheck/migrations/versions/496427d03f87_.py @@ -7,8 +7,6 @@ """ from alembic import op import sqlalchemy as sa -import imp -import os from GeoHealthCheck.migrations import alembic_helpers # revision identifiers, used by Alembic. diff --git a/GeoHealthCheck/migrations/versions/992013af402f_.py b/GeoHealthCheck/migrations/versions/992013af402f_.py index cd221963..7505b3b1 100644 --- a/GeoHealthCheck/migrations/versions/992013af402f_.py +++ b/GeoHealthCheck/migrations/versions/992013af402f_.py @@ -7,8 +7,6 @@ """ from alembic import op import sqlalchemy as sa -import imp -import os from GeoHealthCheck.migrations import alembic_helpers # revision identifiers, used by Alembic. diff --git a/GeoHealthCheck/models.py b/GeoHealthCheck/models.py index 45f9d5b0..627b4ae6 100644 --- a/GeoHealthCheck/models.py +++ b/GeoHealthCheck/models.py @@ -31,7 +31,7 @@ import json import logging from flask_babel import gettext as _ -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from itsdangerous import TimedJSONWebSignatureSerializer as Serializer from sqlalchemy import func, and_ @@ -58,7 +58,7 @@ def flush_runs(): all_runs = Run.query.all() run_count = 0 for run in all_runs: - days_old = (datetime.utcnow() - run.checked_datetime).days + days_old = (datetime.now(timezone.utc) - run.checked_datetime).days if days_old > retention_days: run_count += 1 DB.session.delete(run) @@ -87,7 +87,7 @@ class Run(DB.Model): report = deferred(DB.Column(DB.Text, default={})) def __init__(self, resource, result, - checked_datetime=datetime.utcnow()): + checked_datetime=datetime.now(timezone.utc)): self.resource = resource self.success = result.success self.response_time = result.response_time_str @@ -680,13 +680,13 @@ def __init__(self, resource, owner, interval_mins): self.init_datetimes(interval_mins) def init_datetimes(self, interval_mins): - self.start_time = datetime.utcnow() + self.start_time = datetime.now(timezone.utc) # Subtract some space from end-time to allow obtain at scheduled time minutes = interval_mins - 1 self.end_time = self.start_time + timedelta(minutes=minutes) def has_expired(self): - now = datetime.utcnow() + now = datetime.now(timezone.utc) return now > self.end_time def obtain(self, owner, frequency): @@ -723,7 +723,7 @@ def __init__(self, username, password, email, role='user'): self.set_password(password) self.email = email self.role = role - self.registered_on = datetime.utcnow() + self.registered_on = datetime.now(timezone.utc) def authenticate(self, password): return util.verify_hash(password, self.password) diff --git a/GeoHealthCheck/probe.py b/GeoHealthCheck/probe.py index b8f497e0..937e0866 100644 --- a/GeoHealthCheck/probe.py +++ b/GeoHealthCheck/probe.py @@ -1,7 +1,7 @@ import logging import sys -import datetime +from datetime import datetime, timezone import requests from factory import Factory @@ -128,7 +128,7 @@ def get_metadata_cached(self, resource, version='any'): metadata = None if key in Probe.METADATA_CACHE: entry = Probe.METADATA_CACHE[key] - delta = datetime.datetime.utcnow() - entry['time'] + delta = datetime.now(timezone.utc) - entry['time'] metadata = entry['metadata'] # Don't keep cache forever, refresh every N mins @@ -144,7 +144,7 @@ def get_metadata_cached(self, resource, version='any'): # Store entry with time, for expiry later entry = { "metadata": metadata, - "time": datetime.datetime.utcnow() + "time": datetime.now(timezone.utc) } Probe.METADATA_CACHE[key] = entry diff --git a/GeoHealthCheck/result.py b/GeoHealthCheck/result.py index ae890ef2..d635c6b7 100644 --- a/GeoHealthCheck/result.py +++ b/GeoHealthCheck/result.py @@ -1,4 +1,4 @@ -import datetime +from datetime import datetime, timezone class Result(object): @@ -36,10 +36,10 @@ def set(self, success, message): self.message = message def start(self): - self.start_time = datetime.datetime.utcnow() + self.start_time = datetime.now(timezone.utc) def stop(self): - self.end_time = datetime.datetime.utcnow() + self.end_time = datetime.now(timezone.utc) delta = self.end_time - self.start_time self.response_time_secs = delta.seconds diff --git a/docker/README.md b/docker/README.md index a1b0977f..2a8d8199 100644 --- a/docker/README.md +++ b/docker/README.md @@ -43,7 +43,7 @@ For example [run.sh](run.sh) which launches GHC using the robust `gunicorn` WSGI ## Run ``` -docker run -d --name ghc_web -p 8083:80 -v ghc_sqlitedb:/GeoHealthCheck/DB geopython/geohealthcheck:latest +docker run -d --name ghc_web --rm -p 8083:80 -v ghc_sqlitedb:/GeoHealthCheck/DB geopython/geohealthcheck:latest ``` go to http://localhost:8083 (port 80 in GHC Container is mapped to 8083 on host). @@ -56,7 +56,7 @@ This mode can be disabled by passing `GHC_RUNNER_IN_WEBAPP` to as an ENV var to the Docker container: ``` -docker run --name ghc_web -e GHC_RUNNER_IN_WEBAPP=False -p 8083:80 -v ghc_sqlitedb:/GeoHealthCheck/DB geopython/geohealthcheck:latest +docker run --name ghc_web --rm -e GHC_RUNNER_IN_WEBAPP=False -p 8083:80 -v ghc_sqlitedb:/GeoHealthCheck/DB geopython/geohealthcheck:latest ``` @@ -66,13 +66,13 @@ You can then run `GHC Runner` as a separate container by overriding the default `ENTRYPOINT` with `/run-runner.sh`: ``` -docker run -d --name ghc_runner --entrypoint "/run-runner.sh" -v ghc_sqlitedb:/GeoHealthCheck/DB geopython/geohealthcheck:latest +docker run -d --name ghc_runner --rm --entrypoint "/run-runner.sh" -v ghc_sqlitedb:/GeoHealthCheck/DB geopython/geohealthcheck:latest ``` But the most optimal way to run GHC with scheduled jobs and optionally Postgres as backend DB, is to use [Docker Compose](https://docs.docker.com/compose), see below. -## Using docker-compose +## Using docker compose This allows a complete Docker setup, including scheduling and optionally using Postgres/PostGIS as database (recommended). @@ -80,7 +80,7 @@ See the [Docker Compose Documentation](https://docs.docker.com/compose) for more info. GHC Webapp and Runner are in this case deployed as separate processes (Docker containers). -*Note that the `docker-compose` YAML files below are meant as examples to be adapted to your* +*Note that the `docker compose` YAML files below are meant as examples to be adapted to your* *local deployment situation.* ### Using sqlite DB (default) @@ -92,7 +92,7 @@ To run (`-d` allows running in background): ``` cd docker/compose -docker-compose -f docker-compose.yml up [-d] +docker compose -f docker-compose.yml up [-d] # go to http://localhost:8083 (port 80 in GHC Container is mapped to 8083 on host) @@ -105,10 +105,9 @@ similar but uses Postgres as the database. To run: - ``` cd docker/compose -docker-compose -f docker-compose.postgis.yml up [-d] +docker compose -f docker-compose.postgis.yml up [-d] # go to http://localhost:8083 (port 80 in GHC Container is mapped to 8083 on host) diff --git a/docker/compose/README.md b/docker/compose/README.md index 3deade94..2c4ccd3c 100644 --- a/docker/compose/README.md +++ b/docker/compose/README.md @@ -5,3 +5,16 @@ Within this directory are *examples* for running GHC using Docker compose. You should copy and adapt these for your own deployment. + +## Running with SQLite DB +```shell +docker compose up -d +docker compose down --remove-orphans + +``` +## Running with PostGIS DB +```shell +docker compose -f docker-compose.postgis.yml up -d +docker compose -f docker-compose.postgis.yml down --remove-orphans + +``` diff --git a/docker/compose/docker-compose.postgis.yml b/docker/compose/docker-compose.postgis.yml index 71a1d16b..a0b44a7f 100644 --- a/docker/compose/docker-compose.postgis.yml +++ b/docker/compose/docker-compose.postgis.yml @@ -1,9 +1,8 @@ # GHC Docker setup with Postgres as backend DB. # # To run: -# sudo docker-compose -f docker-compose.postgis.yml up [-d] +# sudo docker compose -f docker-compose.postgis.yml up [-d] # -version: "3" services: ghc_web: @@ -63,7 +62,7 @@ services: # - ./../GeoHealthCheck/plugins:/plugins:ro postgis_ghc: - image: mdillon/postgis:10-alpine + image: postgis/postgis:14-3.5-alpine container_name: postgis_ghc diff --git a/docker/compose/ghc-postgis.env b/docker/compose/ghc-postgis.env index 67bd5892..4a8df6ed 100644 --- a/docker/compose/ghc-postgis.env +++ b/docker/compose/ghc-postgis.env @@ -1,7 +1,11 @@ # We mainly need to override the default (sqlite) DB URI SQLALCHEMY_DATABASE_URI=postgresql://ghc:ghc@postgis_ghc:5432/ghc -# Postgres Docker container settings +# PostgreSQL/PostGIS Docker container settings +POSTGRES_HOST=postgis_ghc +POSTGRES_PORT=5432 POSTGRES_USER=ghc +PGUSER=ghc POSTGRES_PASSWORD=ghc +PGPASSWORD=ghc POSTGRES_DB=ghc diff --git a/docker/docker-clean.sh b/docker/docker-clean.sh index 87716567..90dcef51 100755 --- a/docker/docker-clean.sh +++ b/docker/docker-clean.sh @@ -1,3 +1,14 @@ #!/bin/bash -docker rm $(docker ps -a -q) -docker rmi $(docker images -f dangling=true -q) + +# +# Remove all exited containers +for c in $(docker ps -a -f status=exited -q) +do + docker rm ${c} +done + +# And dangling images +for i in $(docker images -f dangling=true -q) +do + docker rmi ${i} +done diff --git a/docker/install-docker-ubuntu.sh b/docker/install-docker-ubuntu.sh deleted file mode 100755 index 0119b2fc..00000000 --- a/docker/install-docker-ubuntu.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -# -# This prepares an empty Ubuntu system for running Docker. -# -# Just van den Broecke - 2017 -# DEPRECATED - 2021 update: there are much quicker ways -# See https://docs.docker.com/engine/install/ubuntu/ -# -# Below was based on -# https://docs.docker.com/engine/installation/linux/ubuntu/ -# as on may 26 2017. -# Run as root or prepend all commands with "sudo"! -# - -# Optional, comment out for your locale -# set time right and configure timezone and locale -# echo "Europe/Amsterdam" > /etc/timezone -# dpkg-reconfigure -f noninteractive tzdata - -# Bring system uptodate -apt-get update -apt-get -y upgrade - -# Install packages to allow apt to use a repository over HTTPS -apt-get install -y software-properties-common apt-transport-https ca-certificates curl - -# Add keys and extra repos -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - -# Verify key -apt-key fingerprint 0EBFCD88 - -# Add Docker repo to deb config -add-apt-repository \ - "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ - $(lsb_release -cs) \ - stable" - -# Bring packages uptodate -apt-get update - -# The linux-image-extra package allows you use the aufs storage driver. -# at popup keep locally installed config option -# apt-get install -y linux-image-extra-$(uname -r) -apt-get install -y linux-image-extra-$(uname -r) linux-image-extra-virtual - -# https://askubuntu.com/questions/98416/error-kernel-headers-not-found-but-they-are-in-place -apt-get install -y build-essential linux-headers-`uname -r` dkms - -# Install Docker CE -apt-get install docker-ce - -# If you are installing on Ubuntu 14.04 or 12.04, apparmor is required. -# You can install it using (usually already installed) -# apt-get install -y apparmor - -# Start the docker daemon. Usually already running -# service docker start - -# Docker compose -export dockerComposeVersion="1.20.1" -curl -L https://github.com/docker/compose/releases/download/${dockerComposeVersion}/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose -chmod +x /usr/local/bin/docker-compose diff --git a/docker/scripts/configure.sh b/docker/scripts/configure.sh index dd851f4e..980faa5e 100755 --- a/docker/scripts/configure.sh +++ b/docker/scripts/configure.sh @@ -13,6 +13,8 @@ echo "Using DB_TYPE=${DB_TYPE}" # Create DB shorthand function create_db() { pushd /GeoHealthCheck/ || exit 1 + source bin/activate + paver create -u ${ADMIN_NAME} -p ${ADMIN_PWD} -e ${ADMIN_EMAIL} popd || exit 1 } @@ -45,7 +47,7 @@ case ${DB_TYPE} in echo "Check if Postgres is avail/ready..." until pg_isready -h "${DB_HOST}"; do echo "Exit code=$? - Postgres not ready - sleeping" - sleep 1 + sleep $[ ( $RANDOM % 6 ) + 1 ]s done # Check if we need to create DB tables diff --git a/docker/scripts/install.sh b/docker/scripts/install.sh index 117ffc13..5f448c93 100755 --- a/docker/scripts/install.sh +++ b/docker/scripts/install.sh @@ -5,10 +5,14 @@ # e.g. fetching Metadata (Caps) and testing all layers # Install Python packages for installation and setup +python3 -m venv GeoHealthCheck pushd /GeoHealthCheck || exit 1 +source bin/activate + + # Docker-specific deps -pip install -r docker/scripts/requirements.txt +pip install --no-cache-dir -r docker/scripts/requirements.txt # Sets up GHC itself paver setup diff --git a/docker/scripts/run-runner.sh b/docker/scripts/run-runner.sh index a3e3b000..4b559027 100755 --- a/docker/scripts/run-runner.sh +++ b/docker/scripts/run-runner.sh @@ -12,6 +12,7 @@ echo "START /run-runner.sh" export PYTHONPATH=/GeoHealthCheck/GeoHealthCheck:$PYTHONPATH cd /GeoHealthCheck +source bin/activate paver runner_daemon echo "END /run-runner.sh" diff --git a/docker/scripts/run-tests.sh b/docker/scripts/run-tests.sh index 489a7fbf..d75693c7 100755 --- a/docker/scripts/run-tests.sh +++ b/docker/scripts/run-tests.sh @@ -17,6 +17,7 @@ echo "START /run-tests.sh" export PYTHONPATH=/GeoHealthCheck/GeoHealthCheck:$PYTHONPATH cd /GeoHealthCheck +source bin/activate paver run_tests echo "END /run-tests.sh" diff --git a/docker/scripts/run-web.sh b/docker/scripts/run-web.sh index b825996d..47bcbaf1 100755 --- a/docker/scripts/run-web.sh +++ b/docker/scripts/run-web.sh @@ -13,7 +13,9 @@ echo "START /run-web.sh" # Make sure PYTHONPATH includes GeoHealthCheck export PYTHONPATH=/GeoHealthCheck/GeoHealthCheck:$PYTHONPATH -cd /GeoHealthCheck +pushd /GeoHealthCheck || exit 1 + +source bin/activate paver upgrade @@ -21,7 +23,7 @@ paver upgrade [ "${SCRIPT_NAME}" = '/' ] && export SCRIPT_NAME="" && echo "make SCRIPT_NAME empty from /" echo "Running GHC WSGI on ${HOST}:${PORT} with ${WSGI_WORKERS} workers and SCRIPT_NAME=${SCRIPT_NAME}" -exec gunicorn --workers ${WSGI_WORKERS} \ +exec gunicorn --pythonpath /GeoHealthCheck/lib/python3.12/site-packages/ --workers ${WSGI_WORKERS} \ --worker-class=${WSGI_WORKER_CLASS} \ --timeout ${WSGI_WORKER_TIMEOUT} \ --name="Gunicorn_GHC" \ diff --git a/requirements.txt b/requirements.txt index f4bb6441..c43ec0a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,14 +12,15 @@ itsdangerous==1.1.0 pyproj >=2.6.1 lxml >= 4.8.0, <= 4.9.4 OWSLib==0.20.0 -jsonschema==3.0.2 # downgrade from 3.2.0 on sept 29, 2020, issue 331, consider better fix -openapi-spec-validator==0.2.8 +jsonschema==4.26.0 # downgrade from 3.2.0 to 3.0.2 on sept 29, 2020, issue 331, consider better fix +openapi-spec-validator==0.9.0 +psycopg2-binary Sphinx==5.3.0 sphinx-rtd-theme==1.3.0 sphinx-autoapi requests>=2.23.0 WTForms==2.2.1 -APScheduler==3.6.1 +APScheduler==3.11.2 passlib==1.7.1 Werkzeug==0.16.1 -tzlocal<3.0 # Fix based on https://github.com/Yelp/elastalert/issues/2968 +tzlocal # Fix based on https://github.com/Yelp/elastalert/issues/2968