From 7d506aa71b3014970ffc8352c1430e7bd8f61a38 Mon Sep 17 00:00:00 2001 From: Mial Lewis Date: Thu, 29 Jan 2026 14:16:23 +0000 Subject: [PATCH 01/20] add_query_endpoint --- blank.env | 3 +++ docker-compose.yml | 9 --------- web/services/urls.py | 1 + web/services/views.py | 27 +++++++++++++++++++++++++++ web/settings.py | 11 +++++++++++ 5 files changed, 42 insertions(+), 9 deletions(-) diff --git a/blank.env b/blank.env index df82549..0f8f6aa 100644 --- a/blank.env +++ b/blank.env @@ -17,6 +17,9 @@ DB_PORT=5432 SECRET_KEY= DB_USER= DB_PASS= +DB_RO_USER= +DB_RO_PASS= +QUERY_SECRET_KEY= #Trusted origins for CSRF validation. For production usage only specify https://reports.mantidproject.org DJANGO_CSRF_TRUSTED_ORIGINS=http://localhost:8082,https://reports.a.staging-mantidproject.stfc.ac.uk diff --git a/docker-compose.yml b/docker-compose.yml index af74954..7630158 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,9 +10,6 @@ services: POSTGRES_USER: ${DB_USER} POSTGRES_PASSWORD: ${DB_PASS} POSTGRES_DB: ${DB_NAME} - DB_SERVICE: ${DB_SERVICE} - DB_PORT: ${DB_PORT} - SECRET_KEY: ${SECRET_KEY} adminer: image: adminer @@ -34,12 +31,6 @@ services: depends_on: - postgres env_file: .env - environment: - DB_SERVICE: ${DB_SERVICE} - DB_PORT: ${DB_PORT} - SECRET_KEY: ${SECRET_KEY} - # Define this in .env for development mode. DO NOT USE IN PRODUCTION - DEBUG: ${DEBUG} nginx-reports: restart: always diff --git a/web/services/urls.py b/web/services/urls.py index 66cfb40..c7075c9 100644 --- a/web/services/urls.py +++ b/web/services/urls.py @@ -17,6 +17,7 @@ path("by/user", views.usage_by_users, name="by-users"), path("host", views.host_list, name="host-list"), path("user", views.user_list, name="user-list"), + path("query", views.query, name="query"), # url(r'feature', views.feature_usage, name='feature_usage'), ] diff --git a/web/services/views.py b/web/services/views.py index ef8e81e..d11d6aa 100644 --- a/web/services/views.py +++ b/web/services/views.py @@ -15,10 +15,13 @@ import django_filters from rest_framework.reverse import reverse from django.http import HttpResponse +from django.db import connections + import json import datetime import hashlib import services.plots as plotsfile +from os import environ OS_NAMES = ["Linux", "Windows NT", "Darwin"] UTC = datetime.tzinfo("UTC") @@ -327,6 +330,30 @@ def by_root(request, format=None): } ) +@api_view(("POST",)) +def query(request, format=None): + sql_err, sql = get_parameter(request, "sql") + token_err, token = get_parameter(request, "token") + if sql_err or token_err: + return response.Response(status=400, data=f"Invalid Parameters: {[x for x in [sql_err, token_err] if x]}") + verified = verify_token(token) + if not verified: + return response.Response(status=401, data="UNAUTHORIZED") + conn=connections["readonly"] + with conn.cursor() as cur: + cur.execute(sql) + res = cur.fetchall() + return response.Response(res) + +def get_parameter(request, param): + val = request.POST.get[param] + err = "" + if not val: + err = f"No {param} parameter provided" + return err, val + +def verify_token(token): + return token==environ["QUERY_SECRET_KEY"] class FeatureViewSet(viewsets.ModelViewSet): """ diff --git a/web/settings.py b/web/settings.py index a5b2213..f826223 100644 --- a/web/settings.py +++ b/web/settings.py @@ -94,6 +94,17 @@ "PASSWORD": os.environ["DB_PASS"], "HOST": os.environ["DB_SERVICE"], "PORT": os.environ["DB_PORT"], + }, + "readonly": { + "ENGINE": "django.db.backends.postgresql_psycopg2", + "NAME": os.environ["DB_NAME"], + "USER": os.environ["DB_RO_USER"], + "PASSWORD": os.environ["DB_RO_PASS"], + "HOST": os.environ["DB_SERVICE"], + "PORT": os.environ["DB_PORT"], + "OPTIONS": { + "options": "-c default_transaction_read_only=on", + }, } } From 109716646b3d9208f3ee171ba86cb7c15056af7e Mon Sep 17 00:00:00 2001 From: Mial Lewis Date: Thu, 29 Jan 2026 16:25:38 +0000 Subject: [PATCH 02/20] add reload to debug --- web/run_django.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/run_django.sh b/web/run_django.sh index 5a52ae3..b36ddb8 100755 --- a/web/run_django.sh +++ b/web/run_django.sh @@ -8,7 +8,7 @@ python manage.py migrate --noinput # If running in DEBUG mode add debug logging to gunicorn if [ -n "${DEBUG}" ]; then - DEBUG_ARGS="--log-level debug --capture-output" + DEBUG_ARGS="--log-level debug --capture-output --reload" else DEBUG_ARGS= fi From 6d4560278767654b7fc7586409fb95ebfea9e9b7 Mon Sep 17 00:00:00 2001 From: Mial Lewis Date: Thu, 29 Jan 2026 16:59:19 +0000 Subject: [PATCH 03/20] correct typo --- web/services/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/services/views.py b/web/services/views.py index d11d6aa..0704f45 100644 --- a/web/services/views.py +++ b/web/services/views.py @@ -346,7 +346,7 @@ def query(request, format=None): return response.Response(res) def get_parameter(request, param): - val = request.POST.get[param] + val = request.POST.get(param) err = "" if not val: err = f"No {param} parameter provided" From 21056789d5700b642fe5eab1e81bd1300c684e87 Mon Sep 17 00:00:00 2001 From: Mial Lewis Date: Fri, 30 Jan 2026 14:44:58 +0000 Subject: [PATCH 04/20] improve security first pass --- nginx/confs/mantidreports.conf | 18 ++++++++++++++++++ web/services/views.py | 29 +++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/nginx/confs/mantidreports.conf b/nginx/confs/mantidreports.conf index ef494e9..9165235 100644 --- a/nginx/confs/mantidreports.conf +++ b/nginx/confs/mantidreports.conf @@ -32,4 +32,22 @@ server { proxy_read_timeout 60s; } + location /api/query { + # allow ISIS VPN traffic + allow 130.246.180.13/32; + allow 130.246.180.101/32; + allow 130.246.186.163/32; + allow 130.246.223.126/32; + deny all; + + proxy_pass http://web:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 60s; + proxy_send_timeout 60s; + proxy_read_timeout 60s; + } } diff --git a/web/services/views.py b/web/services/views.py index 0704f45..19f918e 100644 --- a/web/services/views.py +++ b/web/services/views.py @@ -22,6 +22,7 @@ import hashlib import services.plots as plotsfile from os import environ +import re OS_NAMES = ["Linux", "Windows NT", "Darwin"] UTC = datetime.tzinfo("UTC") @@ -332,19 +333,29 @@ def by_root(request, format=None): @api_view(("POST",)) def query(request, format=None): - sql_err, sql = get_parameter(request, "sql") - token_err, token = get_parameter(request, "token") - if sql_err or token_err: - return response.Response(status=400, data=f"Invalid Parameters: {[x for x in [sql_err, token_err] if x]}") - verified = verify_token(token) - if not verified: + if not verify_token(request): return response.Response(status=401, data="UNAUTHORIZED") + sql_err, sql = get_parameter(request, "sql") + if sql_err: + return response.Response(status=400, data=f"Invalid Parameters: {sql_err}") conn=connections["readonly"] with conn.cursor() as cur: cur.execute(sql) res = cur.fetchall() return response.Response(res) +def get_bearer_token(request): + """ + Expect: Authorization: Bearer + """ + auth = request.headers.get("Authorization", "") + if not auth: + return None + parts = auth.split(None, 1) # ["Bearer", ""] + if len(parts) != 2 or parts[0].lower() != "bearer": + return None + return parts[1].strip() or None + def get_parameter(request, param): val = request.POST.get(param) err = "" @@ -352,8 +363,10 @@ def get_parameter(request, param): err = f"No {param} parameter provided" return err, val -def verify_token(token): - return token==environ["QUERY_SECRET_KEY"] +def verify_token(request) -> bool: + token = get_bearer_token(request) + secret = environ.get("QUERY_SECRET_KEY", "") + return token==secret class FeatureViewSet(viewsets.ModelViewSet): """ From 719ed0c41b2da4e27554a8c37a640d3e06fc9756 Mon Sep 17 00:00:00 2001 From: Mial Lewis Date: Fri, 30 Jan 2026 14:50:45 +0000 Subject: [PATCH 05/20] udpdate IP range --- nginx/confs/mantidreports.conf | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nginx/confs/mantidreports.conf b/nginx/confs/mantidreports.conf index 9165235..a94776a 100644 --- a/nginx/confs/mantidreports.conf +++ b/nginx/confs/mantidreports.conf @@ -34,10 +34,7 @@ server { location /api/query { # allow ISIS VPN traffic - allow 130.246.180.13/32; - allow 130.246.180.101/32; - allow 130.246.186.163/32; - allow 130.246.223.126/32; + allow 130.246.0.0/16; deny all; proxy_pass http://web:8000; From 4bb05c5e438dc8fb7307e8f7a13db1f6d61f30ba Mon Sep 17 00:00:00 2001 From: Mial Lewis Date: Fri, 30 Jan 2026 16:05:51 +0000 Subject: [PATCH 06/20] add realip to cong --- nginx/confs/mantidreports.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nginx/confs/mantidreports.conf b/nginx/confs/mantidreports.conf index a94776a..ce9c645 100644 --- a/nginx/confs/mantidreports.conf +++ b/nginx/confs/mantidreports.conf @@ -32,6 +32,10 @@ server { proxy_read_timeout 60s; } + set_real_ip_from 172.0.17.0/24; + real_ip_header X-Forwarded-For; + real_ip_recursive on; + location /api/query { # allow ISIS VPN traffic allow 130.246.0.0/16; From 752374185825751639801c1d5d4d1fc39b03de49 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 30 Jan 2026 16:08:59 +0000 Subject: [PATCH 07/20] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- web/services/views.py | 10 +++++++--- web/settings.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/web/services/views.py b/web/services/views.py index 19f918e..5fa75d6 100644 --- a/web/services/views.py +++ b/web/services/views.py @@ -22,7 +22,6 @@ import hashlib import services.plots as plotsfile from os import environ -import re OS_NAMES = ["Linux", "Windows NT", "Darwin"] UTC = datetime.tzinfo("UTC") @@ -331,6 +330,7 @@ def by_root(request, format=None): } ) + @api_view(("POST",)) def query(request, format=None): if not verify_token(request): @@ -338,12 +338,13 @@ def query(request, format=None): sql_err, sql = get_parameter(request, "sql") if sql_err: return response.Response(status=400, data=f"Invalid Parameters: {sql_err}") - conn=connections["readonly"] + conn = connections["readonly"] with conn.cursor() as cur: cur.execute(sql) res = cur.fetchall() return response.Response(res) + def get_bearer_token(request): """ Expect: Authorization: Bearer @@ -356,6 +357,7 @@ def get_bearer_token(request): return None return parts[1].strip() or None + def get_parameter(request, param): val = request.POST.get(param) err = "" @@ -363,10 +365,12 @@ def get_parameter(request, param): err = f"No {param} parameter provided" return err, val + def verify_token(request) -> bool: token = get_bearer_token(request) secret = environ.get("QUERY_SECRET_KEY", "") - return token==secret + return token == secret + class FeatureViewSet(viewsets.ModelViewSet): """ diff --git a/web/settings.py b/web/settings.py index f826223..34d7bab 100644 --- a/web/settings.py +++ b/web/settings.py @@ -105,7 +105,7 @@ "OPTIONS": { "options": "-c default_transaction_read_only=on", }, - } + }, } # Internationalization From f0493035d701a4aacf42db0bc7e5d2b45735b1a8 Mon Sep 17 00:00:00 2001 From: Waruna Wickramasingha Date: Fri, 12 Jun 2026 11:43:30 +0100 Subject: [PATCH 08/20] renamed server config file --- docker-compose.yml | 3 +-- nginx/confs/{mantidreports.conf => reports_server.conf} | 0 2 files changed, 1 insertion(+), 2 deletions(-) rename nginx/confs/{mantidreports.conf => reports_server.conf} (100%) diff --git a/docker-compose.yml b/docker-compose.yml index af74954..170c352 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3.2' services: postgres: image: postgres:11.5 @@ -49,7 +48,7 @@ services: - "${HOST_PORT}:80" volumes: - webdata:/usr/src/app - - "./nginx/confs/mantidreports.conf:/opt/bitnami/nginx/conf/server_blocks/mantidreports.conf:ro" + - "./nginx/confs/reports_server.conf:/opt/bitnami/nginx/conf/server_blocks/reports_server.conf:ro" networks: - default - nginx_net diff --git a/nginx/confs/mantidreports.conf b/nginx/confs/reports_server.conf similarity index 100% rename from nginx/confs/mantidreports.conf rename to nginx/confs/reports_server.conf From 46b086c35c7f069a29450aa88db472169062d697 Mon Sep 17 00:00:00 2001 From: Mial Lewis Date: Thu, 29 Jan 2026 14:16:23 +0000 Subject: [PATCH 09/20] add_query_endpoint --- blank.env | 3 +++ docker-compose.yml | 9 --------- web/services/urls.py | 1 + web/services/views.py | 27 +++++++++++++++++++++++++++ web/settings.py | 11 +++++++++++ 5 files changed, 42 insertions(+), 9 deletions(-) diff --git a/blank.env b/blank.env index df82549..0f8f6aa 100644 --- a/blank.env +++ b/blank.env @@ -17,6 +17,9 @@ DB_PORT=5432 SECRET_KEY= DB_USER= DB_PASS= +DB_RO_USER= +DB_RO_PASS= +QUERY_SECRET_KEY= #Trusted origins for CSRF validation. For production usage only specify https://reports.mantidproject.org DJANGO_CSRF_TRUSTED_ORIGINS=http://localhost:8082,https://reports.a.staging-mantidproject.stfc.ac.uk diff --git a/docker-compose.yml b/docker-compose.yml index 170c352..80bec7e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,9 +9,6 @@ services: POSTGRES_USER: ${DB_USER} POSTGRES_PASSWORD: ${DB_PASS} POSTGRES_DB: ${DB_NAME} - DB_SERVICE: ${DB_SERVICE} - DB_PORT: ${DB_PORT} - SECRET_KEY: ${SECRET_KEY} adminer: image: adminer @@ -33,12 +30,6 @@ services: depends_on: - postgres env_file: .env - environment: - DB_SERVICE: ${DB_SERVICE} - DB_PORT: ${DB_PORT} - SECRET_KEY: ${SECRET_KEY} - # Define this in .env for development mode. DO NOT USE IN PRODUCTION - DEBUG: ${DEBUG} nginx-reports: restart: always diff --git a/web/services/urls.py b/web/services/urls.py index 66cfb40..c7075c9 100644 --- a/web/services/urls.py +++ b/web/services/urls.py @@ -17,6 +17,7 @@ path("by/user", views.usage_by_users, name="by-users"), path("host", views.host_list, name="host-list"), path("user", views.user_list, name="user-list"), + path("query", views.query, name="query"), # url(r'feature', views.feature_usage, name='feature_usage'), ] diff --git a/web/services/views.py b/web/services/views.py index 52b0676..6c42ef0 100644 --- a/web/services/views.py +++ b/web/services/views.py @@ -15,10 +15,13 @@ import django_filters from rest_framework.reverse import reverse from django.http import HttpResponse +from django.db import connections + import json import datetime import hashlib import services.plots as plotsfile +from os import environ OS_NAMES = ["Linux", "Windows NT", "Darwin"] UTC = datetime.tzinfo("UTC") @@ -327,6 +330,30 @@ def by_root(request, format=None): } ) +@api_view(("POST",)) +def query(request, format=None): + sql_err, sql = get_parameter(request, "sql") + token_err, token = get_parameter(request, "token") + if sql_err or token_err: + return response.Response(status=400, data=f"Invalid Parameters: {[x for x in [sql_err, token_err] if x]}") + verified = verify_token(token) + if not verified: + return response.Response(status=401, data="UNAUTHORIZED") + conn=connections["readonly"] + with conn.cursor() as cur: + cur.execute(sql) + res = cur.fetchall() + return response.Response(res) + +def get_parameter(request, param): + val = request.POST.get[param] + err = "" + if not val: + err = f"No {param} parameter provided" + return err, val + +def verify_token(token): + return token==environ["QUERY_SECRET_KEY"] class FeatureViewSet(viewsets.ModelViewSet): """ diff --git a/web/settings.py b/web/settings.py index a5b2213..f826223 100644 --- a/web/settings.py +++ b/web/settings.py @@ -94,6 +94,17 @@ "PASSWORD": os.environ["DB_PASS"], "HOST": os.environ["DB_SERVICE"], "PORT": os.environ["DB_PORT"], + }, + "readonly": { + "ENGINE": "django.db.backends.postgresql_psycopg2", + "NAME": os.environ["DB_NAME"], + "USER": os.environ["DB_RO_USER"], + "PASSWORD": os.environ["DB_RO_PASS"], + "HOST": os.environ["DB_SERVICE"], + "PORT": os.environ["DB_PORT"], + "OPTIONS": { + "options": "-c default_transaction_read_only=on", + }, } } From 430b52e5d53b046746926413791a80376d1ef81f Mon Sep 17 00:00:00 2001 From: Mial Lewis Date: Thu, 29 Jan 2026 16:25:38 +0000 Subject: [PATCH 10/20] add reload to debug --- web/run_django.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/run_django.sh b/web/run_django.sh index 5a52ae3..b36ddb8 100755 --- a/web/run_django.sh +++ b/web/run_django.sh @@ -8,7 +8,7 @@ python manage.py migrate --noinput # If running in DEBUG mode add debug logging to gunicorn if [ -n "${DEBUG}" ]; then - DEBUG_ARGS="--log-level debug --capture-output" + DEBUG_ARGS="--log-level debug --capture-output --reload" else DEBUG_ARGS= fi From d6f45a2c9f250de19a0a4acf455e7bd4dcf2ee6a Mon Sep 17 00:00:00 2001 From: Mial Lewis Date: Thu, 29 Jan 2026 16:59:19 +0000 Subject: [PATCH 11/20] correct typo --- web/services/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/services/views.py b/web/services/views.py index 6c42ef0..6b42754 100644 --- a/web/services/views.py +++ b/web/services/views.py @@ -346,7 +346,7 @@ def query(request, format=None): return response.Response(res) def get_parameter(request, param): - val = request.POST.get[param] + val = request.POST.get(param) err = "" if not val: err = f"No {param} parameter provided" From a0e0d562e4050a00cdd8c42a68a59229b7cd4266 Mon Sep 17 00:00:00 2001 From: Mial Lewis Date: Fri, 30 Jan 2026 14:44:58 +0000 Subject: [PATCH 12/20] improve security first pass --- nginx/confs/reports_server.conf | 18 ++++++++++++++++++ web/services/views.py | 29 +++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/nginx/confs/reports_server.conf b/nginx/confs/reports_server.conf index ef494e9..9165235 100644 --- a/nginx/confs/reports_server.conf +++ b/nginx/confs/reports_server.conf @@ -32,4 +32,22 @@ server { proxy_read_timeout 60s; } + location /api/query { + # allow ISIS VPN traffic + allow 130.246.180.13/32; + allow 130.246.180.101/32; + allow 130.246.186.163/32; + allow 130.246.223.126/32; + deny all; + + proxy_pass http://web:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 60s; + proxy_send_timeout 60s; + proxy_read_timeout 60s; + } } diff --git a/web/services/views.py b/web/services/views.py index 6b42754..8b2d218 100644 --- a/web/services/views.py +++ b/web/services/views.py @@ -22,6 +22,7 @@ import hashlib import services.plots as plotsfile from os import environ +import re OS_NAMES = ["Linux", "Windows NT", "Darwin"] UTC = datetime.tzinfo("UTC") @@ -332,19 +333,29 @@ def by_root(request, format=None): @api_view(("POST",)) def query(request, format=None): - sql_err, sql = get_parameter(request, "sql") - token_err, token = get_parameter(request, "token") - if sql_err or token_err: - return response.Response(status=400, data=f"Invalid Parameters: {[x for x in [sql_err, token_err] if x]}") - verified = verify_token(token) - if not verified: + if not verify_token(request): return response.Response(status=401, data="UNAUTHORIZED") + sql_err, sql = get_parameter(request, "sql") + if sql_err: + return response.Response(status=400, data=f"Invalid Parameters: {sql_err}") conn=connections["readonly"] with conn.cursor() as cur: cur.execute(sql) res = cur.fetchall() return response.Response(res) +def get_bearer_token(request): + """ + Expect: Authorization: Bearer + """ + auth = request.headers.get("Authorization", "") + if not auth: + return None + parts = auth.split(None, 1) # ["Bearer", ""] + if len(parts) != 2 or parts[0].lower() != "bearer": + return None + return parts[1].strip() or None + def get_parameter(request, param): val = request.POST.get(param) err = "" @@ -352,8 +363,10 @@ def get_parameter(request, param): err = f"No {param} parameter provided" return err, val -def verify_token(token): - return token==environ["QUERY_SECRET_KEY"] +def verify_token(request) -> bool: + token = get_bearer_token(request) + secret = environ.get("QUERY_SECRET_KEY", "") + return token==secret class FeatureViewSet(viewsets.ModelViewSet): """ From 5399c6c05f03f0b14903d06d782a1d82ca8bb117 Mon Sep 17 00:00:00 2001 From: Mial Lewis Date: Fri, 30 Jan 2026 14:50:45 +0000 Subject: [PATCH 13/20] udpdate IP range --- nginx/confs/reports_server.conf | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nginx/confs/reports_server.conf b/nginx/confs/reports_server.conf index 9165235..a94776a 100644 --- a/nginx/confs/reports_server.conf +++ b/nginx/confs/reports_server.conf @@ -34,10 +34,7 @@ server { location /api/query { # allow ISIS VPN traffic - allow 130.246.180.13/32; - allow 130.246.180.101/32; - allow 130.246.186.163/32; - allow 130.246.223.126/32; + allow 130.246.0.0/16; deny all; proxy_pass http://web:8000; From 3adeb69f36d32092f3ec73edcb03df31be848f33 Mon Sep 17 00:00:00 2001 From: Mial Lewis Date: Fri, 30 Jan 2026 16:05:51 +0000 Subject: [PATCH 14/20] add realip to cong --- nginx/confs/reports_server.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nginx/confs/reports_server.conf b/nginx/confs/reports_server.conf index a94776a..ce9c645 100644 --- a/nginx/confs/reports_server.conf +++ b/nginx/confs/reports_server.conf @@ -32,6 +32,10 @@ server { proxy_read_timeout 60s; } + set_real_ip_from 172.0.17.0/24; + real_ip_header X-Forwarded-For; + real_ip_recursive on; + location /api/query { # allow ISIS VPN traffic allow 130.246.0.0/16; From 051353a0ccd16e4c9a41b8fbdc66e4b68d71b614 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 30 Jan 2026 16:08:59 +0000 Subject: [PATCH 15/20] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- web/services/views.py | 10 +++++++--- web/settings.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/web/services/views.py b/web/services/views.py index 8b2d218..5674c8e 100644 --- a/web/services/views.py +++ b/web/services/views.py @@ -22,7 +22,6 @@ import hashlib import services.plots as plotsfile from os import environ -import re OS_NAMES = ["Linux", "Windows NT", "Darwin"] UTC = datetime.tzinfo("UTC") @@ -331,6 +330,7 @@ def by_root(request, format=None): } ) + @api_view(("POST",)) def query(request, format=None): if not verify_token(request): @@ -338,12 +338,13 @@ def query(request, format=None): sql_err, sql = get_parameter(request, "sql") if sql_err: return response.Response(status=400, data=f"Invalid Parameters: {sql_err}") - conn=connections["readonly"] + conn = connections["readonly"] with conn.cursor() as cur: cur.execute(sql) res = cur.fetchall() return response.Response(res) + def get_bearer_token(request): """ Expect: Authorization: Bearer @@ -356,6 +357,7 @@ def get_bearer_token(request): return None return parts[1].strip() or None + def get_parameter(request, param): val = request.POST.get(param) err = "" @@ -363,10 +365,12 @@ def get_parameter(request, param): err = f"No {param} parameter provided" return err, val + def verify_token(request) -> bool: token = get_bearer_token(request) secret = environ.get("QUERY_SECRET_KEY", "") - return token==secret + return token == secret + class FeatureViewSet(viewsets.ModelViewSet): """ diff --git a/web/settings.py b/web/settings.py index f826223..34d7bab 100644 --- a/web/settings.py +++ b/web/settings.py @@ -105,7 +105,7 @@ "OPTIONS": { "options": "-c default_transaction_read_only=on", }, - } + }, } # Internationalization From 6767f12c93ab4d143afac9d7f8cd04ed05ecefe1 Mon Sep 17 00:00:00 2001 From: Waruna Wickramasingha Date: Fri, 17 Jul 2026 18:03:22 +0100 Subject: [PATCH 16/20] rebased to main and to be tested --- nginx/confs/reports_server.conf | 6 +----- web/services/views.py | 33 +++++++++++++++++++-------------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/nginx/confs/reports_server.conf b/nginx/confs/reports_server.conf index ce9c645..2494d23 100644 --- a/nginx/confs/reports_server.conf +++ b/nginx/confs/reports_server.conf @@ -32,13 +32,9 @@ server { proxy_read_timeout 60s; } - set_real_ip_from 172.0.17.0/24; - real_ip_header X-Forwarded-For; - real_ip_recursive on; - location /api/query { # allow ISIS VPN traffic - allow 130.246.0.0/16; + allow ; deny all; proxy_pass http://web:8000; diff --git a/web/services/views.py b/web/services/views.py index 5674c8e..781f593 100644 --- a/web/services/views.py +++ b/web/services/views.py @@ -3,7 +3,7 @@ # Create your views here. from django.views.decorators.cache import cache_page from services.models import Message, Usage, FeatureUsage, Location -from rest_framework import response, viewsets +from rest_framework import response, viewsets, status from rest_framework.decorators import api_view from rest_framework.permissions import IsAuthenticatedOrReadOnly, AllowAny from services.serializer import ( @@ -22,6 +22,7 @@ import hashlib import services.plots as plotsfile from os import environ +from hmac import compare_digest OS_NAMES = ["Linux", "Windows NT", "Darwin"] UTC = datetime.tzinfo("UTC") @@ -334,15 +335,20 @@ def by_root(request, format=None): @api_view(("POST",)) def query(request, format=None): if not verify_token(request): - return response.Response(status=401, data="UNAUTHORIZED") + return response.Response(status=status.HTTP_401_UNAUTHORIZED, data="UNAUTHORIZED") + sql_err, sql = get_parameter(request, "sql") if sql_err: - return response.Response(status=400, data=f"Invalid Parameters: {sql_err}") - conn = connections["readonly"] - with conn.cursor() as cur: - cur.execute(sql) - res = cur.fetchall() - return response.Response(res) + return response.Response(status=status.HTTP_400_BAD_REQUEST, data=f"Invalid Parameters: {sql_err}") + + try: + conn = connections["readonly"] + with conn.cursor() as cur: + cur.execute(sql) + res = cur.fetchall() + return response.Response(res) + except Exception: + return response.Response({"error": "Query failed"}, status=status.HTTP_400_BAD_REQUEST) def get_bearer_token(request): @@ -360,17 +366,16 @@ def get_bearer_token(request): def get_parameter(request, param): val = request.POST.get(param) - err = "" - if not val: - err = f"No {param} parameter provided" - return err, val + if val is None or val.strip() == "": + return f"No {param} parameter provided", None + return None, val def verify_token(request) -> bool: token = get_bearer_token(request) secret = environ.get("QUERY_SECRET_KEY", "") - return token == secret - + return compare_digest(token, secret) + class FeatureViewSet(viewsets.ModelViewSet): """ From a7948eac1d8573134f23956c0aaefd76c409ed65 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 17:05:51 +0000 Subject: [PATCH 17/20] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- web/services/views.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/web/services/views.py b/web/services/views.py index 781f593..c9190c7 100644 --- a/web/services/views.py +++ b/web/services/views.py @@ -335,12 +335,16 @@ def by_root(request, format=None): @api_view(("POST",)) def query(request, format=None): if not verify_token(request): - return response.Response(status=status.HTTP_401_UNAUTHORIZED, data="UNAUTHORIZED") - + return response.Response( + status=status.HTTP_401_UNAUTHORIZED, data="UNAUTHORIZED" + ) + sql_err, sql = get_parameter(request, "sql") if sql_err: - return response.Response(status=status.HTTP_400_BAD_REQUEST, data=f"Invalid Parameters: {sql_err}") - + return response.Response( + status=status.HTTP_400_BAD_REQUEST, data=f"Invalid Parameters: {sql_err}" + ) + try: conn = connections["readonly"] with conn.cursor() as cur: @@ -348,7 +352,9 @@ def query(request, format=None): res = cur.fetchall() return response.Response(res) except Exception: - return response.Response({"error": "Query failed"}, status=status.HTTP_400_BAD_REQUEST) + return response.Response( + {"error": "Query failed"}, status=status.HTTP_400_BAD_REQUEST + ) def get_bearer_token(request): @@ -375,7 +381,7 @@ def verify_token(request) -> bool: token = get_bearer_token(request) secret = environ.get("QUERY_SECRET_KEY", "") return compare_digest(token, secret) - + class FeatureViewSet(viewsets.ModelViewSet): """ From 0880502a10be33f4e0299f80f21e310f61f90c6f Mon Sep 17 00:00:00 2001 From: Waruna Wickramasingha Date: Mon, 20 Jul 2026 13:45:13 +0100 Subject: [PATCH 18/20] added logs --- web/services/views.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/web/services/views.py b/web/services/views.py index 781f593..ada486b 100644 --- a/web/services/views.py +++ b/web/services/views.py @@ -23,6 +23,8 @@ import services.plots as plotsfile from os import environ from hmac import compare_digest +import logging +logger = logging.getLogger(__name__) OS_NAMES = ["Linux", "Windows NT", "Darwin"] UTC = datetime.tzinfo("UTC") @@ -335,12 +337,18 @@ def by_root(request, format=None): @api_view(("POST",)) def query(request, format=None): if not verify_token(request): + logger.warning("Unauthorized query attempt") return response.Response(status=status.HTTP_401_UNAUTHORIZED, data="UNAUTHORIZED") - sql_err, sql = get_parameter(request, "sql") - if sql_err: - return response.Response(status=status.HTTP_400_BAD_REQUEST, data=f"Invalid Parameters: {sql_err}") + param_err, sql = get_parameter(request, "sql") + if param_err: + logger.warning(f"Invalid query parameters: {param_err}") + return response.Response(status=status.HTTP_400_BAD_REQUEST, data=f"Invalid Parameters: {param_err}") + if not sql: + logger.warning("No sql parameter provided") + return response.Response(status=status.HTTP_400_BAD_REQUEST, data="No sql parameter provided") + try: conn = connections["readonly"] with conn.cursor() as cur: @@ -348,6 +356,7 @@ def query(request, format=None): res = cur.fetchall() return response.Response(res) except Exception: + logger.exception("Query execution failed") return response.Response({"error": "Query failed"}, status=status.HTTP_400_BAD_REQUEST) @@ -357,9 +366,11 @@ def get_bearer_token(request): """ auth = request.headers.get("Authorization", "") if not auth: + logger.warning("No Authorization header provided") return None parts = auth.split(None, 1) # ["Bearer", ""] if len(parts) != 2 or parts[0].lower() != "bearer": + logger.warning("Invalid Authorization header format") return None return parts[1].strip() or None @@ -367,6 +378,7 @@ def get_bearer_token(request): def get_parameter(request, param): val = request.POST.get(param) if val is None or val.strip() == "": + logger.warning(f"No {param} parameter provided") return f"No {param} parameter provided", None return None, val @@ -374,6 +386,9 @@ def get_parameter(request, param): def verify_token(request) -> bool: token = get_bearer_token(request) secret = environ.get("QUERY_SECRET_KEY", "") + if not token or not secret: + logger.warning("Missing token or secret") + return False return compare_digest(token, secret) From 3643dc883137071b74232c5d88499dd67eea0658 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 20 Jul 2026 12:47:13 +0000 Subject: [PATCH 19/20] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- web/services/views.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/web/services/views.py b/web/services/views.py index 3a584ff..4e92fad 100644 --- a/web/services/views.py +++ b/web/services/views.py @@ -24,6 +24,7 @@ from os import environ from hmac import compare_digest import logging + logger = logging.getLogger(__name__) OS_NAMES = ["Linux", "Windows NT", "Darwin"] @@ -338,16 +339,22 @@ def by_root(request, format=None): def query(request, format=None): if not verify_token(request): logger.warning("Unauthorized query attempt") - return response.Response(status=status.HTTP_401_UNAUTHORIZED, data="UNAUTHORIZED") - + return response.Response( + status=status.HTTP_401_UNAUTHORIZED, data="UNAUTHORIZED" + ) + param_err, sql = get_parameter(request, "sql") if param_err: logger.warning(f"Invalid query parameters: {param_err}") - return response.Response(status=status.HTTP_400_BAD_REQUEST, data=f"Invalid Parameters: {param_err}") - + return response.Response( + status=status.HTTP_400_BAD_REQUEST, data=f"Invalid Parameters: {param_err}" + ) + if not sql: logger.warning("No sql parameter provided") - return response.Response(status=status.HTTP_400_BAD_REQUEST, data="No sql parameter provided") + return response.Response( + status=status.HTTP_400_BAD_REQUEST, data="No sql parameter provided" + ) try: conn = connections["readonly"] @@ -357,7 +364,9 @@ def query(request, format=None): return response.Response(res) except Exception: logger.exception("Query execution failed") - return response.Response({"error": "Query failed"}, status=status.HTTP_400_BAD_REQUEST) + return response.Response( + {"error": "Query failed"}, status=status.HTTP_400_BAD_REQUEST + ) def get_bearer_token(request): From 2c1b1d4c12094ddced38a4669e068eb1d81482c2 Mon Sep 17 00:00:00 2001 From: Waruna Wickramasingha Date: Mon, 20 Jul 2026 16:39:13 +0100 Subject: [PATCH 20/20] logs updated --- web/services/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/web/services/views.py b/web/services/views.py index 3a584ff..41b09a8 100644 --- a/web/services/views.py +++ b/web/services/views.py @@ -378,7 +378,6 @@ def get_bearer_token(request): def get_parameter(request, param): val = request.POST.get(param) if val is None or val.strip() == "": - logger.warning(f"No {param} parameter provided") return f"No {param} parameter provided", None return None, val