From b91be3cfbd01fb5fdc5db8e8d1b1848604553014 Mon Sep 17 00:00:00 2001 From: Gutts-n <57202549+Gutts-n@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:50:48 -0300 Subject: [PATCH 1/2] Add request_source field (#145) --- ckanext/analytics/event.py | 10 ++++++++++ ckanext/analytics/tests/test_event.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/ckanext/analytics/event.py b/ckanext/analytics/event.py index 7aa8af7..7fd3277 100644 --- a/ckanext/analytics/event.py +++ b/ckanext/analytics/event.py @@ -140,6 +140,11 @@ class RequestEvent: REAL_IP_HEADER = "X-Real-IP" FORWARDED_FOR_HEADER = "X-Forwarded-For" + #: Set by known internal callers (e.g. the data explorer sends + #: ``data-explorer``) so usage reporting can tell UI-driven traffic apart + #: from genuine external API use. Absent on ordinary calls. + REQUEST_SOURCE_HEADER = "Request-Source" + def __init__(self, request: Any, response: Any, attribution: Any = None) -> None: self.request = request self.response = response @@ -179,6 +184,7 @@ def as_dict(self) -> dict[str, Any]: "user_agent": self.request.user_agent.string or None, "request_ip": self.request_ip, "user": self.user, + "request_source": self.request_source, **{kind: entities.get(kind) for kind in ENTITIES}, } @@ -245,6 +251,10 @@ def request_ip(self) -> str | None: return self.request.remote_addr + @property + def request_source(self) -> str | None: + return self.request.headers.get(self.REQUEST_SOURCE_HEADER) or None + def params(self) -> dict[str, Any]: """Entity references the caller sent, wherever they put them. diff --git a/ckanext/analytics/tests/test_event.py b/ckanext/analytics/tests/test_event.py index 757abda..d7ebe7d 100644 --- a/ckanext/analytics/tests/test_event.py +++ b/ckanext/analytics/tests/test_event.py @@ -28,6 +28,7 @@ "user_agent", "request_ip", "user", + "request_source", "dataset", "resource", "organization", @@ -215,6 +216,20 @@ def test_the_ip_falls_back_to_the_peer_for_a_direct_call(client, recorded): assert recorded[0]["request_ip"] == "127.0.0.1" +def test_request_source_is_recorded_when_the_header_is_sent(client, recorded): + client.get( + "/api/3/action/package_show", headers={"Request-Source": "data-explorer"} + ) + + assert recorded[0]["request_source"] == "data-explorer" + + +def test_request_source_is_none_when_the_header_is_absent(client, recorded): + client.get("/api/3/action/package_show") + + assert recorded[0]["request_source"] is None + + def test_user_agent_is_none_when_the_client_sends_none(client, recorded): client.get("/api/3/action/package_show", headers={"User-Agent": ""}) From 833dfaf34ad29a583be737a75d231ee1760d6ba9 Mon Sep 17 00:00:00 2001 From: Gutts-n <57202549+Gutts-n@users.noreply.github.com> Date: Thu, 27 Aug 2026 08:10:20 -0300 Subject: [PATCH 2/2] Ignore analytics for configured header (#152) --- ckanext/analytics/event.py | 36 ++++++++++++++++------ ckanext/analytics/tests/test_event.py | 44 ++++++++++++++++++++++----- 2 files changed, 62 insertions(+), 18 deletions(-) diff --git a/ckanext/analytics/event.py b/ckanext/analytics/event.py index 7fd3277..e7ac953 100644 --- a/ckanext/analytics/event.py +++ b/ckanext/analytics/event.py @@ -10,6 +10,10 @@ which is why this hangs off ``request_finished`` and not ``action_succeeded`` - the latter fires only on success, so every 403, 409 and 500 would be missing. +A request carrying the configured ignore header/value (``CKANEXT_ANALYTICS_IGNORE_HEADER`` +/ ``CKANEXT_ANALYTICS_IGNORE_VALUES``) is skipped entirely - not logged with a +distinguishing field, just never recorded. + ``Attribution`` holds the rules, ``RequestEvent`` turns a request into a dict, and ``record_request`` is the listener CKAN calls. Flask only, no CKAN import, so the whole lifecycle can be driven against a stub app and the real signal. @@ -36,6 +40,18 @@ #: Named in every event, so one stream can carry more than one service. SERVICE = os.environ.get("CKANEXT_ANALYTICS_SERVICE", "ckan") +#: A request whose ``IGNORE_HEADER`` value is in ``IGNORE_VALUES`` skips +#: analytics entirely - not logged with a distinguishing field, just never +#: recorded. Known internal callers (e.g. the data explorer) set this so +#: their UI-driven traffic never counts as API usage. Either empty disables +#: the check. Comma-separated, matched case-insensitively. +IGNORE_HEADER = os.environ.get("CKANEXT_ANALYTICS_IGNORE_HEADER", "") +IGNORE_VALUES = frozenset( + v.strip().lower() + for v in os.environ.get("CKANEXT_ANALYTICS_IGNORE_VALUES", "").split(",") + if v.strip() +) + class Attribution: """Which entity an action refers to, and in which parameter. @@ -140,11 +156,6 @@ class RequestEvent: REAL_IP_HEADER = "X-Real-IP" FORWARDED_FOR_HEADER = "X-Forwarded-For" - #: Set by known internal callers (e.g. the data explorer sends - #: ``data-explorer``) so usage reporting can tell UI-driven traffic apart - #: from genuine external API use. Absent on ordinary calls. - REQUEST_SOURCE_HEADER = "Request-Source" - def __init__(self, request: Any, response: Any, attribution: Any = None) -> None: self.request = request self.response = response @@ -157,8 +168,18 @@ def from_request(cls, request: Any, response: Any) -> RequestEvent | None: request.endpoint ): return None + if cls.is_ignored(request): + return None return cls(request, response) + @staticmethod + def is_ignored(request: Any) -> bool: + """Whether ``IGNORE_HEADER`` / ``IGNORE_VALUES`` say to skip this request.""" + if not IGNORE_HEADER or not IGNORE_VALUES: + return False + value = request.headers.get(IGNORE_HEADER) + return value is not None and value.strip().lower() in IGNORE_VALUES + @staticmethod def is_download_endpoint(endpoint: str | None) -> bool: """A resource blueprint's download view, whoever provides it. @@ -184,7 +205,6 @@ def as_dict(self) -> dict[str, Any]: "user_agent": self.request.user_agent.string or None, "request_ip": self.request_ip, "user": self.user, - "request_source": self.request_source, **{kind: entities.get(kind) for kind in ENTITIES}, } @@ -251,10 +271,6 @@ def request_ip(self) -> str | None: return self.request.remote_addr - @property - def request_source(self) -> str | None: - return self.request.headers.get(self.REQUEST_SOURCE_HEADER) or None - def params(self) -> dict[str, Any]: """Entity references the caller sent, wherever they put them. diff --git a/ckanext/analytics/tests/test_event.py b/ckanext/analytics/tests/test_event.py index d7ebe7d..dec0fb7 100644 --- a/ckanext/analytics/tests/test_event.py +++ b/ckanext/analytics/tests/test_event.py @@ -28,7 +28,6 @@ "user_agent", "request_ip", "user", - "request_source", "dataset", "resource", "organization", @@ -216,18 +215,47 @@ def test_the_ip_falls_back_to_the_peer_for_a_direct_call(client, recorded): assert recorded[0]["request_ip"] == "127.0.0.1" -def test_request_source_is_recorded_when_the_header_is_sent(client, recorded): - client.get( - "/api/3/action/package_show", headers={"Request-Source": "data-explorer"} - ) +def test_a_request_with_the_ignore_header_is_not_recorded(client, recorded, monkeypatch): + monkeypatch.setattr(event, "IGNORE_HEADER", "Request-Source") + monkeypatch.setattr(event, "IGNORE_VALUES", frozenset({"data-explorer"})) + + client.get("/api/3/action/package_show", headers={"Request-Source": "data-explorer"}) + + assert recorded == [] + + +def test_the_ignore_header_match_is_case_insensitive(client, recorded, monkeypatch): + monkeypatch.setattr(event, "IGNORE_HEADER", "Request-Source") + monkeypatch.setattr(event, "IGNORE_VALUES", frozenset({"data-explorer"})) + + client.get("/api/3/action/package_show", headers={"request-source": "Data-Explorer"}) + + assert recorded == [] - assert recorded[0]["request_source"] == "data-explorer" +def test_a_request_without_the_ignore_header_is_still_recorded(client, recorded, monkeypatch): + monkeypatch.setattr(event, "IGNORE_HEADER", "Request-Source") + monkeypatch.setattr(event, "IGNORE_VALUES", frozenset({"data-explorer"})) -def test_request_source_is_none_when_the_header_is_absent(client, recorded): client.get("/api/3/action/package_show") - assert recorded[0]["request_source"] is None + assert len(recorded) == 1 + + +def test_a_request_with_a_different_header_value_is_still_recorded(client, recorded, monkeypatch): + monkeypatch.setattr(event, "IGNORE_HEADER", "Request-Source") + monkeypatch.setattr(event, "IGNORE_VALUES", frozenset({"data-explorer"})) + + client.get("/api/3/action/package_show", headers={"Request-Source": "some-other-tool"}) + + assert len(recorded) == 1 + + +def test_the_ignore_check_is_disabled_when_unconfigured(client, recorded): + """Default (module-level IGNORE_HEADER/IGNORE_VALUES empty) never skips.""" + client.get("/api/3/action/package_show", headers={"Request-Source": "data-explorer"}) + + assert len(recorded) == 1 def test_user_agent_is_none_when_the_client_sends_none(client, recorded):