From f648ce3da24e64865aa358af078414aec96e9344 Mon Sep 17 00:00:00 2001 From: Andy Chosak Date: Tue, 21 Jul 2026 14:50:19 -0400 Subject: [PATCH] Optimize pagination query The search API response includes a set of "break points" which can be used as pagination offsets in subsequent queries. These break points are always computed if the search response would contain more than one page of results. Break points are computed via a second OpenSearch query. Currently, that second query is essentially a duplicate of the original search, which means that includes all of its bells and whistles: computing aggregations, highlighting results, etc. We don't actually need any of that stuff, and it's very expensive to compute. The second search can and should only be used to return break points. This change modifies that second search to remove those extraneous features from the OpenSearch query. --- complaint_search/es_interface.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/complaint_search/es_interface.py b/complaint_search/es_interface.py index 59a32ce..7e609e1 100644 --- a/complaint_search/es_interface.py +++ b/complaint_search/es_interface.py @@ -305,6 +305,14 @@ def search(agg_exclude=None, **kwargs): if hit_total and hit_total > user_batch_size: # We have more than one page of results and need pagination pagination_body = copy.deepcopy(body) + + # When determining break points, we don't need to recompute + # aggregations, re-highlight, or return result source. + pagination_body.pop("aggs", None) + pagination_body.pop("highlight", None) + pagination_body["_source"] = False + pagination_body["track_total_hits"] = False + # cleaner to get page from frontend, but 'frm' works for now page = params.get("frm", user_batch_size) / user_batch_size pagination_body["size"] = get_pagination_query_size(