From 64c51cf285da2a701dd8819f007e67351d99a40b Mon Sep 17 00:00:00 2001 From: Betsy Lorton Date: Mon, 27 Jul 2026 13:59:02 -0400 Subject: [PATCH 01/13] Create materialized view of eval results And populate it after evalutors run --- django/evaluate_m2/evaluate.py | 13 +- .../management/commands/materialized_view.py | 32 +++ .../0003_evaluatorresultmaterializedview.py | 99 ++++++++ django/evaluate_m2/models.py | 229 +++++++++++++++++- 4 files changed, 369 insertions(+), 4 deletions(-) create mode 100644 django/evaluate_m2/management/commands/materialized_view.py create mode 100644 django/evaluate_m2/migrations/0003_evaluatorresultmaterializedview.py diff --git a/django/evaluate_m2/evaluate.py b/django/evaluate_m2/evaluate.py index 44ba86f8..b273c30b 100644 --- a/django/evaluate_m2/evaluate.py +++ b/django/evaluate_m2/evaluate.py @@ -6,7 +6,11 @@ from django.utils.module_loading import import_string from evaluate_m2.evaluate_utils import create_eval_insert_query -from evaluate_m2.models import EvaluatorMetadata, EvaluatorResultSummary +from evaluate_m2.models import ( + EvaluatorMetadata, + EvaluatorResultMaterializedView, + EvaluatorResultSummary, +) from evaluate_m2.upload_utils import stream_results_files_to_s3 from parse_m2.models import Metro2Event @@ -36,8 +40,10 @@ def load_evaluators(self): # runs evaluators to produce results def run_evaluators(self, event: Metro2Event): """ - Given an event, run all evaluators on the Account Activity associated - to the event and save the results to the database. + Given an event, run all evaluators on the Account Activity records + associated with the event and save the results to the database. + Then, create or update the evaluator result materialized view, so + the results can be queried. """ logger = logging.getLogger('evaluate.run_evaluators') @@ -46,6 +52,7 @@ def run_evaluators(self, event: Metro2Event): if record_set.exists(): for eval_name, eval_func in self.evaluators.items(): self.run_single_evaluator(event, eval_name, eval_func, record_set) + EvaluatorResultMaterializedView.create_or_refresh_materialized_view() else: logger.info(f"No AccountActivity found for the event '{event.id}'") diff --git a/django/evaluate_m2/management/commands/materialized_view.py b/django/evaluate_m2/management/commands/materialized_view.py new file mode 100644 index 00000000..d3607b56 --- /dev/null +++ b/django/evaluate_m2/management/commands/materialized_view.py @@ -0,0 +1,32 @@ +import logging + +from django.core.management.base import BaseCommand + +from evaluate_m2.models import EvaluatorResultMaterializedView + + +class Command(BaseCommand): + """ + Run this command by running the following: + > python manage.py materialized_view + """ + help = ( + "Creates the materialized view that connects with the " + "EvaluatorResultMaterializedView model, if it doesn't exist. " + "This is what allows the evaluator results view to work. " + "If the materialized view does exist, this command refreshes " + "it to incorporate any new evaluator results." + ) + + + def handle(self, *args, **options): + logger = logging.getLogger('commands.single_evaluator') + + EvaluatorResultMaterializedView.create_or_refresh_materialized_view() + + logger.info( + self.style.SUCCESS( + "Success: Evaluator Result materialized view now exists" + " and contains all evaluator results" + ) + ) \ No newline at end of file diff --git a/django/evaluate_m2/migrations/0003_evaluatorresultmaterializedview.py b/django/evaluate_m2/migrations/0003_evaluatorresultmaterializedview.py new file mode 100644 index 00000000..831a799d --- /dev/null +++ b/django/evaluate_m2/migrations/0003_evaluatorresultmaterializedview.py @@ -0,0 +1,99 @@ +# Generated by Django 5.2.14 on 2026-07-24 18:41 + +import django.core.serializers.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('evaluate_m2', '0002_evaluatormetadata_additional_notes_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='EvaluatorResultMaterializedView', + fields=[ + ('id', models.IntegerField(db_column='id', primary_key=True, serialize=False)), + ('event_id', models.IntegerField(db_column='event_id')), + ('evaluator_id', models.CharField(db_column='evaluator_id')), + ('source_record_id', models.IntegerField(db_column='source_record_id')), + ('activity_date', models.DateField(db_column='activity_date')), + ('cons_acct_num', models.CharField(db_column='cons_acct_num')), + ('port_type', models.CharField(db_column='port_type')), + ('acct_type', models.CharField(db_column='acct_type')), + ('date_open', models.DateField(db_column='date_open')), + ('credit_limit', models.IntegerField(db_column='credit_limit')), + ('hcola', models.IntegerField(db_column='hcola')), + ('id_num', models.CharField(db_column='id_num')), + ('terms_dur', models.CharField(db_column='terms_dur')), + ('terms_freq', models.CharField(db_column='terms_freq')), + ('smpa', models.IntegerField(db_column='smpa')), + ('actual_pmt_amt', models.IntegerField(db_column='actual_pmt_amt')), + ('acct_stat', models.CharField(db_column='acct_stat')), + ('pmt_rating', models.CharField(db_column='pmt_rating')), + ('php', models.CharField(db_column='php')), + ('php1', models.CharField(db_column='php1')), + ('spc_com_cd', models.CharField(db_column='spc_com_cd')), + ('compl_cond_cd', models.CharField(db_column='compl_cond_cd')), + ('current_bal', models.IntegerField(db_column='current_bal')), + ('amt_past_due', models.IntegerField(db_column='amt_past_due')), + ('orig_chg_off_amt', models.IntegerField(db_column='orig_chg_off_amt')), + ('doai', models.DateField(db_column='doai')), + ('dofd', models.DateField(db_column='dofd')), + ('date_closed', models.DateField(db_column='date_closed')), + ('int_type_ind', models.CharField(db_column='int_type_ind')), + ('surname', models.CharField(db_column='surname')), + ('first_name', models.CharField(db_column='first_name')), + ('middle_name', models.CharField(db_column='middle_name')), + ('gen_code', models.CharField(db_column='gen_code')), + ('ssn', models.CharField(db_column='ssn')), + ('dob', models.CharField(db_column='dob')), + ('phone_num', models.CharField(db_column='phone_num')), + ('ecoa', models.CharField(db_column='ecoa')), + ('ecoa_assoc', models.JSONField(db_column='ecoa_assoc', encoder=django.core.serializers.json.DjangoJSONEncoder)), + ('cons_info_ind', models.CharField(db_column='cons_info_ind')), + ('cons_info_ind_assoc', models.JSONField(db_column='cons_info_ind_assoc', encoder=django.core.serializers.json.DjangoJSONEncoder)), + ('addr_line_1', models.CharField(db_column='addr_line_1')), + ('addr_line_2', models.CharField(db_column='addr_line_2')), + ('city', models.CharField(db_column='city')), + ('state', models.CharField(db_column='state')), + ('zip', models.CharField(db_column='zip')), + ('addr_ind', models.CharField(db_column='addr_ind')), + ('res_cd', models.CharField(db_column='res_cd')), + ('purch_sold_ind', models.CharField(db_column='purch_sold_ind')), + ('purch_sold_name', models.CharField(db_column='purch_sold_name')), + ('spc_pmt_ind', models.CharField(db_column='spc_pmt_ind')), + ('deferred_pmt_st_dt', models.DateField(db_column='deferred_pmt_st_dt')), + ('balloon_pmt_due_dt', models.DateField(db_column='balloon_pmt_due_dt')), + ('balloon_pmt_amt', models.IntegerField(db_column='balloon_pmt_amt')), + ('change_ind', models.CharField(db_column='change_ind')), + ('new_acc_num', models.CharField(db_column='new_acc_num')), + ('new_id_num', models.CharField(db_column='new_id_num')), + ('prior_activity_date', models.DateField(db_column='prior_activity_date')), + ('prior_port_type', models.CharField(db_column='prior_port_type')), + ('prior_acct_type', models.CharField(db_column='prior_acct_type')), + ('prior_date_open', models.DateField(db_column='prior_date_open')), + ('prior_id_num', models.CharField(db_column='prior_id_num')), + ('prior_acct_stat', models.CharField(db_column='prior_acct_stat')), + ('prior_pmt_rating', models.CharField(db_column='prior_pmt_rating')), + ('prior_current_bal', models.IntegerField(db_column='prior_current_bal')), + ('prior_orig_chg_off_amt', models.IntegerField(db_column='prior_orig_chg_off_amt')), + ('prior_dofd', models.DateField(db_column='prior_dofd')), + ('prior_date_closed', models.DateField(db_column='prior_date_closed')), + ('prior_surname', models.CharField(db_column='prior_surname')), + ('prior_first_name', models.CharField(db_column='prior_first_name')), + ('prior_ecoa', models.CharField(db_column='prior_ecoa')), + ('prior_ecoa_assoc', models.JSONField(db_column='prior_ecoa_assoc', encoder=django.core.serializers.json.DjangoJSONEncoder)), + ('prior_cons_info_ind', models.CharField(db_column='prior_cons_info_ind')), + ('prior_cons_info_ind_assoc', models.JSONField(db_column='prior_cons_info_ind_assoc', encoder=django.core.serializers.json.DjangoJSONEncoder)), + ('prior_change_ind', models.CharField(db_column='prior_change_ind')), + ('prior_new_acc_num', models.CharField(db_column='prior_new_acc_num')), + ('prior_new_id_num', models.CharField(db_column='prior_new_id_num')), + ], + options={ + 'db_table': 'mv_all_evaluator_results', + 'managed': False, + }, + ), + ] diff --git a/django/evaluate_m2/models.py b/django/evaluate_m2/models.py index 5db517f5..93f67409 100644 --- a/django/evaluate_m2/models.py +++ b/django/evaluate_m2/models.py @@ -1,8 +1,9 @@ +import logging from datetime import date from django.conf import settings from django.core.serializers.json import DjangoJSONEncoder -from django.db import models +from django.db import connection, models from django.db.models import DEFERRED, JSONField from django_prose_editor.fields import ProseEditorField @@ -256,3 +257,229 @@ def create_csv_row_data(self, fields_list: list[str]): self.result_summary.event.name, ] + list(field_values) return response + + +class EvaluatorResultMaterializedView(models.Model): + class Meta: + managed = False + db_table = 'mv_all_evaluator_results' + + table_name = 'mv_all_evaluator_results' + + @classmethod + def create_or_refresh_materialized_view(cls): + lg = logging.getLogger('eval_mv.create_or_refresh_materialized_view') + if cls.materialized_view_exists(): + lg.info("Refreshing the evaluator results materialized view...") + with connection.cursor() as c: + c.execute(cls.refresh_view_command, [cls.table_name]) + else: + lg.info("Creating the evaluator results materialized view...") + with connection.cursor() as c: + c.execute(cls.create_view_command, [cls.table_name]) + c.execute(cls.create_index_command, [cls.table_name]) + lg.info("... Done.") + + @classmethod + def materialized_view_exists(cls) -> bool: + with connection.cursor() as c: + command = "select * from pg_matviews where matviewname = %s" + result = c.execute(command, [cls.table_name]) + return result.rowcount > 0 + + # The fields on this model correspond to the + # column names in the materialized view + id = models.IntegerField(db_column="id", primary_key=True) + event_id = models.IntegerField(db_column="event_id") + evaluator_id = models.CharField(db_column="evaluator_id") + source_record_id = models.IntegerField(db_column="source_record_id") + # account activity fields + activity_date = models.DateField(db_column="activity_date") + cons_acct_num = models.CharField(db_column="cons_acct_num") + port_type = models.CharField(db_column="port_type") + acct_type = models.CharField(db_column="acct_type") + date_open = models.DateField(db_column="date_open") + credit_limit = models.IntegerField(db_column="credit_limit") + hcola = models.IntegerField(db_column="hcola") + id_num = models.CharField(db_column="id_num") + terms_dur = models.CharField(db_column="terms_dur") + terms_freq = models.CharField(db_column="terms_freq") + smpa = models.IntegerField(db_column="smpa") + actual_pmt_amt = models.IntegerField(db_column="actual_pmt_amt") + acct_stat = models.CharField(db_column="acct_stat") + pmt_rating = models.CharField(db_column="pmt_rating") + php = models.CharField(db_column="php") + php1 = models.CharField(db_column="php1") + spc_com_cd = models.CharField(db_column="spc_com_cd") + compl_cond_cd = models.CharField(db_column="compl_cond_cd") + current_bal = models.IntegerField(db_column="current_bal") + amt_past_due = models.IntegerField(db_column="amt_past_due") + orig_chg_off_amt = models.IntegerField(db_column="orig_chg_off_amt") + doai = models.DateField(db_column="doai") + dofd = models.DateField(db_column="dofd") + date_closed = models.DateField(db_column="date_closed") + int_type_ind = models.CharField(db_column="int_type_ind") + surname = models.CharField(db_column="surname") + first_name = models.CharField(db_column="first_name") + middle_name = models.CharField(db_column="middle_name") + gen_code = models.CharField(db_column="gen_code") + ssn = models.CharField(db_column="ssn") + dob = models.CharField(db_column="dob") + phone_num = models.CharField(db_column="phone_num") + ecoa = models.CharField(db_column="ecoa") + ecoa_assoc = models.JSONField( + encoder=DjangoJSONEncoder, db_column="ecoa_assoc") + cons_info_ind = models.CharField(db_column="cons_info_ind") + cons_info_ind_assoc = models.JSONField( + encoder=DjangoJSONEncoder, db_column="cons_info_ind_assoc") + addr_line_1 = models.CharField(db_column="addr_line_1") + addr_line_2 = models.CharField(db_column="addr_line_2") + city = models.CharField(db_column="city") + state = models.CharField(db_column="state") + zip = models.CharField(db_column="zip") + addr_ind = models.CharField(db_column="addr_ind") + res_cd = models.CharField(db_column="res_cd") + # extra segment columns + purch_sold_ind = models.CharField(db_column="purch_sold_ind") + purch_sold_name = models.CharField(db_column="purch_sold_name") + spc_pmt_ind = models.CharField(db_column="spc_pmt_ind") + deferred_pmt_st_dt = models.DateField(db_column="deferred_pmt_st_dt") + balloon_pmt_due_dt = models.DateField(db_column="balloon_pmt_due_dt") + balloon_pmt_amt = models.IntegerField(db_column="balloon_pmt_amt") + change_ind = models.CharField(db_column="change_ind") + new_acc_num = models.CharField(db_column="new_acc_num") + new_id_num = models.CharField(db_column="new_id_num") + # prior record values + prior_activity_date = models.DateField(db_column="prior_activity_date") + prior_port_type = models.CharField(db_column="prior_port_type") + prior_acct_type = models.CharField(db_column="prior_acct_type") + prior_date_open = models.DateField(db_column="prior_date_open") + prior_id_num = models.CharField(db_column="prior_id_num") + prior_acct_stat = models.CharField(db_column="prior_acct_stat") + prior_pmt_rating = models.CharField(db_column="prior_pmt_rating") + prior_current_bal = models.IntegerField(db_column="prior_current_bal") + prior_orig_chg_off_amt = models.IntegerField(db_column="prior_orig_chg_off_amt") + prior_dofd = models.DateField(db_column="prior_dofd") + prior_date_closed = models.DateField(db_column="prior_date_closed") + prior_surname = models.CharField(db_column="prior_surname") + prior_first_name = models.CharField(db_column="prior_first_name") + prior_ecoa = models.CharField(db_column="prior_ecoa") + prior_ecoa_assoc = models.JSONField( + encoder=DjangoJSONEncoder, db_column="prior_ecoa_assoc") + prior_cons_info_ind = models.CharField(db_column="prior_cons_info_ind") + prior_cons_info_ind_assoc = models.JSONField( + encoder=DjangoJSONEncoder, db_column="prior_cons_info_ind_assoc") + prior_change_ind = models.CharField(db_column="prior_change_ind") + prior_new_acc_num = models.CharField(db_column="prior_new_acc_num") + prior_new_id_num = models.CharField(db_column="prior_new_id_num") + + refresh_view_command = """ + REFRESH MATERIALIZED VIEW %s; + """ + create_index_command = """ + CREATE INDEX idx_event_id_evaluator_id ON %s (event_id, evaluator_id); + """ + + # The column names in the materialized view correspond to the + # fields on the model + create_view_command = """ + CREATE MATERIALIZED VIEW %s AS ( + SELECT + r.id, + a.event_id, + s.evaluator_id, + a.id as source_record_id, + + a.activity_date, + a.cons_acct_num, + a.port_type, + a.acct_type, + a.date_open, + a.credit_limit, + a.hcola, + a.id_num, + a.terms_dur, + a.terms_freq, + a.smpa, + a.actual_pmt_amt, + a.acct_stat, + a.pmt_rating, + a.php, + a.php1, + a.spc_com_cd, + a.compl_cond_cd, + a.current_bal, + a.amt_past_due, + a.orig_chg_off_amt, + a.doai, + a.dofd, + a.date_closed, + a.int_type_ind, + a.surname, + a.first_name, + a.middle_name, + a.gen_code, + a.ssn, + a.dob, + a.phone_num, + a.ecoa, + a.ecoa_assoc, + a.cons_info_ind, + a.cons_info_ind_assoc, + a.addr_line_1, + a.addr_line_2, + a.city, + a.state, + a.zip, + a.addr_ind, + a.res_cd, + + k2.purch_sold_ind, + k2.purch_sold_name, + k4.spc_pmt_ind, + k4.deferred_pmt_st_dt, + k4.balloon_pmt_due_dt, + k4.balloon_pmt_amt, + l1.change_ind, + l1.new_acc_num, + l1.new_id_num, + + prev.activity_date as prior_activity_date, + prev.port_type as prior_port_type, + prev.acct_type as prior_acct_type, + prev.date_open as prior_date_open, + prev.id_num as prior_id_num, + prev.acct_stat as prior_acct_stat, + prev.pmt_rating as prior_pmt_rating, + prev.current_bal as prior_current_bal, + prev.orig_chg_off_amt as prior_orig_chg_off_amt, + prev.dofd as prior_dofd, + prev.date_closed as prior_date_closed, + prev.surname as prior_surname, + prev.first_name as prior_first_name, + prev.ecoa as prior_ecoa, + prev.ecoa_assoc as prior_ecoa_assoc, + prev.cons_info_ind as prior_cons_info_ind, + prev.cons_info_ind_assoc as prior_cons_info_ind_assoc, + prev_l1.change_ind as prior_change_ind, + prev_l1.new_acc_num as prior_new_acc_num, + prev_l1.new_id_num as prior_new_id_num + + FROM + parse_m2_accountactivity as a + INNER JOIN evaluate_m2_evaluatorresult as r + on r.source_record_id = a.id + INNER JOIN evaluate_m2_evaluatorresultsummary as s + on r.result_summary_id = s.id + LEFT JOIN parse_m2_k2 as k2 + on a.id = k2.account_activity_id + LEFT JOIN parse_m2_k4 as k4 + on a.id = k4.account_activity_id + LEFT JOIN parse_m2_l1 as l1 + on a.id = l1.account_activity_id + LEFT JOIN parse_m2_accountactivity as prev + on a.previous_values_id = prev.id + LEFT JOIN parse_m2_l1 as prev_l1 + on prev.id = prev_l1.account_activity_id + ); + """ From 979568a1285a408fadca9960aad588b48dc2e8c7 Mon Sep 17 00:00:00 2001 From: Betsy Lorton Date: Mon, 27 Jul 2026 20:13:48 -0400 Subject: [PATCH 02/13] Fix syntax error --- django/evaluate_m2/models.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/django/evaluate_m2/models.py b/django/evaluate_m2/models.py index 93f67409..f1294489 100644 --- a/django/evaluate_m2/models.py +++ b/django/evaluate_m2/models.py @@ -272,12 +272,12 @@ def create_or_refresh_materialized_view(cls): if cls.materialized_view_exists(): lg.info("Refreshing the evaluator results materialized view...") with connection.cursor() as c: - c.execute(cls.refresh_view_command, [cls.table_name]) + c.execute(cls.refresh_view_command) else: lg.info("Creating the evaluator results materialized view...") with connection.cursor() as c: - c.execute(cls.create_view_command, [cls.table_name]) - c.execute(cls.create_index_command, [cls.table_name]) + c.execute(cls.create_view_command) + c.execute(cls.create_index_command) lg.info("... Done.") @classmethod @@ -374,16 +374,16 @@ def materialized_view_exists(cls) -> bool: prior_new_id_num = models.CharField(db_column="prior_new_id_num") refresh_view_command = """ - REFRESH MATERIALIZED VIEW %s; + REFRESH MATERIALIZED VIEW mv_all_evaluator_results; """ create_index_command = """ - CREATE INDEX idx_event_id_evaluator_id ON %s (event_id, evaluator_id); + CREATE INDEX idx_event_id_evaluator_id ON mv_all_evaluator_results (event_id, evaluator_id); """ # The column names in the materialized view correspond to the # fields on the model create_view_command = """ - CREATE MATERIALIZED VIEW %s AS ( + CREATE MATERIALIZED VIEW mv_all_evaluator_results AS ( SELECT r.id, a.event_id, From 93e17aee21097d92df204e487e9fe4f7e2ae6d63 Mon Sep 17 00:00:00 2001 From: Will Barton Date: Mon, 27 Jul 2026 16:15:29 -0400 Subject: [PATCH 03/13] Gracefully error when materialized view doesn't exist --- django/evaluate_m2/exception_utils.py | 19 +++++++++++-------- django/evaluate_m2/views.py | 20 +++++++++++++++++++- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/django/evaluate_m2/exception_utils.py b/django/evaluate_m2/exception_utils.py index 685b78d3..3ba16f73 100644 --- a/django/evaluate_m2/exception_utils.py +++ b/django/evaluate_m2/exception_utils.py @@ -3,6 +3,16 @@ from rest_framework import status +def format_error(status: int, error_string: str, message: str, path: str): + return { + "timestamp": datetime.now(), + "status": status, + "error": error_string, + "message": message, + "path": path, + } + + def get_evaluate_m2_not_found_exception( error_string:str, event_id: str, evaluator_id: str, path: str, acct_num=''): msg = '' @@ -22,12 +32,5 @@ def get_evaluate_m2_not_found_exception( f'Evaluator result does not exist for event ID {event_id} or ' f'evaluator ID {evaluator_id}.' ) - error= { - 'timestamp': datetime.now(), - 'status': status.HTTP_404_NOT_FOUND, - 'error': 'Not Found', - 'message': msg, - 'path': path - } - return error + return format_error(status.HTTP_404_NOT_FOUND, 'Not Found', msg, path) diff --git a/django/evaluate_m2/views.py b/django/evaluate_m2/views.py index f2c5054d..131a2e3a 100644 --- a/django/evaluate_m2/views.py +++ b/django/evaluate_m2/views.py @@ -4,6 +4,7 @@ from datetime import date from django.conf import settings +from django.db import ProgrammingError from django.http import Http404, HttpResponse, StreamingHttpResponse from django.shortcuts import get_list_or_404 @@ -15,7 +16,7 @@ from django_application.s3_utils import s3_session from evaluate_m2 import upload_utils -from evaluate_m2.exception_utils import get_evaluate_m2_not_found_exception +from evaluate_m2.exception_utils import get_evaluate_m2_not_found_exception, format_error from evaluate_m2.filters import EvaluatorResultFilterSet from evaluate_m2.models import ( EvaluatorMetadata, @@ -279,6 +280,23 @@ def get(self, request, *args, **kwargs): ) logger.error(error['message']) return Response(error, status=status.HTTP_404_NOT_FOUND) + except ProgrammingError as e: + # Gracefully handle when the materialized view doesn't exist as a 503. + # It will raise a ProgrammingError, which we introspect to make sure + # it's caused by an UndefinedTable, otherwise we let it raise. + import psycopg + if not isinstance(e.__cause__, psycopg.errors.UndefinedTable): + raise + return Response( + format_error( + 503, + "Not available", + "Materialized view for evaluator results is not available", + request.path + ), + status=503 + ) + return Response(error, status=503) def list(self, request, *args, **kwargs): result_summary = self.get_result_summary() From 56917581cf4e1b57b75c2a04af7c60c91f20c981 Mon Sep 17 00:00:00 2001 From: Will Barton Date: Wed, 29 Jul 2026 09:41:12 -0400 Subject: [PATCH 04/13] Add materialized view serializer and filters This modifies the existing filter `EvaluatorResultFilterSet` to use the materialized view model, and adjusts field names to the model while retaining the API-facing fieldnames. It does the same for a new serializer to serialize the materialized view, mapping materialized view fieldnames to the expected API fieldnames (which use Django __ notation for relations). --- django/evaluate_m2/filters.py | 189 +++++++++++++----------------- django/evaluate_m2/serializers.py | 44 ++++++- 2 files changed, 125 insertions(+), 108 deletions(-) diff --git a/django/evaluate_m2/filters.py b/django/evaluate_m2/filters.py index 61c779e4..92ce0b2a 100644 --- a/django/evaluate_m2/filters.py +++ b/django/evaluate_m2/filters.py @@ -1,6 +1,6 @@ import django_filters.rest_framework -from evaluate_m2.models import EvaluatorResult +from evaluate_m2.models import EvaluatorResultMaterializedView class AnyCharFilter(django_filters.BaseInFilter, django_filters.CharFilter): @@ -20,7 +20,7 @@ def filter(self, qs, value): class EvaluatorResultFilterSet(django_filters.rest_framework.FilterSet): - """This filter set specifies `EvaluatorResult` fields to filter. + """This filter set specifies `EvaluatorResultMaterializedView` fields to filter. Because the fields that we would filter `EvaluatorResult` objects by exist on their `source_record` relation, the fields here simply map the @@ -29,44 +29,44 @@ class EvaluatorResultFilterSet(django_filters.rest_framework.FilterSet): For example, the API might allow filtering `EvaluatorResults` by `acct_stat`, but the Django queryset will for `EvaluatorResults` will need - to be filtered by `source_record__acct_stat`. + to be filtered by `acct_stat`. """ - acct_type = django_filters.CharFilter(field_name="source_record__acct_type") + acct_type = django_filters.CharFilter(field_name="acct_type") acct_stat = AnyCharFilter( - field_name="source_record__acct_stat", + field_name="acct_stat", ) - compl_cond_cd = AnyCharFilter(field_name="source_record__compl_cond_cd") - php = AnyCharFilter(field_name="source_record__php") - php1 = AnyCharFilter(field_name="source_record__php1") - pmt_rating = AnyCharFilter(field_name="source_record__pmt_rating") - spc_com_cd = AnyCharFilter(field_name="source_record__spc_com_cd") - terms_freq = AnyCharFilter(field_name="source_record__terms_freq") - cons_info_ind = AnyCharFilter(field_name="source_record__cons_info_ind") - cons_info_ind_assoc = AnyCharFilter(field_name="source_record__cons_info_ind_assoc") - l1__change_ind = AnyCharFilter(field_name="source_record__l1__change_ind") + compl_cond_cd = AnyCharFilter(field_name="compl_cond_cd") + php = AnyCharFilter(field_name="php") + php1 = AnyCharFilter(field_name="php1") + pmt_rating = AnyCharFilter(field_name="pmt_rating") + spc_com_cd = AnyCharFilter(field_name="spc_com_cd") + terms_freq = AnyCharFilter(field_name="terms_freq") + cons_info_ind = AnyCharFilter(field_name="cons_info_ind") + cons_info_ind_assoc = AnyCharFilter(field_name="cons_info_ind_assoc") + l1__change_ind = AnyCharFilter(field_name="change_ind") # Dates, as a boolean where the date either exists or does not dofd = django_filters.BooleanFilter( - field_name="source_record__dofd", + field_name="dofd", lookup_expr="isnull", exclude=True, ) date_closed = django_filters.BooleanFilter( - field_name="source_record__date_closed", + field_name="date_closed", lookup_expr="isnull", exclude=True, ) # Amounts, as ranges of values with _max and _min fields amt_past_due = django_filters.RangeFilter( - field_name="source_record__amt_past_due", + field_name="amt_past_due", ) current_bal = django_filters.RangeFilter( - field_name="source_record__current_bal", + field_name="current_bal", ) smpa = django_filters.RangeFilter( - field_name="source_record__smpa", + field_name="smpa", ) # Sort ordering filter for all the relevant fields from AccountActivity @@ -74,101 +74,76 @@ class EvaluatorResultFilterSet(django_filters.rest_framework.FilterSet): # to the field name on AccountActivity. sort = django_filters.OrderingFilter( fields=( - ("source_record__activity_date", "activity_date"), - ("source_record__cons_acct_num", "cons_acct_num"), - ("source_record__port_type", "port_type"), - ("source_record__acct_type", "acct_type"), - ("source_record__date_open", "date_open"), - ("source_record__credit_limit", "credit_limit"), - ("source_record__hcola", "hcola"), - ("source_record__id_num", "id_num"), - ("source_record__terms_dur", "terms_dur"), - ("source_record__terms_freq", "terms_freq"), - ("source_record__smpa", "smpa"), - ("source_record__actual_pmt_amt", "actual_pmt_amt"), - ("source_record__acct_stat", "acct_stat"), - ("source_record__pmt_rating", "pmt_rating"), - ("source_record__php", "php"), - ("source_record__php1", "php1"), - ("source_record__spc_com_cd", "spc_com_cd"), - ("source_record__compl_cond_cd", "compl_cond_cd"), - ("source_record__current_bal", "current_bal"), - ("source_record__amt_past_due", "amt_past_due"), - ("source_record__orig_chg_off_amt", "orig_chg_off_amt"), - ("source_record__doai", "doai"), - ("source_record__dofd", "dofd"), - ("source_record__date_closed", "date_closed"), - ("source_record__dolp", "dolp"), - ("source_record__int_type_ind", "int_type_ind"), - ("source_record__cons_info_ind", "cons_info_ind"), - ("source_record__ecoa", "ecoa"), - ("source_record__cons_info_ind_assoc", "cons_info_ind_assoc"), - ("source_record__ecoa_assoc", "ecoa_assoc"), - ("source_record__first_name", "first_name"), - ("source_record__surname", "surname"), - ("source_record__k2__purch_sold_ind", "k2__purch_sold_ind"), - ("source_record__k2__purch_sold_name", "k2__purch_sold_name"), - ("source_record__k4__balloon_pmt_amt", "k4__balloon_pmt_amt"), - ("source_record__l1__change_ind", "l1__change_ind"), - ("source_record__l1__new_id_num", "l1__new_id_num"), - ("source_record__l1__new_acc_num", "l1__new_acc_num"), - ( - "source_record__previous_values__cons_info_ind", - "previous_values__cons_info_ind", - ), + ("activity_date", "activity_date"), + ("cons_acct_num", "cons_acct_num"), + ("port_type", "port_type"), + ("acct_type", "acct_type"), + ("date_open", "date_open"), + ("credit_limit", "credit_limit"), + ("hcola", "hcola"), + ("id_num", "id_num"), + ("terms_dur", "terms_dur"), + ("terms_freq", "terms_freq"), + ("smpa", "smpa"), + ("actual_pmt_amt", "actual_pmt_amt"), + ("acct_stat", "acct_stat"), + ("pmt_rating", "pmt_rating"), + ("php", "php"), + ("php1", "php1"), + ("spc_com_cd", "spc_com_cd"), + ("compl_cond_cd", "compl_cond_cd"), + ("current_bal", "current_bal"), + ("amt_past_due", "amt_past_due"), + ("orig_chg_off_amt", "orig_chg_off_amt"), + ("doai", "doai"), + ("dofd", "dofd"), + ("date_closed", "date_closed"), + ("dolp", "dolp"), + ("int_type_ind", "int_type_ind"), + ("cons_info_ind", "cons_info_ind"), + ("ecoa", "ecoa"), + ("cons_info_ind_assoc", "cons_info_ind_assoc"), + ("ecoa_assoc", "ecoa_assoc"), + ("first_name", "first_name"), + ("surname", "surname"), + + # For all of these that are relations on the original models but + # direct fields on the materialized view, we keep the original + # Django __ related fieldname on the sort parameter side. That's + # the name the front-end knows the field as. + ("purch_sold_ind", "k2__purch_sold_ind"), + ("purch_sold_name", "k2__purch_sold_name"), + ("balloon_pmt_amt", "k4__balloon_pmt_amt"), + ("change_ind", "l1__change_ind"), + ("new_id_num", "l1__new_id_num"), + ("new_acc_num", "l1__new_acc_num"), + ("prior_cons_info_ind", "previous_values__cons_info_ind"), ( - "source_record__previous_values__cons_info_ind_assoc", + "prior_cons_info_ind_assoc", "previous_values__cons_info_ind_assoc", ), - ("source_record__previous_values__ecoa", "previous_values__ecoa"), - ( - "source_record__previous_values__first_name", - "previous_values__first_name", - ), - ("source_record__previous_values__surname", "previous_values__surname"), - ( - "source_record__previous_values__l1__change_ind", - "previous_values__l1__change_ind", - ), - ( - "source_record__previous_values__l1__new_acc_num", - "previous_values__l1__new_acc_num", - ), - ( - "source_record__previous_values__l1__new_id_num", - "previous_values__l1__new_id_num", - ), - ( - "source_record__previous_values__activity_date", - "previous_values__activity_date", - ), - ("source_record__previous_values__port_type", "previous_values__port_type"), - ("source_record__previous_values__acct_type", "previous_values__acct_type"), - ("source_record__previous_values__date_open", "previous_values__date_open"), - ("source_record__previous_values__acct_stat", "previous_values__acct_stat"), - ( - "source_record__previous_values__pmt_rating", - "previous_values__pmt_rating", - ), - ( - "source_record__previous_values__current_bal", - "previous_values__current_bal", - ), - ( - "source_record__previous_values__orig_chg_off_amt", - "previous_values__orig_chg_off_amt", - ), - ("source_record__previous_values__dofd", "previous_values__dofd"), - ( - "source_record__previous_values__date_closed", - "previous_values__date_closed", - ), - ("source_record__previous_values__id_num", "previous_values__id_num"), + ("prior_ecoa", "previous_values__ecoa"), + ("prior_first_name", "previous_values__first_name"), + ("prior_surname", "previous_values__surname"), + ("prior_change_ind", "previous_values__l1__change_ind"), + ("prior_new_acc_num", "previous_values__l1__new_acc_num"), + ("prior_new_id_num", "previous_values__l1__new_id_num"), + ("prior_activity_date", "previous_values__activity_date"), + ("prior_port_type", "previous_values__port_type"), + ("prior_acct_type", "previous_values__acct_type"), + ("prior_date_open", "previous_values__date_open"), + ("prior_acct_stat", "previous_values__acct_stat"), + ("prior_pmt_rating", "previous_values__pmt_rating"), + ("prior_current_bal", "previous_values__current_bal"), + ("prior_orig_chg_off_amt", "previous_values__orig_chg_off_amt"), + ("prior_dofd", "previous_values__dofd"), + ("prior_date_closed", "previous_values__date_closed"), + ("prior_id_num", "previous_values__id_num"), ) ) class Meta: - model = EvaluatorResult + model = EvaluatorResultMaterializedView fields = [ "acct_type", "acct_stat", diff --git a/django/evaluate_m2/serializers.py b/django/evaluate_m2/serializers.py index 1313c251..bc14cacd 100644 --- a/django/evaluate_m2/serializers.py +++ b/django/evaluate_m2/serializers.py @@ -7,7 +7,11 @@ plain_to_code_field_map, ) -from .models import EvaluatorMetadata, EvaluatorResultSummary +from evaluate_m2.models import ( + EvaluatorMetadata, + EvaluatorResultSummary, + EvaluatorResultMaterializedView, +) class EventsViewSerializer(serializers.ModelSerializer): @@ -224,3 +228,41 @@ def validate(self, data): if invalid_fields: raise serializers.ValidationError(f"Invalid field names: {invalid_fields}") return data + + +class EvaluatorResultSerializer(serializers.ModelSerializer): + # Outside of the materialized view, the API serializes related objects + # using Django's __ notation. These fields preserve those fieldnames. + k2__purch_sold_ind = serializers.CharField(source="purch_sold_ind") + k2__purch_sold_name = serializers.CharField(source="purch_sold_name") + k4__spc_pmt_ind = serializers.CharField(source="spc_pmt_ind") + k4__deferred_pmt_st_dt = serializers.CharField(source="deferred_pmt_st_dt") + k4__balloon_pmt_due_dt = serializers.CharField(source="balloon_pmt_due_dt") + k4__balloon_pmt_amt = serializers.CharField(source="balloon_pmt_amt") + l1__change_ind = serializers.CharField(source="change_ind") + l1__new_acc_num = serializers.CharField(source="new_acc_num") + l1__new_id_num = serializers.CharField(source="new_id_num") + previous_value__activity_date = serializers.CharField(source="prior_activity_date") + previous_value__port_type = serializers.CharField(source="prior_port_type") + previous_value__acct_type = serializers.CharField(source="prior_acct_type") + previous_value__date_open = serializers.CharField(source="prior_date_open") + previous_value__id_num = serializers.CharField(source="prior_id_num") + previous_value__acct_stat = serializers.CharField(source="prior_acct_stat") + previous_value__pmt_rating = serializers.CharField(source="prior_pmt_rating") + previous_value__current_bal = serializers.CharField(source="prior_current_bal") + previous_value__orig_chg_off_amt = serializers.CharField(source="prior_orig_chg_off_amt") + previous_value__dofd = serializers.CharField(source="prior_dofd") + previous_value__date_closed = serializers.CharField(source="prior_date_closed") + previous_value__surname = serializers.CharField(source="prior_surname") + previous_value__first_name = serializers.CharField(source="prior_first_name") + previous_value__ecoa = serializers.CharField(source="prior_ecoa") + previous_value__ecoa_assoc = serializers.CharField(source="prior_ecoa_assoc") + previous_value__cons_info_ind = serializers.CharField(source="prior_cons_info_ind") + previous_value__cons_info_ind_assoc = serializers.CharField(source="prior_cons_info_ind_assoc") + previous_value__l1__change_ind = serializers.CharField(source="prior_change_ind") + previous_value__l1__new_acc_num = serializers.CharField(source="prior_new_acc_num") + previous_value__l1__new_id_num = serializers.CharField(source="prior_new_id_num") + + class Meta: + model = EvaluatorResultMaterializedView + fields = '__all__' From 8351c6aa45f9139a8d991c2e7045589d3d523433 Mon Sep 17 00:00:00 2001 From: Will Barton Date: Wed, 29 Jul 2026 09:43:10 -0400 Subject: [PATCH 05/13] Refactor the evaluator view to use materialized view There are a few simplifications the materialized view brings to the EvaluatorResultsView. First, we can get rid of the result summary query method, we can simplfy the queryset, and we can use the serializer on the class directly. I've also broken the sampling out of the list() method into its own method, because I think we can eliminate the S3 sampling entirely at some point with the materialized view. --- django/evaluate_m2/tests/test_views.py | 8 +++ django/evaluate_m2/views.py | 95 ++++++++++++-------------- 2 files changed, 53 insertions(+), 50 deletions(-) diff --git a/django/evaluate_m2/tests/test_views.py b/django/evaluate_m2/tests/test_views.py index 663db4d3..96ebc67e 100644 --- a/django/evaluate_m2/tests/test_views.py +++ b/django/evaluate_m2/tests/test_views.py @@ -6,6 +6,7 @@ EvaluatorMetadata, EvaluatorResult, EvaluatorResultSummary, + EvaluatorResultMaterializedView, ) from evaluate_m2.serializers import EvaluatorMetadataSerializer from evaluate_m2.tests.evaluator_test_helper import acct_record @@ -56,6 +57,8 @@ def setUp(self) -> None: alternate_explanation='Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua', # noqa: E501 ) + EvaluatorResultMaterializedView.create_or_refresh_materialized_view() + def get_account_activity( self, id: int, @@ -182,6 +185,8 @@ def create_activity_data(self, create_zero_hit:bool=False): inconsistency_end=acct_date ) + EvaluatorResultMaterializedView.create_or_refresh_materialized_view() + ######################################## # Tests for Eval Metadata download def test_download_eval_metadata(self): @@ -232,6 +237,7 @@ def test_download_evaluator_results_csv(self): def test_evaluator_results_view(self): self.create_activity_data() response = self.client.get('/api/events/1/evaluator/Status-DOFD-1/') + # the response should be a JSON self.assertEqual(response.status_code, 200) self.assertEqual(response.headers['Content-Type'], 'application/json') @@ -275,6 +281,8 @@ def test_evaluator_results_view_max_20_results(self): self.eval_rs3.sample_ids = [1,3,5,7,9] self.eval_rs3.save() + EvaluatorResultMaterializedView.create_or_refresh_materialized_view() + response = self.client.get('/api/events/1/evaluator/Status-DOFD-6/') # the response should be a JSON self.assertEqual(response.status_code, 200) diff --git a/django/evaluate_m2/views.py b/django/evaluate_m2/views.py index 131a2e3a..58a60217 100644 --- a/django/evaluate_m2/views.py +++ b/django/evaluate_m2/views.py @@ -22,11 +22,13 @@ EvaluatorMetadata, EvaluatorResult, EvaluatorResultSummary, + EvaluatorResultMaterializedView, ) from evaluate_m2.pagination import EvaluatorResultsPaginator from evaluate_m2.serializers import ( EvaluatorMetadataSerializer, EventsViewSerializer, + EvaluatorResultSerializer, ) from evaluate_m2.views_utils import ( get_object, @@ -59,6 +61,7 @@ def download_evaluator_metadata_csv(request): return response + @api_view(('GET',)) def download_evaluator_results_csv(request, event_id, evaluator_id): logger = logging.getLogger('views.download_evaluator_results_csv') @@ -132,6 +135,7 @@ def account_summary_view(request, event_id, account_number): logger.error(error['message']) return Response(error, status=status.HTTP_404_NOT_FOUND) + @api_view(('GET',)) def account_pii_view(request, event_id, account_number): logger = logging.getLogger('views.account_pii_view') @@ -154,6 +158,7 @@ def account_pii_view(request, event_id, account_number): logger.error(error['message']) return Response(error, status=status.HTTP_404_NOT_FOUND) + @api_view() def events_view(request, event_id): logger = logging.getLogger('views.evaluator_results_view') @@ -189,6 +194,7 @@ def events_view(request, event_id): logger.error(error['message']) return Response(error, status=status.HTTP_404_NOT_FOUND) + ########################################### ## Helper methods for eval results when S3_ENABLED == True def fetch_csv_results_from_s3(request, event_id, evaluator_id): @@ -210,6 +216,7 @@ def fetch_csv_results_from_s3(request, event_id, evaluator_id): logger.error(error['message']) return Response(error, status=status.HTTP_404_NOT_FOUND) + def fetch_json_results_from_s3(request, event_id, evaluator_id): logger = logging.getLogger('views.fetch_json_results_from_s3') s3 = s3_session() @@ -227,39 +234,20 @@ def fetch_json_results_from_s3(request, event_id, evaluator_id): class EvaluatorResultsView(generics.ListAPIView): + serializer_class = EvaluatorResultSerializer pagination_class = EvaluatorResultsPaginator filter_backends = [ django_filters.rest_framework.DjangoFilterBackend, ] filterset_class = EvaluatorResultFilterSet - def get_result_summary(self): - # Get the EvaluatorResultSummary object for this event_id and - # evaluator_id. These are queried directly so we can validate that - # they exist and error appropriately if they do not. - event_id = self.kwargs["event_id"] - evaluator_id = self.kwargs["evaluator_id"] - event = Metro2Event.objects.get(id=event_id) - evaluator = EvaluatorMetadata.objects.get(id=evaluator_id) - result_summary = EvaluatorResultSummary.objects.get( - event=event, evaluator=evaluator) - return result_summary - def get_queryset(self): - # Get all EvaluatorResult objects for this event_id and evaluator_id event_id = self.kwargs["event_id"] evaluator_id = self.kwargs["evaluator_id"] - - result_summary = EvaluatorResultSummary.objects.get( - event__id=event_id, evaluator__id=evaluator_id) - - queryset = EvaluatorResult.objects.filter( - result_summary=result_summary, - ).select_related( - "source_record" - ).order_by("source_record__activity_date") - - return queryset + return EvaluatorResultMaterializedView.objects.filter( + event_id=event_id, + evaluator_id=evaluator_id, + ).order_by("activity_date") def get(self, request, *args, **kwargs): # Override the default `get()` so we can error appropriately if @@ -298,42 +286,49 @@ def get(self, request, *args, **kwargs): ) return Response(error, status=503) - def list(self, request, *args, **kwargs): - result_summary = self.get_result_summary() - event = result_summary.event - evaluator = result_summary.evaluator + def get_sample_queryset(self, queryset): + # We need sample ids from the EvaluatorResultSummary + event_id = self.kwargs["event_id"] + evaluator_id = self.kwargs["evaluator_id"] + result_summary = EvaluatorResultSummary.objects.get( + event_id=event_id, + evaluator__id=evaluator_id + ) + sample_ids = result_summary.sample_ids + + if sample_ids and (len(sample_ids) > 0): + queryset = queryset.filter(source_record_id__in=sample_ids) + else: + # OR select a random set of sample length from the queryset + queryset = queryset.order_by("?")[:settings.M2_RESULT_SAMPLE_SIZE] + + return queryset + def list(self, request, *args, **kwargs): # TODO: replace using DRF permissions/check_permissions() + event_id = self.kwargs["event_id"] + event = Metro2Event.objects.get(id=event_id) if not has_permissions_for_request(request, event): return HttpResponse('Unauthorized', status=401) + # Default to sample view view_param = self.request.query_params.get("view", "sample") - # If we're asked for a sample and S3 is enabled, quickly return - # the results from there. - if settings.S3_ENABLED and view_param == "sample": - return fetch_json_results_from_s3(request, event.id, evaluator.id) + if view_param == "sample": + # If we're asked for a sample and S3 is enabled, quickly return + # the results from there. + if settings.S3_ENABLED: + return fetch_json_results_from_s3(request, event.id, evaluator.id) - # Get the result set, performing any filtering as needed - queryset = self.filter_queryset(self.get_queryset()) + # Get a sample queryset + queryset = self.get_sample_queryset(self.get_queryset()) + else: + # Get the full result set, performing any filtering as needed + queryset = self.filter_queryset(self.get_queryset()) - # If we're asked for a sample, filter the result set by sample_ids - if view_param == "sample": - sample_ids = result_summary.sample_ids - if sample_ids and (len(sample_ids) > 0): - queryset = queryset.filter(source_record__id__in=sample_ids) - else: - # OR select a random set of sample length from the queryset - queryset = queryset.order_by("?")[:settings.M2_RESULT_SAMPLE_SIZE] - - # Regardless, paginate the results. + # Paginate and serialize results. # Pagination size should be the same as the sample size, so the # sample view will be exactly one page long. page = self.paginate_queryset(queryset) - paged_records = [result.source_record for result in page] - serializer = AccountActivitySerializer( - paged_records, - include_fields=evaluator.result_summary_fields(), - many=True, - ) + serializer = self.get_serializer(page, many=True) return self.get_paginated_response(serializer.data) From 1fb99cfbb4541eee1556756d78b00e973c051f36 Mon Sep 17 00:00:00 2001 From: Betsy Lorton Date: Wed, 29 Jul 2026 14:26:03 -0400 Subject: [PATCH 06/13] Quick fix for fetching sample results from S3 --- django/evaluate_m2/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django/evaluate_m2/views.py b/django/evaluate_m2/views.py index 58a60217..5289449a 100644 --- a/django/evaluate_m2/views.py +++ b/django/evaluate_m2/views.py @@ -318,7 +318,8 @@ def list(self, request, *args, **kwargs): # If we're asked for a sample and S3 is enabled, quickly return # the results from there. if settings.S3_ENABLED: - return fetch_json_results_from_s3(request, event.id, evaluator.id) + evaluator_id = self.kwargs["evaluator_id"] + return fetch_json_results_from_s3(request, event.id, evaluator_id) # Get a sample queryset queryset = self.get_sample_queryset(self.get_queryset()) From 6786e473dc119fc260b8fed1a954e89cd814bc93 Mon Sep 17 00:00:00 2001 From: Betsy Lorton Date: Wed, 29 Jul 2026 15:56:20 -0400 Subject: [PATCH 07/13] Fix test: expect new less detailed error message --- django/evaluate_m2/tests/test_views.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/django/evaluate_m2/tests/test_views.py b/django/evaluate_m2/tests/test_views.py index 96ebc67e..7f586e5e 100644 --- a/django/evaluate_m2/tests/test_views.py +++ b/django/evaluate_m2/tests/test_views.py @@ -294,8 +294,7 @@ def test_evaluator_results_view_with_error_no_evaluator_metadata(self): response = self.client.get('/api/events/1/evaluator/NON_EXISTENT/') self.assertEqual(response.headers['Content-Type'], 'application/json') - self.assertContains(response, 'Evaluator: NON_EXISTENT does not exist.', - status_code=404) + self.assertContains(response, 'record(s) not found', status_code=404) def test_evaluator_results_view_with_error_no_event(self): response = self.client.get('/api/events/1/evaluator/Status-DOFD-1/') From f93c342252264a3ef9d0c2c3c276591708ba060d Mon Sep 17 00:00:00 2001 From: Will Barton Date: Fri, 31 Jul 2026 09:32:46 -0400 Subject: [PATCH 08/13] Allow null filtering in AnyCharFilter With the materialized view, blank values are no longer empty strings, but are nulls. This adds an `isnull` filter to our AnyCharFilter so that we can filter for nulls. --- django/evaluate_m2/filters.py | 29 ++++++++++++++++-------- django/evaluate_m2/tests/test_filters.py | 11 ++++++--- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/django/evaluate_m2/filters.py b/django/evaluate_m2/filters.py index 92ce0b2a..23deabcd 100644 --- a/django/evaluate_m2/filters.py +++ b/django/evaluate_m2/filters.py @@ -1,4 +1,7 @@ +from django.db.models import Q + import django_filters.rest_framework +from django_filters.constants import EMPTY_VALUES from evaluate_m2.models import EvaluatorResultMaterializedView @@ -6,17 +9,25 @@ class AnyCharFilter(django_filters.BaseInFilter, django_filters.CharFilter): """Subclass CharFilter to allow multiple Char choices""" - # If this value is given, filter on an empty sting. - empty_value = "blank" + # If this value is given, filter on a null value + null_value = "blank" - # django-filter cannot filter on empty strings by default. - # The documentaiton offers a couple of approaches to doing so, this is - # based on one of them: - # https://django-filter.readthedocs.io/en/stable/guide/tips.html#filtering-by-an-empty-string + # Override the filter method to construct a query that will filter for + # values and null. def filter(self, qs, value): - if value is not None and self.empty_value in value: - value = ["" if v == self.empty_value else v for v in value] - return super().filter(qs, value) + if value in EMPTY_VALUES: + return qs + + query = Q() + if value is not None and self.null_value in value: + # Construct values that do not include the the null placeholder + # and add a null value query + value = [v for v in value if v != self.null_value] + query |= Q(**{f"{self.field_name}__isnull": True}) + + # Filter in the given value list + query |= Q(**{f"{self.field_name}__{self.lookup_expr}": value}) + return self.get_method(qs)(query) class EvaluatorResultFilterSet(django_filters.rest_framework.FilterSet): diff --git a/django/evaluate_m2/tests/test_filters.py b/django/evaluate_m2/tests/test_filters.py index 23ef5c03..14e77a3c 100644 --- a/django/evaluate_m2/tests/test_filters.py +++ b/django/evaluate_m2/tests/test_filters.py @@ -1,5 +1,6 @@ from unittest import mock +from django.db.models import Q from django.test import SimpleTestCase from evaluate_m2.filters import ( @@ -8,9 +9,13 @@ class AnyCharFilterTestCase(SimpleTestCase): - def test_filtering(self): + def test_filtering_empty_string(self): qs = mock.Mock(spec=["filter"]) - f = AnyCharFilter() + f = AnyCharFilter("test_field") + expected_q = ( + Q(test_field__isnull=True) | + Q(test_field__in=["value", "other"]) + ) result = f.filter(qs, ["value", "blank", "other"]) - qs.filter.assert_called_once_with(None__in=["value", "", "other"]) + qs.filter.assert_called_once_with(expected_q) self.assertNotEqual(qs, result) From add8c012f4ef6d6942d9300f102f73186df96dbb Mon Sep 17 00:00:00 2001 From: Will Barton Date: Fri, 31 Jul 2026 09:35:48 -0400 Subject: [PATCH 09/13] Fix linting --- django/evaluate_m2/models.py | 3 ++- django/evaluate_m2/serializers.py | 11 +++++++---- django/evaluate_m2/tests/test_views.py | 2 +- django/evaluate_m2/views.py | 9 ++++++--- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/django/evaluate_m2/models.py b/django/evaluate_m2/models.py index f1294489..550b6379 100644 --- a/django/evaluate_m2/models.py +++ b/django/evaluate_m2/models.py @@ -377,7 +377,8 @@ def materialized_view_exists(cls) -> bool: REFRESH MATERIALIZED VIEW mv_all_evaluator_results; """ create_index_command = """ - CREATE INDEX idx_event_id_evaluator_id ON mv_all_evaluator_results (event_id, evaluator_id); + CREATE INDEX idx_event_id_evaluator_id + ON mv_all_evaluator_results (event_id, evaluator_id); """ # The column names in the materialized view correspond to the diff --git a/django/evaluate_m2/serializers.py b/django/evaluate_m2/serializers.py index bc14cacd..a231162f 100644 --- a/django/evaluate_m2/serializers.py +++ b/django/evaluate_m2/serializers.py @@ -6,11 +6,10 @@ parse_fields_from_csv, plain_to_code_field_map, ) - from evaluate_m2.models import ( EvaluatorMetadata, - EvaluatorResultSummary, EvaluatorResultMaterializedView, + EvaluatorResultSummary, ) @@ -250,7 +249,9 @@ class EvaluatorResultSerializer(serializers.ModelSerializer): previous_value__acct_stat = serializers.CharField(source="prior_acct_stat") previous_value__pmt_rating = serializers.CharField(source="prior_pmt_rating") previous_value__current_bal = serializers.CharField(source="prior_current_bal") - previous_value__orig_chg_off_amt = serializers.CharField(source="prior_orig_chg_off_amt") + previous_value__orig_chg_off_amt = serializers.CharField( + source="prior_orig_chg_off_amt" + ) previous_value__dofd = serializers.CharField(source="prior_dofd") previous_value__date_closed = serializers.CharField(source="prior_date_closed") previous_value__surname = serializers.CharField(source="prior_surname") @@ -258,7 +259,9 @@ class EvaluatorResultSerializer(serializers.ModelSerializer): previous_value__ecoa = serializers.CharField(source="prior_ecoa") previous_value__ecoa_assoc = serializers.CharField(source="prior_ecoa_assoc") previous_value__cons_info_ind = serializers.CharField(source="prior_cons_info_ind") - previous_value__cons_info_ind_assoc = serializers.CharField(source="prior_cons_info_ind_assoc") + previous_value__cons_info_ind_assoc = serializers.CharField( + source="prior_cons_info_ind_assoc" + ) previous_value__l1__change_ind = serializers.CharField(source="prior_change_ind") previous_value__l1__new_acc_num = serializers.CharField(source="prior_new_acc_num") previous_value__l1__new_id_num = serializers.CharField(source="prior_new_id_num") diff --git a/django/evaluate_m2/tests/test_views.py b/django/evaluate_m2/tests/test_views.py index 7f586e5e..145c8a9c 100644 --- a/django/evaluate_m2/tests/test_views.py +++ b/django/evaluate_m2/tests/test_views.py @@ -5,8 +5,8 @@ from evaluate_m2.models import ( EvaluatorMetadata, EvaluatorResult, - EvaluatorResultSummary, EvaluatorResultMaterializedView, + EvaluatorResultSummary, ) from evaluate_m2.serializers import EvaluatorMetadataSerializer from evaluate_m2.tests.evaluator_test_helper import acct_record diff --git a/django/evaluate_m2/views.py b/django/evaluate_m2/views.py index 5289449a..5d84539d 100644 --- a/django/evaluate_m2/views.py +++ b/django/evaluate_m2/views.py @@ -16,19 +16,22 @@ from django_application.s3_utils import s3_session from evaluate_m2 import upload_utils -from evaluate_m2.exception_utils import get_evaluate_m2_not_found_exception, format_error +from evaluate_m2.exception_utils import ( + format_error, + get_evaluate_m2_not_found_exception, +) from evaluate_m2.filters import EvaluatorResultFilterSet from evaluate_m2.models import ( EvaluatorMetadata, EvaluatorResult, - EvaluatorResultSummary, EvaluatorResultMaterializedView, + EvaluatorResultSummary, ) from evaluate_m2.pagination import EvaluatorResultsPaginator from evaluate_m2.serializers import ( EvaluatorMetadataSerializer, - EventsViewSerializer, EvaluatorResultSerializer, + EventsViewSerializer, ) from evaluate_m2.views_utils import ( get_object, From 6f3ddb3c5cd6abe441ba3680dbd03c903a6bb15b Mon Sep 17 00:00:00 2001 From: Will Barton Date: Fri, 31 Jul 2026 11:01:43 -0400 Subject: [PATCH 10/13] Add JSONArrayContainsFilter to filter JSONField arrays This also refactors AnyCharFilter a bit to pull out the null-inclusive filtering to reuse in JSONArrayContainsFilter. --- django/evaluate_m2/filters.py | 65 +++++++++++++++++++----- django/evaluate_m2/tests/test_filters.py | 22 ++++++-- 2 files changed, 70 insertions(+), 17 deletions(-) diff --git a/django/evaluate_m2/filters.py b/django/evaluate_m2/filters.py index 23deabcd..01d7fd61 100644 --- a/django/evaluate_m2/filters.py +++ b/django/evaluate_m2/filters.py @@ -1,3 +1,5 @@ +from functools import reduce + from django.db.models import Q import django_filters.rest_framework @@ -6,27 +8,62 @@ from evaluate_m2.models import EvaluatorResultMaterializedView -class AnyCharFilter(django_filters.BaseInFilter, django_filters.CharFilter): - """Subclass CharFilter to allow multiple Char choices""" - +class NullInclusiveFilterMixin: # If this value is given, filter on a null value null_value = "blank" - # Override the filter method to construct a query that will filter for - # values and null. - def filter(self, qs, value): - if value in EMPTY_VALUES: - return qs - + def get_null_query(self, values, null_value="blank"): query = Q() - if value is not None and self.null_value in value: + + if values is not None and self.null_value in values: # Construct values that do not include the the null placeholder # and add a null value query - value = [v for v in value if v != self.null_value] + values = [v for v in values if v != self.null_value] query |= Q(**{f"{self.field_name}__isnull": True}) - # Filter in the given value list - query |= Q(**{f"{self.field_name}__{self.lookup_expr}": value}) + return values, query + + +class AnyCharFilter(django_filters.BaseInFilter, NullInclusiveFilterMixin): + """Match any char value in a CharField""" + + # Override the filter method to construct a query that will filter for + # values and null. + def filter(self, qs, values): + if values in EMPTY_VALUES: + return qs + + # Pull out a null value as a separate query + values, query = self.get_null_query(values) + + # Filter for remaining values + if len(values) > 0: + query |= Q(**{f"{self.field_name}__{self.lookup_expr}": values}) + + return self.get_method(qs)(query) + + +class JSONArrayContainsFilter(django_filters.BaseCSVFilter, NullInclusiveFilterMixin): + """Match any value in a JSONField array""" + + def __init__(self, *args, **kwargs): + kwargs.setdefault("lookup_expr", "contains") + super().__init__(*args, **kwargs) + + def filter(self, qs, values): + if values in EMPTY_VALUES: + return qs + + # Pull out a null value as a separate query + values, query = self.get_null_query(values) + + if len(values) > 0: + # Construct a Q object for every potential value + query |= reduce( + Q.__or__, + (Q(**{f"{self.field_name}__{self.lookup_expr}": [v]}) for v in values) + ) + return self.get_method(qs)(query) @@ -54,7 +91,7 @@ class EvaluatorResultFilterSet(django_filters.rest_framework.FilterSet): spc_com_cd = AnyCharFilter(field_name="spc_com_cd") terms_freq = AnyCharFilter(field_name="terms_freq") cons_info_ind = AnyCharFilter(field_name="cons_info_ind") - cons_info_ind_assoc = AnyCharFilter(field_name="cons_info_ind_assoc") + cons_info_ind_assoc = JSONArrayContainsFilter(field_name="cons_info_ind_assoc") l1__change_ind = AnyCharFilter(field_name="change_ind") # Dates, as a boolean where the date either exists or does not diff --git a/django/evaluate_m2/tests/test_filters.py b/django/evaluate_m2/tests/test_filters.py index 14e77a3c..061aa633 100644 --- a/django/evaluate_m2/tests/test_filters.py +++ b/django/evaluate_m2/tests/test_filters.py @@ -3,19 +3,35 @@ from django.db.models import Q from django.test import SimpleTestCase -from evaluate_m2.filters import ( - AnyCharFilter, -) +from evaluate_m2.filters import AnyCharFilter, JSONArrayContainsFilter class AnyCharFilterTestCase(SimpleTestCase): def test_filtering_empty_string(self): qs = mock.Mock(spec=["filter"]) f = AnyCharFilter("test_field") + expected_q = ( Q(test_field__isnull=True) | Q(test_field__in=["value", "other"]) ) + + result = f.filter(qs, ["value", "blank", "other"]) + qs.filter.assert_called_once_with(expected_q) + self.assertNotEqual(qs, result) + + +class JSONArrayContainsFilterTestCase(SimpleTestCase): + def test_filtering_empty_string(self): + qs = mock.Mock(spec=["filter"]) + f = JSONArrayContainsFilter("test_field") + + expected_q = ( + Q(test_field__isnull=True) | + Q(test_field__contains=["value", ]) | + Q(test_field__contains=["other", ]) + ) + result = f.filter(qs, ["value", "blank", "other"]) qs.filter.assert_called_once_with(expected_q) self.assertNotEqual(qs, result) From 312fd5b1ba5acbda4034694078637aaf098994c6 Mon Sep 17 00:00:00 2001 From: Will Barton Date: Fri, 31 Jul 2026 11:02:08 -0400 Subject: [PATCH 11/13] Add an admin for filterable fields in materialized view --- django/evaluate_m2/admin.py | 47 +++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/django/evaluate_m2/admin.py b/django/evaluate_m2/admin.py index 14c69587..daa61ffd 100644 --- a/django/evaluate_m2/admin.py +++ b/django/evaluate_m2/admin.py @@ -4,6 +4,7 @@ from evaluate_m2.models import ( EvaluatorMetadata, EvaluatorResult, + EvaluatorResultMaterializedView, EvaluatorResultSummary, ) @@ -42,6 +43,7 @@ def has_change_permission(self, request, obj=None): def has_delete_permission(self, request, obj=None): return False + class EvaluatorResultSummaryAdmin(admin.ModelAdmin): list_display = ['event', 'evaluator', 'hits'] @@ -54,6 +56,7 @@ def has_change_permission(self, request, obj=None): def has_delete_permission(self, request, obj=None): return False + class EvaluatorResultAdmin(admin.ModelAdmin): list_display = ['result_summary', 'date', 'source_record', 'acct_num'] @@ -65,6 +68,50 @@ def has_change_permission(self, request, obj=None): return False def has_delete_permission(self, request, obj=None): return False + + +class EvaluatorResultMaterializedViewAdmin(admin.ModelAdmin): + list_display = [ + 'event_id', + 'evaluator_id', + 'source_record_id', + 'activity_date', + 'cons_acct_num', + "acct_type", + "acct_stat", + "compl_cond_cd", + "php", + "php1", + "pmt_rating", + "spc_com_cd", + "terms_freq", + "cons_info_ind", + "cons_info_ind_assoc", + "change_ind", + "dofd", + "date_closed", + "amt_past_due", + "current_bal", + "smpa", + ] + + def has_add_permission(self, request, obj=None): + return False + + def has_view_permission(self, request, obj=None): + return True + + def has_change_permission(self, request, obj=None): + return False + + def has_delete_permission(self, request, obj=None): + return False + + admin.site.register(EvaluatorMetadata, EvaluatorMetadataAdmin) admin.site.register(EvaluatorResultSummary, EvaluatorResultSummaryAdmin) admin.site.register(EvaluatorResult, EvaluatorResultAdmin) +admin.site.register( + EvaluatorResultMaterializedView, + EvaluatorResultMaterializedViewAdmin +) From 3d8aaec4c98f45909f64c546d63bba2fcbb76dc5 Mon Sep 17 00:00:00 2001 From: Will Barton Date: Fri, 31 Jul 2026 11:18:14 -0400 Subject: [PATCH 12/13] Add materialized view to the seed data script --- django/seed-data.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/django/seed-data.sh b/django/seed-data.sh index b4035094..c309011d 100755 --- a/django/seed-data.sh +++ b/django/seed-data.sh @@ -15,6 +15,10 @@ echo echo "Parse the data file and run the evaluators" python manage.py add_seed_data_local -e Sample-Dataset-007 +echo +echo "Generate the evaluator results materialized view" +python manage.py materialized_view + echo echo "Add the user as a member of the event" python manage.py add_user_to_event --event=Sample-Dataset-007 --user=admin From ee94c404d172eccff58b2c87213e95e79fb844da Mon Sep 17 00:00:00 2001 From: Will Barton Date: Fri, 31 Jul 2026 12:21:35 -0400 Subject: [PATCH 13/13] Add dolp to materialized view for sorting test --- django/evaluate_m2/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/django/evaluate_m2/models.py b/django/evaluate_m2/models.py index 550b6379..bb65ecab 100644 --- a/django/evaluate_m2/models.py +++ b/django/evaluate_m2/models.py @@ -318,6 +318,7 @@ def materialized_view_exists(cls) -> bool: doai = models.DateField(db_column="doai") dofd = models.DateField(db_column="dofd") date_closed = models.DateField(db_column="date_closed") + dolp = models.DateField(db_column="dolp") int_type_ind = models.CharField(db_column="int_type_ind") surname = models.CharField(db_column="surname") first_name = models.CharField(db_column="first_name") @@ -415,6 +416,7 @@ def materialized_view_exists(cls) -> bool: a.doai, a.dofd, a.date_closed, + a.dolp, a.int_type_ind, a.surname, a.first_name,