diff --git a/django/evaluate_m2/admin.py b/django/evaluate_m2/admin.py index 14c69587..daa61ffd 100644 --- a/django/evaluate_m2/admin.py +++ b/django/evaluate_m2/admin.py @@ -4,6 +4,7 @@ from evaluate_m2.models import ( EvaluatorMetadata, EvaluatorResult, + EvaluatorResultMaterializedView, EvaluatorResultSummary, ) @@ -42,6 +43,7 @@ def has_change_permission(self, request, obj=None): def has_delete_permission(self, request, obj=None): return False + class EvaluatorResultSummaryAdmin(admin.ModelAdmin): list_display = ['event', 'evaluator', 'hits'] @@ -54,6 +56,7 @@ def has_change_permission(self, request, obj=None): def has_delete_permission(self, request, obj=None): return False + class EvaluatorResultAdmin(admin.ModelAdmin): list_display = ['result_summary', 'date', 'source_record', 'acct_num'] @@ -65,6 +68,50 @@ def has_change_permission(self, request, obj=None): return False def has_delete_permission(self, request, obj=None): return False + + +class EvaluatorResultMaterializedViewAdmin(admin.ModelAdmin): + list_display = [ + 'event_id', + 'evaluator_id', + 'source_record_id', + 'activity_date', + 'cons_acct_num', + "acct_type", + "acct_stat", + "compl_cond_cd", + "php", + "php1", + "pmt_rating", + "spc_com_cd", + "terms_freq", + "cons_info_ind", + "cons_info_ind_assoc", + "change_ind", + "dofd", + "date_closed", + "amt_past_due", + "current_bal", + "smpa", + ] + + def has_add_permission(self, request, obj=None): + return False + + def has_view_permission(self, request, obj=None): + return True + + def has_change_permission(self, request, obj=None): + return False + + def has_delete_permission(self, request, obj=None): + return False + + admin.site.register(EvaluatorMetadata, EvaluatorMetadataAdmin) admin.site.register(EvaluatorResultSummary, EvaluatorResultSummaryAdmin) admin.site.register(EvaluatorResult, EvaluatorResultAdmin) +admin.site.register( + EvaluatorResultMaterializedView, + EvaluatorResultMaterializedViewAdmin +) diff --git a/django/evaluate_m2/evaluate.py b/django/evaluate_m2/evaluate.py index 44ba86f8..b273c30b 100644 --- a/django/evaluate_m2/evaluate.py +++ b/django/evaluate_m2/evaluate.py @@ -6,7 +6,11 @@ from django.utils.module_loading import import_string from evaluate_m2.evaluate_utils import create_eval_insert_query -from evaluate_m2.models import EvaluatorMetadata, EvaluatorResultSummary +from evaluate_m2.models import ( + EvaluatorMetadata, + EvaluatorResultMaterializedView, + EvaluatorResultSummary, +) from evaluate_m2.upload_utils import stream_results_files_to_s3 from parse_m2.models import Metro2Event @@ -36,8 +40,10 @@ def load_evaluators(self): # runs evaluators to produce results def run_evaluators(self, event: Metro2Event): """ - Given an event, run all evaluators on the Account Activity associated - to the event and save the results to the database. + Given an event, run all evaluators on the Account Activity records + associated with the event and save the results to the database. + Then, create or update the evaluator result materialized view, so + the results can be queried. """ logger = logging.getLogger('evaluate.run_evaluators') @@ -46,6 +52,7 @@ def run_evaluators(self, event: Metro2Event): if record_set.exists(): for eval_name, eval_func in self.evaluators.items(): self.run_single_evaluator(event, eval_name, eval_func, record_set) + EvaluatorResultMaterializedView.create_or_refresh_materialized_view() else: logger.info(f"No AccountActivity found for the event '{event.id}'") diff --git a/django/evaluate_m2/exception_utils.py b/django/evaluate_m2/exception_utils.py index 685b78d3..3ba16f73 100644 --- a/django/evaluate_m2/exception_utils.py +++ b/django/evaluate_m2/exception_utils.py @@ -3,6 +3,16 @@ from rest_framework import status +def format_error(status: int, error_string: str, message: str, path: str): + return { + "timestamp": datetime.now(), + "status": status, + "error": error_string, + "message": message, + "path": path, + } + + def get_evaluate_m2_not_found_exception( error_string:str, event_id: str, evaluator_id: str, path: str, acct_num=''): msg = '' @@ -22,12 +32,5 @@ def get_evaluate_m2_not_found_exception( f'Evaluator result does not exist for event ID {event_id} or ' f'evaluator ID {evaluator_id}.' ) - error= { - 'timestamp': datetime.now(), - 'status': status.HTTP_404_NOT_FOUND, - 'error': 'Not Found', - 'message': msg, - 'path': path - } - return error + return format_error(status.HTTP_404_NOT_FOUND, 'Not Found', msg, path) diff --git a/django/evaluate_m2/filters.py b/django/evaluate_m2/filters.py index 61c779e4..01d7fd61 100644 --- a/django/evaluate_m2/filters.py +++ b/django/evaluate_m2/filters.py @@ -1,26 +1,74 @@ +from functools import reduce + +from django.db.models import Q + import django_filters.rest_framework +from django_filters.constants import EMPTY_VALUES + +from evaluate_m2.models import EvaluatorResultMaterializedView + + +class NullInclusiveFilterMixin: + # If this value is given, filter on a null value + null_value = "blank" + + def get_null_query(self, values, null_value="blank"): + query = Q() + + if values is not None and self.null_value in values: + # Construct values that do not include the the null placeholder + # and add a null value query + values = [v for v in values if v != self.null_value] + query |= Q(**{f"{self.field_name}__isnull": True}) + + return values, query + + +class AnyCharFilter(django_filters.BaseInFilter, NullInclusiveFilterMixin): + """Match any char value in a CharField""" -from evaluate_m2.models import EvaluatorResult + # Override the filter method to construct a query that will filter for + # values and null. + def filter(self, qs, values): + if values in EMPTY_VALUES: + return qs + # Pull out a null value as a separate query + values, query = self.get_null_query(values) -class AnyCharFilter(django_filters.BaseInFilter, django_filters.CharFilter): - """Subclass CharFilter to allow multiple Char choices""" + # Filter for remaining values + if len(values) > 0: + query |= Q(**{f"{self.field_name}__{self.lookup_expr}": values}) - # If this value is given, filter on an empty sting. - empty_value = "blank" + return self.get_method(qs)(query) - # django-filter cannot filter on empty strings by default. - # The documentaiton offers a couple of approaches to doing so, this is - # based on one of them: - # https://django-filter.readthedocs.io/en/stable/guide/tips.html#filtering-by-an-empty-string - def filter(self, qs, value): - if value is not None and self.empty_value in value: - value = ["" if v == self.empty_value else v for v in value] - return super().filter(qs, value) + +class JSONArrayContainsFilter(django_filters.BaseCSVFilter, NullInclusiveFilterMixin): + """Match any value in a JSONField array""" + + def __init__(self, *args, **kwargs): + kwargs.setdefault("lookup_expr", "contains") + super().__init__(*args, **kwargs) + + def filter(self, qs, values): + if values in EMPTY_VALUES: + return qs + + # Pull out a null value as a separate query + values, query = self.get_null_query(values) + + if len(values) > 0: + # Construct a Q object for every potential value + query |= reduce( + Q.__or__, + (Q(**{f"{self.field_name}__{self.lookup_expr}": [v]}) for v in values) + ) + + return self.get_method(qs)(query) class EvaluatorResultFilterSet(django_filters.rest_framework.FilterSet): - """This filter set specifies `EvaluatorResult` fields to filter. + """This filter set specifies `EvaluatorResultMaterializedView` fields to filter. Because the fields that we would filter `EvaluatorResult` objects by exist on their `source_record` relation, the fields here simply map the @@ -29,44 +77,44 @@ class EvaluatorResultFilterSet(django_filters.rest_framework.FilterSet): For example, the API might allow filtering `EvaluatorResults` by `acct_stat`, but the Django queryset will for `EvaluatorResults` will need - to be filtered by `source_record__acct_stat`. + to be filtered by `acct_stat`. """ - acct_type = django_filters.CharFilter(field_name="source_record__acct_type") + acct_type = django_filters.CharFilter(field_name="acct_type") acct_stat = AnyCharFilter( - field_name="source_record__acct_stat", + field_name="acct_stat", ) - compl_cond_cd = AnyCharFilter(field_name="source_record__compl_cond_cd") - php = AnyCharFilter(field_name="source_record__php") - php1 = AnyCharFilter(field_name="source_record__php1") - pmt_rating = AnyCharFilter(field_name="source_record__pmt_rating") - spc_com_cd = AnyCharFilter(field_name="source_record__spc_com_cd") - terms_freq = AnyCharFilter(field_name="source_record__terms_freq") - cons_info_ind = AnyCharFilter(field_name="source_record__cons_info_ind") - cons_info_ind_assoc = AnyCharFilter(field_name="source_record__cons_info_ind_assoc") - l1__change_ind = AnyCharFilter(field_name="source_record__l1__change_ind") + compl_cond_cd = AnyCharFilter(field_name="compl_cond_cd") + php = AnyCharFilter(field_name="php") + php1 = AnyCharFilter(field_name="php1") + pmt_rating = AnyCharFilter(field_name="pmt_rating") + spc_com_cd = AnyCharFilter(field_name="spc_com_cd") + terms_freq = AnyCharFilter(field_name="terms_freq") + cons_info_ind = AnyCharFilter(field_name="cons_info_ind") + cons_info_ind_assoc = JSONArrayContainsFilter(field_name="cons_info_ind_assoc") + l1__change_ind = AnyCharFilter(field_name="change_ind") # Dates, as a boolean where the date either exists or does not dofd = django_filters.BooleanFilter( - field_name="source_record__dofd", + field_name="dofd", lookup_expr="isnull", exclude=True, ) date_closed = django_filters.BooleanFilter( - field_name="source_record__date_closed", + field_name="date_closed", lookup_expr="isnull", exclude=True, ) # Amounts, as ranges of values with _max and _min fields amt_past_due = django_filters.RangeFilter( - field_name="source_record__amt_past_due", + field_name="amt_past_due", ) current_bal = django_filters.RangeFilter( - field_name="source_record__current_bal", + field_name="current_bal", ) smpa = django_filters.RangeFilter( - field_name="source_record__smpa", + field_name="smpa", ) # Sort ordering filter for all the relevant fields from AccountActivity @@ -74,101 +122,76 @@ class EvaluatorResultFilterSet(django_filters.rest_framework.FilterSet): # to the field name on AccountActivity. sort = django_filters.OrderingFilter( fields=( - ("source_record__activity_date", "activity_date"), - ("source_record__cons_acct_num", "cons_acct_num"), - ("source_record__port_type", "port_type"), - ("source_record__acct_type", "acct_type"), - ("source_record__date_open", "date_open"), - ("source_record__credit_limit", "credit_limit"), - ("source_record__hcola", "hcola"), - ("source_record__id_num", "id_num"), - ("source_record__terms_dur", "terms_dur"), - ("source_record__terms_freq", "terms_freq"), - ("source_record__smpa", "smpa"), - ("source_record__actual_pmt_amt", "actual_pmt_amt"), - ("source_record__acct_stat", "acct_stat"), - ("source_record__pmt_rating", "pmt_rating"), - ("source_record__php", "php"), - ("source_record__php1", "php1"), - ("source_record__spc_com_cd", "spc_com_cd"), - ("source_record__compl_cond_cd", "compl_cond_cd"), - ("source_record__current_bal", "current_bal"), - ("source_record__amt_past_due", "amt_past_due"), - ("source_record__orig_chg_off_amt", "orig_chg_off_amt"), - ("source_record__doai", "doai"), - ("source_record__dofd", "dofd"), - ("source_record__date_closed", "date_closed"), - ("source_record__dolp", "dolp"), - ("source_record__int_type_ind", "int_type_ind"), - ("source_record__cons_info_ind", "cons_info_ind"), - ("source_record__ecoa", "ecoa"), - ("source_record__cons_info_ind_assoc", "cons_info_ind_assoc"), - ("source_record__ecoa_assoc", "ecoa_assoc"), - ("source_record__first_name", "first_name"), - ("source_record__surname", "surname"), - ("source_record__k2__purch_sold_ind", "k2__purch_sold_ind"), - ("source_record__k2__purch_sold_name", "k2__purch_sold_name"), - ("source_record__k4__balloon_pmt_amt", "k4__balloon_pmt_amt"), - ("source_record__l1__change_ind", "l1__change_ind"), - ("source_record__l1__new_id_num", "l1__new_id_num"), - ("source_record__l1__new_acc_num", "l1__new_acc_num"), - ( - "source_record__previous_values__cons_info_ind", - "previous_values__cons_info_ind", - ), + ("activity_date", "activity_date"), + ("cons_acct_num", "cons_acct_num"), + ("port_type", "port_type"), + ("acct_type", "acct_type"), + ("date_open", "date_open"), + ("credit_limit", "credit_limit"), + ("hcola", "hcola"), + ("id_num", "id_num"), + ("terms_dur", "terms_dur"), + ("terms_freq", "terms_freq"), + ("smpa", "smpa"), + ("actual_pmt_amt", "actual_pmt_amt"), + ("acct_stat", "acct_stat"), + ("pmt_rating", "pmt_rating"), + ("php", "php"), + ("php1", "php1"), + ("spc_com_cd", "spc_com_cd"), + ("compl_cond_cd", "compl_cond_cd"), + ("current_bal", "current_bal"), + ("amt_past_due", "amt_past_due"), + ("orig_chg_off_amt", "orig_chg_off_amt"), + ("doai", "doai"), + ("dofd", "dofd"), + ("date_closed", "date_closed"), + ("dolp", "dolp"), + ("int_type_ind", "int_type_ind"), + ("cons_info_ind", "cons_info_ind"), + ("ecoa", "ecoa"), + ("cons_info_ind_assoc", "cons_info_ind_assoc"), + ("ecoa_assoc", "ecoa_assoc"), + ("first_name", "first_name"), + ("surname", "surname"), + + # For all of these that are relations on the original models but + # direct fields on the materialized view, we keep the original + # Django __ related fieldname on the sort parameter side. That's + # the name the front-end knows the field as. + ("purch_sold_ind", "k2__purch_sold_ind"), + ("purch_sold_name", "k2__purch_sold_name"), + ("balloon_pmt_amt", "k4__balloon_pmt_amt"), + ("change_ind", "l1__change_ind"), + ("new_id_num", "l1__new_id_num"), + ("new_acc_num", "l1__new_acc_num"), + ("prior_cons_info_ind", "previous_values__cons_info_ind"), ( - "source_record__previous_values__cons_info_ind_assoc", + "prior_cons_info_ind_assoc", "previous_values__cons_info_ind_assoc", ), - ("source_record__previous_values__ecoa", "previous_values__ecoa"), - ( - "source_record__previous_values__first_name", - "previous_values__first_name", - ), - ("source_record__previous_values__surname", "previous_values__surname"), - ( - "source_record__previous_values__l1__change_ind", - "previous_values__l1__change_ind", - ), - ( - "source_record__previous_values__l1__new_acc_num", - "previous_values__l1__new_acc_num", - ), - ( - "source_record__previous_values__l1__new_id_num", - "previous_values__l1__new_id_num", - ), - ( - "source_record__previous_values__activity_date", - "previous_values__activity_date", - ), - ("source_record__previous_values__port_type", "previous_values__port_type"), - ("source_record__previous_values__acct_type", "previous_values__acct_type"), - ("source_record__previous_values__date_open", "previous_values__date_open"), - ("source_record__previous_values__acct_stat", "previous_values__acct_stat"), - ( - "source_record__previous_values__pmt_rating", - "previous_values__pmt_rating", - ), - ( - "source_record__previous_values__current_bal", - "previous_values__current_bal", - ), - ( - "source_record__previous_values__orig_chg_off_amt", - "previous_values__orig_chg_off_amt", - ), - ("source_record__previous_values__dofd", "previous_values__dofd"), - ( - "source_record__previous_values__date_closed", - "previous_values__date_closed", - ), - ("source_record__previous_values__id_num", "previous_values__id_num"), + ("prior_ecoa", "previous_values__ecoa"), + ("prior_first_name", "previous_values__first_name"), + ("prior_surname", "previous_values__surname"), + ("prior_change_ind", "previous_values__l1__change_ind"), + ("prior_new_acc_num", "previous_values__l1__new_acc_num"), + ("prior_new_id_num", "previous_values__l1__new_id_num"), + ("prior_activity_date", "previous_values__activity_date"), + ("prior_port_type", "previous_values__port_type"), + ("prior_acct_type", "previous_values__acct_type"), + ("prior_date_open", "previous_values__date_open"), + ("prior_acct_stat", "previous_values__acct_stat"), + ("prior_pmt_rating", "previous_values__pmt_rating"), + ("prior_current_bal", "previous_values__current_bal"), + ("prior_orig_chg_off_amt", "previous_values__orig_chg_off_amt"), + ("prior_dofd", "previous_values__dofd"), + ("prior_date_closed", "previous_values__date_closed"), + ("prior_id_num", "previous_values__id_num"), ) ) class Meta: - model = EvaluatorResult + model = EvaluatorResultMaterializedView fields = [ "acct_type", "acct_stat", diff --git a/django/evaluate_m2/management/commands/materialized_view.py b/django/evaluate_m2/management/commands/materialized_view.py new file mode 100644 index 00000000..d3607b56 --- /dev/null +++ b/django/evaluate_m2/management/commands/materialized_view.py @@ -0,0 +1,32 @@ +import logging + +from django.core.management.base import BaseCommand + +from evaluate_m2.models import EvaluatorResultMaterializedView + + +class Command(BaseCommand): + """ + Run this command by running the following: + > python manage.py materialized_view + """ + help = ( + "Creates the materialized view that connects with the " + "EvaluatorResultMaterializedView model, if it doesn't exist. " + "This is what allows the evaluator results view to work. " + "If the materialized view does exist, this command refreshes " + "it to incorporate any new evaluator results." + ) + + + def handle(self, *args, **options): + logger = logging.getLogger('commands.single_evaluator') + + EvaluatorResultMaterializedView.create_or_refresh_materialized_view() + + logger.info( + self.style.SUCCESS( + "Success: Evaluator Result materialized view now exists" + " and contains all evaluator results" + ) + ) \ No newline at end of file diff --git a/django/evaluate_m2/migrations/0003_evaluatorresultmaterializedview.py b/django/evaluate_m2/migrations/0003_evaluatorresultmaterializedview.py new file mode 100644 index 00000000..831a799d --- /dev/null +++ b/django/evaluate_m2/migrations/0003_evaluatorresultmaterializedview.py @@ -0,0 +1,99 @@ +# Generated by Django 5.2.14 on 2026-07-24 18:41 + +import django.core.serializers.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('evaluate_m2', '0002_evaluatormetadata_additional_notes_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='EvaluatorResultMaterializedView', + fields=[ + ('id', models.IntegerField(db_column='id', primary_key=True, serialize=False)), + ('event_id', models.IntegerField(db_column='event_id')), + ('evaluator_id', models.CharField(db_column='evaluator_id')), + ('source_record_id', models.IntegerField(db_column='source_record_id')), + ('activity_date', models.DateField(db_column='activity_date')), + ('cons_acct_num', models.CharField(db_column='cons_acct_num')), + ('port_type', models.CharField(db_column='port_type')), + ('acct_type', models.CharField(db_column='acct_type')), + ('date_open', models.DateField(db_column='date_open')), + ('credit_limit', models.IntegerField(db_column='credit_limit')), + ('hcola', models.IntegerField(db_column='hcola')), + ('id_num', models.CharField(db_column='id_num')), + ('terms_dur', models.CharField(db_column='terms_dur')), + ('terms_freq', models.CharField(db_column='terms_freq')), + ('smpa', models.IntegerField(db_column='smpa')), + ('actual_pmt_amt', models.IntegerField(db_column='actual_pmt_amt')), + ('acct_stat', models.CharField(db_column='acct_stat')), + ('pmt_rating', models.CharField(db_column='pmt_rating')), + ('php', models.CharField(db_column='php')), + ('php1', models.CharField(db_column='php1')), + ('spc_com_cd', models.CharField(db_column='spc_com_cd')), + ('compl_cond_cd', models.CharField(db_column='compl_cond_cd')), + ('current_bal', models.IntegerField(db_column='current_bal')), + ('amt_past_due', models.IntegerField(db_column='amt_past_due')), + ('orig_chg_off_amt', models.IntegerField(db_column='orig_chg_off_amt')), + ('doai', models.DateField(db_column='doai')), + ('dofd', models.DateField(db_column='dofd')), + ('date_closed', models.DateField(db_column='date_closed')), + ('int_type_ind', models.CharField(db_column='int_type_ind')), + ('surname', models.CharField(db_column='surname')), + ('first_name', models.CharField(db_column='first_name')), + ('middle_name', models.CharField(db_column='middle_name')), + ('gen_code', models.CharField(db_column='gen_code')), + ('ssn', models.CharField(db_column='ssn')), + ('dob', models.CharField(db_column='dob')), + ('phone_num', models.CharField(db_column='phone_num')), + ('ecoa', models.CharField(db_column='ecoa')), + ('ecoa_assoc', models.JSONField(db_column='ecoa_assoc', encoder=django.core.serializers.json.DjangoJSONEncoder)), + ('cons_info_ind', models.CharField(db_column='cons_info_ind')), + ('cons_info_ind_assoc', models.JSONField(db_column='cons_info_ind_assoc', encoder=django.core.serializers.json.DjangoJSONEncoder)), + ('addr_line_1', models.CharField(db_column='addr_line_1')), + ('addr_line_2', models.CharField(db_column='addr_line_2')), + ('city', models.CharField(db_column='city')), + ('state', models.CharField(db_column='state')), + ('zip', models.CharField(db_column='zip')), + ('addr_ind', models.CharField(db_column='addr_ind')), + ('res_cd', models.CharField(db_column='res_cd')), + ('purch_sold_ind', models.CharField(db_column='purch_sold_ind')), + ('purch_sold_name', models.CharField(db_column='purch_sold_name')), + ('spc_pmt_ind', models.CharField(db_column='spc_pmt_ind')), + ('deferred_pmt_st_dt', models.DateField(db_column='deferred_pmt_st_dt')), + ('balloon_pmt_due_dt', models.DateField(db_column='balloon_pmt_due_dt')), + ('balloon_pmt_amt', models.IntegerField(db_column='balloon_pmt_amt')), + ('change_ind', models.CharField(db_column='change_ind')), + ('new_acc_num', models.CharField(db_column='new_acc_num')), + ('new_id_num', models.CharField(db_column='new_id_num')), + ('prior_activity_date', models.DateField(db_column='prior_activity_date')), + ('prior_port_type', models.CharField(db_column='prior_port_type')), + ('prior_acct_type', models.CharField(db_column='prior_acct_type')), + ('prior_date_open', models.DateField(db_column='prior_date_open')), + ('prior_id_num', models.CharField(db_column='prior_id_num')), + ('prior_acct_stat', models.CharField(db_column='prior_acct_stat')), + ('prior_pmt_rating', models.CharField(db_column='prior_pmt_rating')), + ('prior_current_bal', models.IntegerField(db_column='prior_current_bal')), + ('prior_orig_chg_off_amt', models.IntegerField(db_column='prior_orig_chg_off_amt')), + ('prior_dofd', models.DateField(db_column='prior_dofd')), + ('prior_date_closed', models.DateField(db_column='prior_date_closed')), + ('prior_surname', models.CharField(db_column='prior_surname')), + ('prior_first_name', models.CharField(db_column='prior_first_name')), + ('prior_ecoa', models.CharField(db_column='prior_ecoa')), + ('prior_ecoa_assoc', models.JSONField(db_column='prior_ecoa_assoc', encoder=django.core.serializers.json.DjangoJSONEncoder)), + ('prior_cons_info_ind', models.CharField(db_column='prior_cons_info_ind')), + ('prior_cons_info_ind_assoc', models.JSONField(db_column='prior_cons_info_ind_assoc', encoder=django.core.serializers.json.DjangoJSONEncoder)), + ('prior_change_ind', models.CharField(db_column='prior_change_ind')), + ('prior_new_acc_num', models.CharField(db_column='prior_new_acc_num')), + ('prior_new_id_num', models.CharField(db_column='prior_new_id_num')), + ], + options={ + 'db_table': 'mv_all_evaluator_results', + 'managed': False, + }, + ), + ] diff --git a/django/evaluate_m2/models.py b/django/evaluate_m2/models.py index 5db517f5..bb65ecab 100644 --- a/django/evaluate_m2/models.py +++ b/django/evaluate_m2/models.py @@ -1,8 +1,9 @@ +import logging from datetime import date from django.conf import settings from django.core.serializers.json import DjangoJSONEncoder -from django.db import models +from django.db import connection, models from django.db.models import DEFERRED, JSONField from django_prose_editor.fields import ProseEditorField @@ -256,3 +257,232 @@ def create_csv_row_data(self, fields_list: list[str]): self.result_summary.event.name, ] + list(field_values) return response + + +class EvaluatorResultMaterializedView(models.Model): + class Meta: + managed = False + db_table = 'mv_all_evaluator_results' + + table_name = 'mv_all_evaluator_results' + + @classmethod + def create_or_refresh_materialized_view(cls): + lg = logging.getLogger('eval_mv.create_or_refresh_materialized_view') + if cls.materialized_view_exists(): + lg.info("Refreshing the evaluator results materialized view...") + with connection.cursor() as c: + c.execute(cls.refresh_view_command) + else: + lg.info("Creating the evaluator results materialized view...") + with connection.cursor() as c: + c.execute(cls.create_view_command) + c.execute(cls.create_index_command) + lg.info("... Done.") + + @classmethod + def materialized_view_exists(cls) -> bool: + with connection.cursor() as c: + command = "select * from pg_matviews where matviewname = %s" + result = c.execute(command, [cls.table_name]) + return result.rowcount > 0 + + # The fields on this model correspond to the + # column names in the materialized view + id = models.IntegerField(db_column="id", primary_key=True) + event_id = models.IntegerField(db_column="event_id") + evaluator_id = models.CharField(db_column="evaluator_id") + source_record_id = models.IntegerField(db_column="source_record_id") + # account activity fields + activity_date = models.DateField(db_column="activity_date") + cons_acct_num = models.CharField(db_column="cons_acct_num") + port_type = models.CharField(db_column="port_type") + acct_type = models.CharField(db_column="acct_type") + date_open = models.DateField(db_column="date_open") + credit_limit = models.IntegerField(db_column="credit_limit") + hcola = models.IntegerField(db_column="hcola") + id_num = models.CharField(db_column="id_num") + terms_dur = models.CharField(db_column="terms_dur") + terms_freq = models.CharField(db_column="terms_freq") + smpa = models.IntegerField(db_column="smpa") + actual_pmt_amt = models.IntegerField(db_column="actual_pmt_amt") + acct_stat = models.CharField(db_column="acct_stat") + pmt_rating = models.CharField(db_column="pmt_rating") + php = models.CharField(db_column="php") + php1 = models.CharField(db_column="php1") + spc_com_cd = models.CharField(db_column="spc_com_cd") + compl_cond_cd = models.CharField(db_column="compl_cond_cd") + current_bal = models.IntegerField(db_column="current_bal") + amt_past_due = models.IntegerField(db_column="amt_past_due") + orig_chg_off_amt = models.IntegerField(db_column="orig_chg_off_amt") + doai = models.DateField(db_column="doai") + dofd = models.DateField(db_column="dofd") + date_closed = models.DateField(db_column="date_closed") + dolp = models.DateField(db_column="dolp") + int_type_ind = models.CharField(db_column="int_type_ind") + surname = models.CharField(db_column="surname") + first_name = models.CharField(db_column="first_name") + middle_name = models.CharField(db_column="middle_name") + gen_code = models.CharField(db_column="gen_code") + ssn = models.CharField(db_column="ssn") + dob = models.CharField(db_column="dob") + phone_num = models.CharField(db_column="phone_num") + ecoa = models.CharField(db_column="ecoa") + ecoa_assoc = models.JSONField( + encoder=DjangoJSONEncoder, db_column="ecoa_assoc") + cons_info_ind = models.CharField(db_column="cons_info_ind") + cons_info_ind_assoc = models.JSONField( + encoder=DjangoJSONEncoder, db_column="cons_info_ind_assoc") + addr_line_1 = models.CharField(db_column="addr_line_1") + addr_line_2 = models.CharField(db_column="addr_line_2") + city = models.CharField(db_column="city") + state = models.CharField(db_column="state") + zip = models.CharField(db_column="zip") + addr_ind = models.CharField(db_column="addr_ind") + res_cd = models.CharField(db_column="res_cd") + # extra segment columns + purch_sold_ind = models.CharField(db_column="purch_sold_ind") + purch_sold_name = models.CharField(db_column="purch_sold_name") + spc_pmt_ind = models.CharField(db_column="spc_pmt_ind") + deferred_pmt_st_dt = models.DateField(db_column="deferred_pmt_st_dt") + balloon_pmt_due_dt = models.DateField(db_column="balloon_pmt_due_dt") + balloon_pmt_amt = models.IntegerField(db_column="balloon_pmt_amt") + change_ind = models.CharField(db_column="change_ind") + new_acc_num = models.CharField(db_column="new_acc_num") + new_id_num = models.CharField(db_column="new_id_num") + # prior record values + prior_activity_date = models.DateField(db_column="prior_activity_date") + prior_port_type = models.CharField(db_column="prior_port_type") + prior_acct_type = models.CharField(db_column="prior_acct_type") + prior_date_open = models.DateField(db_column="prior_date_open") + prior_id_num = models.CharField(db_column="prior_id_num") + prior_acct_stat = models.CharField(db_column="prior_acct_stat") + prior_pmt_rating = models.CharField(db_column="prior_pmt_rating") + prior_current_bal = models.IntegerField(db_column="prior_current_bal") + prior_orig_chg_off_amt = models.IntegerField(db_column="prior_orig_chg_off_amt") + prior_dofd = models.DateField(db_column="prior_dofd") + prior_date_closed = models.DateField(db_column="prior_date_closed") + prior_surname = models.CharField(db_column="prior_surname") + prior_first_name = models.CharField(db_column="prior_first_name") + prior_ecoa = models.CharField(db_column="prior_ecoa") + prior_ecoa_assoc = models.JSONField( + encoder=DjangoJSONEncoder, db_column="prior_ecoa_assoc") + prior_cons_info_ind = models.CharField(db_column="prior_cons_info_ind") + prior_cons_info_ind_assoc = models.JSONField( + encoder=DjangoJSONEncoder, db_column="prior_cons_info_ind_assoc") + prior_change_ind = models.CharField(db_column="prior_change_ind") + prior_new_acc_num = models.CharField(db_column="prior_new_acc_num") + prior_new_id_num = models.CharField(db_column="prior_new_id_num") + + refresh_view_command = """ + REFRESH MATERIALIZED VIEW mv_all_evaluator_results; + """ + create_index_command = """ + CREATE INDEX idx_event_id_evaluator_id + ON mv_all_evaluator_results (event_id, evaluator_id); + """ + + # The column names in the materialized view correspond to the + # fields on the model + create_view_command = """ + CREATE MATERIALIZED VIEW mv_all_evaluator_results AS ( + SELECT + r.id, + a.event_id, + s.evaluator_id, + a.id as source_record_id, + + a.activity_date, + a.cons_acct_num, + a.port_type, + a.acct_type, + a.date_open, + a.credit_limit, + a.hcola, + a.id_num, + a.terms_dur, + a.terms_freq, + a.smpa, + a.actual_pmt_amt, + a.acct_stat, + a.pmt_rating, + a.php, + a.php1, + a.spc_com_cd, + a.compl_cond_cd, + a.current_bal, + a.amt_past_due, + a.orig_chg_off_amt, + a.doai, + a.dofd, + a.date_closed, + a.dolp, + a.int_type_ind, + a.surname, + a.first_name, + a.middle_name, + a.gen_code, + a.ssn, + a.dob, + a.phone_num, + a.ecoa, + a.ecoa_assoc, + a.cons_info_ind, + a.cons_info_ind_assoc, + a.addr_line_1, + a.addr_line_2, + a.city, + a.state, + a.zip, + a.addr_ind, + a.res_cd, + + k2.purch_sold_ind, + k2.purch_sold_name, + k4.spc_pmt_ind, + k4.deferred_pmt_st_dt, + k4.balloon_pmt_due_dt, + k4.balloon_pmt_amt, + l1.change_ind, + l1.new_acc_num, + l1.new_id_num, + + prev.activity_date as prior_activity_date, + prev.port_type as prior_port_type, + prev.acct_type as prior_acct_type, + prev.date_open as prior_date_open, + prev.id_num as prior_id_num, + prev.acct_stat as prior_acct_stat, + prev.pmt_rating as prior_pmt_rating, + prev.current_bal as prior_current_bal, + prev.orig_chg_off_amt as prior_orig_chg_off_amt, + prev.dofd as prior_dofd, + prev.date_closed as prior_date_closed, + prev.surname as prior_surname, + prev.first_name as prior_first_name, + prev.ecoa as prior_ecoa, + prev.ecoa_assoc as prior_ecoa_assoc, + prev.cons_info_ind as prior_cons_info_ind, + prev.cons_info_ind_assoc as prior_cons_info_ind_assoc, + prev_l1.change_ind as prior_change_ind, + prev_l1.new_acc_num as prior_new_acc_num, + prev_l1.new_id_num as prior_new_id_num + + FROM + parse_m2_accountactivity as a + INNER JOIN evaluate_m2_evaluatorresult as r + on r.source_record_id = a.id + INNER JOIN evaluate_m2_evaluatorresultsummary as s + on r.result_summary_id = s.id + LEFT JOIN parse_m2_k2 as k2 + on a.id = k2.account_activity_id + LEFT JOIN parse_m2_k4 as k4 + on a.id = k4.account_activity_id + LEFT JOIN parse_m2_l1 as l1 + on a.id = l1.account_activity_id + LEFT JOIN parse_m2_accountactivity as prev + on a.previous_values_id = prev.id + LEFT JOIN parse_m2_l1 as prev_l1 + on prev.id = prev_l1.account_activity_id + ); + """ diff --git a/django/evaluate_m2/serializers.py b/django/evaluate_m2/serializers.py index 1313c251..a231162f 100644 --- a/django/evaluate_m2/serializers.py +++ b/django/evaluate_m2/serializers.py @@ -6,8 +6,11 @@ parse_fields_from_csv, plain_to_code_field_map, ) - -from .models import EvaluatorMetadata, EvaluatorResultSummary +from evaluate_m2.models import ( + EvaluatorMetadata, + EvaluatorResultMaterializedView, + EvaluatorResultSummary, +) class EventsViewSerializer(serializers.ModelSerializer): @@ -224,3 +227,45 @@ def validate(self, data): if invalid_fields: raise serializers.ValidationError(f"Invalid field names: {invalid_fields}") return data + + +class EvaluatorResultSerializer(serializers.ModelSerializer): + # Outside of the materialized view, the API serializes related objects + # using Django's __ notation. These fields preserve those fieldnames. + k2__purch_sold_ind = serializers.CharField(source="purch_sold_ind") + k2__purch_sold_name = serializers.CharField(source="purch_sold_name") + k4__spc_pmt_ind = serializers.CharField(source="spc_pmt_ind") + k4__deferred_pmt_st_dt = serializers.CharField(source="deferred_pmt_st_dt") + k4__balloon_pmt_due_dt = serializers.CharField(source="balloon_pmt_due_dt") + k4__balloon_pmt_amt = serializers.CharField(source="balloon_pmt_amt") + l1__change_ind = serializers.CharField(source="change_ind") + l1__new_acc_num = serializers.CharField(source="new_acc_num") + l1__new_id_num = serializers.CharField(source="new_id_num") + previous_value__activity_date = serializers.CharField(source="prior_activity_date") + previous_value__port_type = serializers.CharField(source="prior_port_type") + previous_value__acct_type = serializers.CharField(source="prior_acct_type") + previous_value__date_open = serializers.CharField(source="prior_date_open") + previous_value__id_num = serializers.CharField(source="prior_id_num") + previous_value__acct_stat = serializers.CharField(source="prior_acct_stat") + previous_value__pmt_rating = serializers.CharField(source="prior_pmt_rating") + previous_value__current_bal = serializers.CharField(source="prior_current_bal") + previous_value__orig_chg_off_amt = serializers.CharField( + source="prior_orig_chg_off_amt" + ) + previous_value__dofd = serializers.CharField(source="prior_dofd") + previous_value__date_closed = serializers.CharField(source="prior_date_closed") + previous_value__surname = serializers.CharField(source="prior_surname") + previous_value__first_name = serializers.CharField(source="prior_first_name") + previous_value__ecoa = serializers.CharField(source="prior_ecoa") + previous_value__ecoa_assoc = serializers.CharField(source="prior_ecoa_assoc") + previous_value__cons_info_ind = serializers.CharField(source="prior_cons_info_ind") + previous_value__cons_info_ind_assoc = serializers.CharField( + source="prior_cons_info_ind_assoc" + ) + previous_value__l1__change_ind = serializers.CharField(source="prior_change_ind") + previous_value__l1__new_acc_num = serializers.CharField(source="prior_new_acc_num") + previous_value__l1__new_id_num = serializers.CharField(source="prior_new_id_num") + + class Meta: + model = EvaluatorResultMaterializedView + fields = '__all__' diff --git a/django/evaluate_m2/tests/test_filters.py b/django/evaluate_m2/tests/test_filters.py index 23ef5c03..061aa633 100644 --- a/django/evaluate_m2/tests/test_filters.py +++ b/django/evaluate_m2/tests/test_filters.py @@ -1,16 +1,37 @@ from unittest import mock +from django.db.models import Q from django.test import SimpleTestCase -from evaluate_m2.filters import ( - AnyCharFilter, -) +from evaluate_m2.filters import AnyCharFilter, JSONArrayContainsFilter class AnyCharFilterTestCase(SimpleTestCase): - def test_filtering(self): + def test_filtering_empty_string(self): qs = mock.Mock(spec=["filter"]) - f = AnyCharFilter() + f = AnyCharFilter("test_field") + + expected_q = ( + Q(test_field__isnull=True) | + Q(test_field__in=["value", "other"]) + ) + + result = f.filter(qs, ["value", "blank", "other"]) + qs.filter.assert_called_once_with(expected_q) + self.assertNotEqual(qs, result) + + +class JSONArrayContainsFilterTestCase(SimpleTestCase): + def test_filtering_empty_string(self): + qs = mock.Mock(spec=["filter"]) + f = JSONArrayContainsFilter("test_field") + + expected_q = ( + Q(test_field__isnull=True) | + Q(test_field__contains=["value", ]) | + Q(test_field__contains=["other", ]) + ) + result = f.filter(qs, ["value", "blank", "other"]) - qs.filter.assert_called_once_with(None__in=["value", "", "other"]) + qs.filter.assert_called_once_with(expected_q) self.assertNotEqual(qs, result) diff --git a/django/evaluate_m2/tests/test_views.py b/django/evaluate_m2/tests/test_views.py index 663db4d3..145c8a9c 100644 --- a/django/evaluate_m2/tests/test_views.py +++ b/django/evaluate_m2/tests/test_views.py @@ -5,6 +5,7 @@ from evaluate_m2.models import ( EvaluatorMetadata, EvaluatorResult, + EvaluatorResultMaterializedView, EvaluatorResultSummary, ) from evaluate_m2.serializers import EvaluatorMetadataSerializer @@ -56,6 +57,8 @@ def setUp(self) -> None: alternate_explanation='Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua', # noqa: E501 ) + EvaluatorResultMaterializedView.create_or_refresh_materialized_view() + def get_account_activity( self, id: int, @@ -182,6 +185,8 @@ def create_activity_data(self, create_zero_hit:bool=False): inconsistency_end=acct_date ) + EvaluatorResultMaterializedView.create_or_refresh_materialized_view() + ######################################## # Tests for Eval Metadata download def test_download_eval_metadata(self): @@ -232,6 +237,7 @@ def test_download_evaluator_results_csv(self): def test_evaluator_results_view(self): self.create_activity_data() response = self.client.get('/api/events/1/evaluator/Status-DOFD-1/') + # the response should be a JSON self.assertEqual(response.status_code, 200) self.assertEqual(response.headers['Content-Type'], 'application/json') @@ -275,6 +281,8 @@ def test_evaluator_results_view_max_20_results(self): self.eval_rs3.sample_ids = [1,3,5,7,9] self.eval_rs3.save() + EvaluatorResultMaterializedView.create_or_refresh_materialized_view() + response = self.client.get('/api/events/1/evaluator/Status-DOFD-6/') # the response should be a JSON self.assertEqual(response.status_code, 200) @@ -286,8 +294,7 @@ def test_evaluator_results_view_with_error_no_evaluator_metadata(self): response = self.client.get('/api/events/1/evaluator/NON_EXISTENT/') self.assertEqual(response.headers['Content-Type'], 'application/json') - self.assertContains(response, 'Evaluator: NON_EXISTENT does not exist.', - status_code=404) + self.assertContains(response, 'record(s) not found', status_code=404) def test_evaluator_results_view_with_error_no_event(self): response = self.client.get('/api/events/1/evaluator/Status-DOFD-1/') diff --git a/django/evaluate_m2/views.py b/django/evaluate_m2/views.py index f2c5054d..5d84539d 100644 --- a/django/evaluate_m2/views.py +++ b/django/evaluate_m2/views.py @@ -4,6 +4,7 @@ from datetime import date from django.conf import settings +from django.db import ProgrammingError from django.http import Http404, HttpResponse, StreamingHttpResponse from django.shortcuts import get_list_or_404 @@ -15,16 +16,21 @@ from django_application.s3_utils import s3_session from evaluate_m2 import upload_utils -from evaluate_m2.exception_utils import get_evaluate_m2_not_found_exception +from evaluate_m2.exception_utils import ( + format_error, + get_evaluate_m2_not_found_exception, +) from evaluate_m2.filters import EvaluatorResultFilterSet from evaluate_m2.models import ( EvaluatorMetadata, EvaluatorResult, + EvaluatorResultMaterializedView, EvaluatorResultSummary, ) from evaluate_m2.pagination import EvaluatorResultsPaginator from evaluate_m2.serializers import ( EvaluatorMetadataSerializer, + EvaluatorResultSerializer, EventsViewSerializer, ) from evaluate_m2.views_utils import ( @@ -58,6 +64,7 @@ def download_evaluator_metadata_csv(request): return response + @api_view(('GET',)) def download_evaluator_results_csv(request, event_id, evaluator_id): logger = logging.getLogger('views.download_evaluator_results_csv') @@ -131,6 +138,7 @@ def account_summary_view(request, event_id, account_number): logger.error(error['message']) return Response(error, status=status.HTTP_404_NOT_FOUND) + @api_view(('GET',)) def account_pii_view(request, event_id, account_number): logger = logging.getLogger('views.account_pii_view') @@ -153,6 +161,7 @@ def account_pii_view(request, event_id, account_number): logger.error(error['message']) return Response(error, status=status.HTTP_404_NOT_FOUND) + @api_view() def events_view(request, event_id): logger = logging.getLogger('views.evaluator_results_view') @@ -188,6 +197,7 @@ def events_view(request, event_id): logger.error(error['message']) return Response(error, status=status.HTTP_404_NOT_FOUND) + ########################################### ## Helper methods for eval results when S3_ENABLED == True def fetch_csv_results_from_s3(request, event_id, evaluator_id): @@ -209,6 +219,7 @@ def fetch_csv_results_from_s3(request, event_id, evaluator_id): logger.error(error['message']) return Response(error, status=status.HTTP_404_NOT_FOUND) + def fetch_json_results_from_s3(request, event_id, evaluator_id): logger = logging.getLogger('views.fetch_json_results_from_s3') s3 = s3_session() @@ -226,39 +237,20 @@ def fetch_json_results_from_s3(request, event_id, evaluator_id): class EvaluatorResultsView(generics.ListAPIView): + serializer_class = EvaluatorResultSerializer pagination_class = EvaluatorResultsPaginator filter_backends = [ django_filters.rest_framework.DjangoFilterBackend, ] filterset_class = EvaluatorResultFilterSet - def get_result_summary(self): - # Get the EvaluatorResultSummary object for this event_id and - # evaluator_id. These are queried directly so we can validate that - # they exist and error appropriately if they do not. - event_id = self.kwargs["event_id"] - evaluator_id = self.kwargs["evaluator_id"] - event = Metro2Event.objects.get(id=event_id) - evaluator = EvaluatorMetadata.objects.get(id=evaluator_id) - result_summary = EvaluatorResultSummary.objects.get( - event=event, evaluator=evaluator) - return result_summary - def get_queryset(self): - # Get all EvaluatorResult objects for this event_id and evaluator_id event_id = self.kwargs["event_id"] evaluator_id = self.kwargs["evaluator_id"] - - result_summary = EvaluatorResultSummary.objects.get( - event__id=event_id, evaluator__id=evaluator_id) - - queryset = EvaluatorResult.objects.filter( - result_summary=result_summary, - ).select_related( - "source_record" - ).order_by("source_record__activity_date") - - return queryset + return EvaluatorResultMaterializedView.objects.filter( + event_id=event_id, + evaluator_id=evaluator_id, + ).order_by("activity_date") def get(self, request, *args, **kwargs): # Override the default `get()` so we can error appropriately if @@ -279,43 +271,68 @@ def get(self, request, *args, **kwargs): ) logger.error(error['message']) return Response(error, status=status.HTTP_404_NOT_FOUND) + except ProgrammingError as e: + # Gracefully handle when the materialized view doesn't exist as a 503. + # It will raise a ProgrammingError, which we introspect to make sure + # it's caused by an UndefinedTable, otherwise we let it raise. + import psycopg + if not isinstance(e.__cause__, psycopg.errors.UndefinedTable): + raise + return Response( + format_error( + 503, + "Not available", + "Materialized view for evaluator results is not available", + request.path + ), + status=503 + ) + return Response(error, status=503) - def list(self, request, *args, **kwargs): - result_summary = self.get_result_summary() - event = result_summary.event - evaluator = result_summary.evaluator + def get_sample_queryset(self, queryset): + # We need sample ids from the EvaluatorResultSummary + event_id = self.kwargs["event_id"] + evaluator_id = self.kwargs["evaluator_id"] + result_summary = EvaluatorResultSummary.objects.get( + event_id=event_id, + evaluator__id=evaluator_id + ) + sample_ids = result_summary.sample_ids + + if sample_ids and (len(sample_ids) > 0): + queryset = queryset.filter(source_record_id__in=sample_ids) + else: + # OR select a random set of sample length from the queryset + queryset = queryset.order_by("?")[:settings.M2_RESULT_SAMPLE_SIZE] + + return queryset + def list(self, request, *args, **kwargs): # TODO: replace using DRF permissions/check_permissions() + event_id = self.kwargs["event_id"] + event = Metro2Event.objects.get(id=event_id) if not has_permissions_for_request(request, event): return HttpResponse('Unauthorized', status=401) + # Default to sample view view_param = self.request.query_params.get("view", "sample") - # If we're asked for a sample and S3 is enabled, quickly return - # the results from there. - if settings.S3_ENABLED and view_param == "sample": - return fetch_json_results_from_s3(request, event.id, evaluator.id) - - # Get the result set, performing any filtering as needed - queryset = self.filter_queryset(self.get_queryset()) - - # If we're asked for a sample, filter the result set by sample_ids if view_param == "sample": - sample_ids = result_summary.sample_ids - if sample_ids and (len(sample_ids) > 0): - queryset = queryset.filter(source_record__id__in=sample_ids) - else: - # OR select a random set of sample length from the queryset - queryset = queryset.order_by("?")[:settings.M2_RESULT_SAMPLE_SIZE] - - # Regardless, paginate the results. + # If we're asked for a sample and S3 is enabled, quickly return + # the results from there. + if settings.S3_ENABLED: + evaluator_id = self.kwargs["evaluator_id"] + return fetch_json_results_from_s3(request, event.id, evaluator_id) + + # Get a sample queryset + queryset = self.get_sample_queryset(self.get_queryset()) + else: + # Get the full result set, performing any filtering as needed + queryset = self.filter_queryset(self.get_queryset()) + + # Paginate and serialize results. # Pagination size should be the same as the sample size, so the # sample view will be exactly one page long. page = self.paginate_queryset(queryset) - paged_records = [result.source_record for result in page] - serializer = AccountActivitySerializer( - paged_records, - include_fields=evaluator.result_summary_fields(), - many=True, - ) + serializer = self.get_serializer(page, many=True) return self.get_paginated_response(serializer.data) diff --git a/django/seed-data.sh b/django/seed-data.sh index b4035094..c309011d 100755 --- a/django/seed-data.sh +++ b/django/seed-data.sh @@ -15,6 +15,10 @@ echo echo "Parse the data file and run the evaluators" python manage.py add_seed_data_local -e Sample-Dataset-007 +echo +echo "Generate the evaluator results materialized view" +python manage.py materialized_view + echo echo "Add the user as a member of the event" python manage.py add_user_to_event --event=Sample-Dataset-007 --user=admin