diff --git a/content_quiz_analysis.py b/content_quiz_analysis.py new file mode 100644 index 0000000..dbc8ac7 --- /dev/null +++ b/content_quiz_analysis.py @@ -0,0 +1,8 @@ +from util import ReadingLogsData +from scripts import content_quiz_attempts_analysis, content_quiz_grade_analysis + +if __name__ == '__main__': + reading_logs_data = ReadingLogsData() + reading_dict, quiz_dict = reading_logs_data.get_parsed_reading_log_data() + content_quiz_attempts_analysis(quiz_dict) + content_quiz_grade_analysis(quiz_dict) \ No newline at end of file diff --git a/schemas/course.py b/schemas/course.py index 6816784..521a0ce 100644 --- a/schemas/course.py +++ b/schemas/course.py @@ -34,7 +34,8 @@ class CourseSchema: ] @staticmethod - def page_is_valid(module_num: int, page_num: int) -> bool: + def page_is_valid(module_num: str, page_num: str) -> bool: + module_num, page_num = int(module_num), int(page_num) if module_num in CourseSchema.CANCELLED_MODULES or \ module_num in CourseSchema.OPTIONAL_MODULES or \ module_num in CourseSchema.SUMMARY_MODULES: diff --git a/scripts/__init__.py b/scripts/__init__.py index 166efaa..41a246b 100644 --- a/scripts/__init__.py +++ b/scripts/__init__.py @@ -12,3 +12,5 @@ from .analyze_module_feedback_survey import analyze_module_feedback, compare_module_feedback from .reading_logs import analyze_num_paragraphs, parse_reading_logs_module, parse_reading_logs_all, \ reading_logs_completion_time +from .content_quiz_analysis.analysis_on_content_quiz import content_quiz_attempts_analysis, content_quiz_grade_analysis + diff --git a/scripts/content_quiz_analysis/__init__.py b/scripts/content_quiz_analysis/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/content_quiz_analysis/analysis_on_content_quiz.py b/scripts/content_quiz_analysis/analysis_on_content_quiz.py new file mode 100644 index 0000000..e4e4c28 --- /dev/null +++ b/scripts/content_quiz_analysis/analysis_on_content_quiz.py @@ -0,0 +1,124 @@ +import matplotlib.pyplot as plt +import numpy as np +from util import ReadingLogsData +from schemas import CourseSchema + + +def content_quiz_attempts_analysis(content_quiz_dict): + r = ReadingLogsData() + + page_content_quiz_attempt_mean = [] + page_content_quiz_attempt_std = [] + content_quiz_attempt_pages = [] + + module_content_quiz_attempt_mean = [] + module_content_quiz_attempt_std = [] + content_quiz_attempt_modules = [] + + module_paragraphs_dict = r.get_module_paragraphs_dict() + for module_num, page_dict in module_paragraphs_dict.items(): + for page_num, page_data in page_dict.items(): + if not CourseSchema.page_is_valid(module_num, page_num): + continue + + # set up for per page analysis + page_attempt = r.page_content_quiz_num_attempts(module_num, page_num) + if page_attempt is not None: + content_quiz_attempt_pages.append(str(module_num)+"-"+str(page_num)) + page_content_quiz_attempt_mean.append(page_attempt[0]) + page_content_quiz_attempt_std.append(page_attempt[1]) + + # set up for per module analysis + module_attempt = r.module_content_quiz_num_attempts(module_num) + if module_attempt is not None: + content_quiz_attempt_modules.append(str(module_num)) + module_content_quiz_attempt_mean.append(module_attempt[0]) + module_content_quiz_attempt_std.append(module_attempt[1]) + + # Per page graph + page_content_quiz_attempt_std.append(np.mean(page_content_quiz_attempt_mean)) + page_content_quiz_attempt_mean.append(np.std(page_content_quiz_attempt_mean)) + content_quiz_attempt_pages.append('Overall') + + plt.scatter(content_quiz_attempt_pages, page_content_quiz_attempt_mean) + plt.errorbar(content_quiz_attempt_pages, page_content_quiz_attempt_mean, yerr=page_content_quiz_attempt_std, + fmt='o', ecolor="skyblue", elinewidth=0.5) + plt.xticks(fontsize=5) + plt.xlabel("Module_number-Page_number") + plt.ylabel("Number of attempts") + plt.title("Mean and Standard Deviation of Extra Attempts for Each Content Quiz") + plt.show() + + # Per module graph + module_content_quiz_attempt_mean.append(np.mean(module_content_quiz_attempt_mean)) + module_content_quiz_attempt_std.append(np.std(module_content_quiz_attempt_mean)) + content_quiz_attempt_modules.append('Overall') + + plt.scatter(content_quiz_attempt_modules, module_content_quiz_attempt_mean) + plt.errorbar(content_quiz_attempt_modules, module_content_quiz_attempt_mean, yerr=module_content_quiz_attempt_std, + fmt='o', ecolor="skyblue", elinewidth=0.5) + plt.xticks(fontsize=5) + plt.xlabel("Module_number") + plt.ylabel("Number of attempts") + plt.title("Mean and Standard Deviation of Extra Attempts for Each Module") + plt.show() + + +def content_quiz_grade_analysis(content_quiz_dict): + r = ReadingLogsData() + + page_content_quiz_grade_mean = [] + page_content_quiz_grade_std = [] + content_quiz_grade_pages = [] + + module_content_quiz_grade_mean = [] + module_content_quiz_grade_std = [] + content_quiz_grade_modules = [] + + module_paragraphs_dict = r.get_module_paragraphs_dict() + for module_num, page_dict in module_paragraphs_dict.items(): + for page_num, page_data in page_dict.items(): + if not CourseSchema.page_is_valid(module_num, page_num): + continue + + # set up for per page analysis + page_grade = r.page_content_quiz_first_attempt_grade(module_num=module_num, page_num=page_num) + if page_grade is not None: + content_quiz_grade_pages.append(str(module_num) + "-" + str(page_num)) + page_content_quiz_grade_mean.append(page_grade[0]) + page_content_quiz_grade_std.append(page_grade[1]) + + # set up for per module analysis + module_grade = r.module_content_quiz_first_attempt_grade(module_num) + if module_grade is not None: + content_quiz_grade_modules.append(str(module_num)) + module_content_quiz_grade_mean.append(module_grade[0]) + module_content_quiz_grade_std.append(module_grade[1]) + + # Per page graph + page_content_quiz_grade_mean.append(np.mean(page_content_quiz_grade_mean)) + page_content_quiz_grade_std.append(np.std(page_content_quiz_grade_mean)) + content_quiz_grade_pages.append('Overall') + + plt.scatter(content_quiz_grade_pages, page_content_quiz_grade_mean) + plt.errorbar(content_quiz_grade_pages, page_content_quiz_grade_mean, yerr=page_content_quiz_grade_std, + fmt='o', ecolor="skyblue", elinewidth=0.5) + plt.xticks(fontsize=5) + plt.xlabel("Module_number-Page_number") + plt.ylabel("Number of grades") + plt.title("Mean and Standard Deviation of First Attempt Grade for Each Content Quiz") + plt.show() + + # Per module graph + module_content_quiz_grade_mean.append(np.mean(module_content_quiz_grade_mean)) + module_content_quiz_grade_std.append(np.std(module_content_quiz_grade_mean)) + content_quiz_grade_modules.append('Overall') + + plt.scatter(content_quiz_grade_modules, module_content_quiz_grade_mean) + plt.errorbar(content_quiz_grade_modules, module_content_quiz_grade_mean, yerr=module_content_quiz_grade_std, + fmt='o', ecolor="skyblue", elinewidth=0.5) + plt.xticks(fontsize=5) + plt.xlabel("Module_number") + plt.ylabel("First attempt grades") + plt.title("Mean and Standard Deviation of First Attempt Grade for Each Module") + plt.show() diff --git a/scripts/reading_logs/parse_reading_logs/parse_reading_logs.py b/scripts/reading_logs/parse_reading_logs/parse_reading_logs.py index d144ae1..82d6227 100644 --- a/scripts/reading_logs/parse_reading_logs/parse_reading_logs.py +++ b/scripts/reading_logs/parse_reading_logs/parse_reading_logs.py @@ -1,3 +1,5 @@ +import pandas as pd +import os import json import math import os @@ -104,14 +106,12 @@ def parsing_each_continue(reading_log_folder_path: str, module_number: str, data for reading_log in os.listdir(reading_log_folder_path): reading_log_path = os.path.join(reading_log_folder_path, reading_log) - if not reading_log.endswith('.json'): + r = ReadingLogsData() + + if not r.is_reading_log_file(reading_log): continue reading_log_name_array = reading_log.split('-') - if '(' in reading_log and ')' in reading_log: # check for duplicate files - continue - if reading_log_name_array[0] != "COSC341": # make sure the file is a reading_log - continue if reading_log_name_array[1] != module_number: # make sure the reading log file is for the correct module continue @@ -146,9 +146,9 @@ def parsing_each_quiz_submit(reading_log_folder_path: str, module_number: str, d for reading_log in os.listdir(reading_log_folder_path): reading_log_path = os.path.join(reading_log_folder_path, reading_log) - if not reading_log.endswith('.json'): - continue - if '(' in reading_log and ')' in reading_log: # check for duplicate files + r = ReadingLogsData() + + if not r.is_reading_log_file(reading_log): continue reading_log_name_array = reading_log.split('-') diff --git a/util/__init__.py b/util/__init__.py index cf62cc9..37a3f29 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -4,6 +4,7 @@ from .const import KEY_PATH, MODULE_PARAGRAPHS_OUTPUT_FILEPATH, CACHE_FOLDER from .canvas_api import setup_submissions_filepath, get_quiz_id_from_file_name, DateTimeEncoder from .data_cleaner import keep_latest_survey_attempt -from .util import normalize -from .plots import set_plot_settings from .reading_logs import ReadingLogsData +from .plots import set_plot_settings +from .util import normalize + diff --git a/util/reading_logs.py b/util/reading_logs.py index 01a71a7..1106d1d 100644 --- a/util/reading_logs.py +++ b/util/reading_logs.py @@ -1,6 +1,8 @@ import json import os -import statistics +import numpy as np +import pandas as pd +from random import random import dill import pandas as pd @@ -11,6 +13,7 @@ START_TIME_KEY = 'start_time' END_TIME_KEY = 'end_time' DURATION_KEY = 'duration' +TAMPERED = -1 class ReadingLogsData: @@ -125,7 +128,7 @@ def page_content_quiz_num_attempts(self, module_num: int, page_num: int, data448 """ Retrieves the average number of content quiz attempts before a correct answer for a page. If given a data448_id, only retrieves that student's reading speed for the page. Without a data448_id, retrieves the - average reading speed of that page. + average number of attempts of that page. Returns None if this page has no content quiz questions. Standard deviation is None if a data448_id is given. @@ -136,24 +139,6 @@ def page_content_quiz_num_attempts(self, module_num: int, page_num: int, data448 :return: (float representing the average number of content quiz attempts, standard deviation for this average) """ page_content_quiz_df = self.get_content_quiz_performance_dict()[f'{module_num}-{page_num}'] - TAMPERED = -1 - - def num_before_ans(*args): - q_response_lists = [] - for val in args: - q_response_lists.append(val) if isinstance(val, list) else q_response_lists.append([val]) - - # Discard if they tampered with the numbers so questions have a different number of attempts - if not all(len(response_set) == len(q_response_lists[0]) for response_set in q_response_lists): - return TAMPERED - - all_correct = ['ans'] * len(q_response_lists) - count = 0 - for q_response_set in zip(*q_response_lists): - if list(q_response_set) == all_correct: - return count / len(q_response_lists) - else: - count += 1 zip_set = [page_content_quiz_df[col] for col in page_content_quiz_df if col.startswith('q')] if not zip_set: @@ -171,11 +156,109 @@ def module_content_quiz_num_attempts(self, module_num: int, data448_id: int = No content_quiz_attempts_per_page = [] module_paragraphs_dict = self.get_module_paragraphs_dict() for page_num in module_paragraphs_dict[str(module_num)].keys(): - average_num_attempts, _ = self.page_content_quiz_num_attempts(module_num, int(page_num), data448_id) - content_quiz_attempts_per_page.append(average_num_attempts) + page_attempts = self.page_content_quiz_num_attempts(module_num, int(page_num), data448_id) + if page_attempts is not None: + average_num_attempts = page_attempts[0] + content_quiz_attempts_per_page.append(average_num_attempts) return aggregate_and_sd(content_quiz_attempts_per_page) + def page_content_quiz_attempts_list(self, module_num: int, page_num: int) -> [float]: + """ + returns a list of extra attempts each student took for the content quiz of the given page + :param module_num: The module number + :param page_num: The page number + :return: + """ + page_content_quiz_df = self.get_content_quiz_performance_dict()[f'{module_num}-{page_num}'] + + zip_set = [page_content_quiz_df[col] for col in page_content_quiz_df if col.startswith('q')] + if not zip_set: + return None + + page_content_quiz_df['num_before_ans'] = [num_before_ans(*a) for a in zip(*zip_set)] + + all_num_attempts = [num for num in page_content_quiz_df['num_before_ans'].values if num != TAMPERED] + + return all_num_attempts + + def page_content_quiz_first_attempt_grade(self, module_num: int, page_num: int, data448_id: int = None) -> float: + """ + Retrieves the average first attempt grade for content quiz. If given a data448_id, only retrieves that student's + reading speed for the page. Without a data448_id, retrieves the average first attempt grade of that page. + + Returns None if this page has no content quiz questions. + + :param module_num: The module number + :param page_num: The page number + :param data448_id: A student's Data 448 id + :return: (float representing the average first attempt grade of the given content quiz, standard deviation of + this average) + """ + + page_content_quiz_df = self.get_content_quiz_performance_dict()[f'{module_num}-{page_num}'] + + zip_set = [page_content_quiz_df[col] for col in page_content_quiz_df if col.startswith('q')] + + if not zip_set: + return None + + cols = [col for col in page_content_quiz_df if col.startswith('q')] + questions_df = page_content_quiz_df[cols] + + def first_attempt_grade(row): + count = 0 + for element in row: + if element == 'ans' or element[0] == 'ans': + count += 1 + return count/len(row) + + questions_df['first_attempt_grade'] = questions_df.apply(lambda x: first_attempt_grade(x), axis=1) + + if data448_id: + return questions_df['first_attempt_grade'][f'{data448_id}'], None + + return aggregate_and_sd(questions_df['first_attempt_grade']) + + def module_content_quiz_first_attempt_grade(self, module_num: int, data448_id: int = None) -> (float, float): + content_quiz_first_attempt_grade = [] + module_paragraphs_dict = self.get_module_paragraphs_dict() + for page_num in module_paragraphs_dict[str(module_num)].keys(): + page_grade = self.page_content_quiz_first_attempt_grade(module_num, int(page_num), data448_id) + if page_grade is not None: + average_grade = page_grade[0] + content_quiz_first_attempt_grade.append(average_grade) + + return aggregate_and_sd(content_quiz_first_attempt_grade) + + def content_quiz_first_attempt_grade_list(self, module_num: int, page_num: int): + """ + Returns the list of student's firat attempt grade for the given page + :param module_num: the module number + :param page_num: the page number + :return: + """ + page_content_quiz_df = self.get_content_quiz_performance_dict()[f'{module_num}-{page_num}'] + + zip_set = [page_content_quiz_df[col] for col in page_content_quiz_df if col.startswith('q')] + + if not zip_set: + return None + + cols = [col for col in page_content_quiz_df if col.startswith('q')] + questions_df = page_content_quiz_df[cols] + + def first_attempt_grade(row): + count = 0 + for element in row: + if element == 'ans' or element[0] == 'ans': + count += 1 + return count / len(row) + + questions_df['first_attempt_grade'] = questions_df.apply(lambda x: first_attempt_grade(x), axis=1) + + return [*questions_df['first_attempt_grade']] + def get_paragraph_list(self, module_num: int, page_num: int) -> [str]: module_paragraphs_dict = self.get_module_paragraphs_dict() page_paragraphs = [] @@ -209,6 +292,17 @@ def page_reading_duration(self, module_num: int, page_num: int, data448_id: int return mean_duration, mean_duration_std + def is_reading_log_file(reading_log_file_name) -> bool: + if not reading_log_file_name.endswith('.json'): + return False + if '(' in reading_log_file_name and ')' in reading_log_file_name: # check for duplicate files + return False + + reading_log_name_array = reading_log_file_name.split('-') + if reading_log_name_array[0] != "COSC341" & reading_log_name_array[3] != "Reading" & reading_log_name_array[ + 4] != "Logs": # make sure the file is a reading_log + return False + def page_reading_duration_list(self, module_num: int, page_num: int) -> [float]: """Returns a list of student page reading duration in minutes""" reading_duration_dict = self.get_reading_duration_dict() @@ -255,7 +349,7 @@ def ms_to_minutes(duration_ms: float): def aggregate_and_sd(values: [], mean=True) -> (float, float): values_list = list(values) if len(values_list) > 1: - sd = statistics.stdev(values_list) + sd = np.std(values_list) else: sd = 0 if mean: @@ -263,3 +357,22 @@ def aggregate_and_sd(values: [], mean=True) -> (float, float): return mean, sd else: return sum(values_list), sd + + +def num_before_ans(*args): + q_response_lists = [] + for val in args: + q_response_lists.append(val) if isinstance(val, list) else q_response_lists.append([val]) + + # Discard if they tampered with the numbers so questions have a different number of attempts + if not all(len(response_set) == len(q_response_lists[0]) for response_set in q_response_lists): + return TAMPERED + + all_correct = ['ans'] * len(q_response_lists) + count = 0 + for q_response_set in zip(*q_response_lists): + if list(q_response_set) == all_correct: + return count / len(q_response_lists) + else: + count += 1 +