FindingCorelationBetweenCsvs.py

import pandas as pd
import sys
from scipy.stats import ks_2samp

first_file = sys.argv[1]
second_file = sys.argv[2]

def corr(first_file, second_file):
    # assuming first column is `class_name_id`
    first_df = pd.read_csv(first_file, index_col=0)
    second_df = pd.read_csv(second_file, index_col=0)
    class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']

    for class_name in class_names:
        # all correlations
        print('\n Class: %s' % class_name)
        print(' Pearson\'s correlation score: %0.6f' % first_df[class_name].corr( second_df[class_name], method='pearson'))
        print(' Kendall\'s correlation score: %0.6f' %first_df[class_name].corr(second_df[class_name], method='kendall'))
        print(' Spearman\'s correlation score: %0.6f' %first_df[class_name].corr(second_df[class_name], method='spearman'))
        ks_stat, p_value = ks_2samp(first_df[class_name].values,second_df[class_name].values)
        print(' Kolmogorov-Smirnov test:    KS-stat = %.6f    p-value = %.3e\n' % (ks_stat, p_value))

corr(first_file, second_file)