CollaborativeFiltering/experiment_test.py at master · JasonLC506/CollaborativeFiltering · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
import numpy as np
from trainingDataGenerator import datagenerator
from MultiMF import MultiMF
from TD import TD
from CD import CD
from TD01Loss import TD01Loss
from CD01Loss import CD01Loss
from MultiMF01Loss import MultiMF01Loss
from PITF import PITF
from NN1Layer import NN1Layer


def fit(data_train, data_valid, method, hyperparameters, max_epoch=10, SGDstep=0.001, SCALE = 0.1):

    training = datagenerator(data_train)
    valid = datagenerator(data_valid)

    model = method()
    model.logfilename += "_" + data_train[5:]
    model.modelconfigurefile += "_" + data_train[5:]
    model.fit(training, valid, model_hyperparameters = hyperparameters, max_epoch = max_epoch, SGDstep = SGDstep)
    return model.modelconfigurefile


def performance(data_test, method, modelconfigurefile):
    test = datagenerator(data_test)
    model = method()
    model.modelconfigLoad(modelconfigurefile)
    model_loss = model.loss(test)
    # model prediction distribution#
    dist = {}
    # 0-1 accuracy#
    correct = 0
    Ntest = 0
    for samp in test.sample(random = False):
        uid, iid, lid = samp
        lid_pred = model.predict(uid, iid, distribution=False)
        # lid_pred = model.predict(uid, iid)
        dist.setdefault(lid,[0,0])[0] += 1
        if lid == lid_pred:
            correct += 1
            dist[lid][1] += 1
        Ntest += 1
    accuracy = 1.0 * correct / Ntest
    print dist
    return [model_loss, accuracy]


def experiement(data_train, data_valid, data_test, method, hyperparameters, max_epoch=10, SGDstep=0.001, SCALE = 0.1):
    modelconfigurefile = fit(data_train, data_valid, method, hyperparameters, max_epoch = max_epoch, SGDstep = SGDstep, SCALE = SCALE)
    return performance(data_test, method, modelconfigurefile)


if __name__ == "__main__":
    np.random.seed(2017)
    datafile = "data/reaction_NYTWaPoWSJ_K10"
    data_train = datafile + "_0.7train"
    data_valid = datafile + "_0.1valid"
    data_test = datafile + "_0.2test"
    max_epoch = 500
    SGDstep = 0.01
    SCALE = 0.1

    # methods_list = [MultiMF, TD, CD]
    # method_names = ["MultiMF", "TD", "CD"]
    # hyperparameters_list_list = [[[3], [5], [15]],
    #                         [[9,9,3], [15,15,5], [45,45,15]],
    #                         [[9], [15], [45]]]
    method_names = ["MultiMF"]
    methods_list = [MultiMF]
    hyperparameters_list_list =[[[4]]]

    # method = NN1Layer
    # hyperparameters = [15]

    for i in range(len(methods_list)):
        method = methods_list[i]
        method_name = method_names[i]
        hyperparameters_list = hyperparameters_list_list[i]
        for hyperparameters in hyperparameters_list:
            print method_name
            print "hyperparameters =", hyperparameters

            # only fit model ##
            # print fit(data_train = data_train,
            #           data_valid = data_valid,
            #           method = method,
            #           hyperparameters = hyperparameters,
            #           max_epoch = max_epoch, SGDstep = SGDstep, SCALE = SCALE)

            ## only check model performance ##

            print performance(data_test = data_test,
                              method = method,
                              modelconfigurefile="modelconfigures/MultiMF_config_reaction_NYTWaPoWSJ_K10_0.7train[4]_SGDstep0.001_SCALE0.1")
                              # modelconfigurefile = "modelconfigures/NN1Layer_config_TDsynthetic_N500_M500_L3_K15_0.7train[15]_SGDstep0.01_SCALE0.1")

            ## fit & performance check ##
            # print experiement(data_train = data_train,
            #                   data_valid = data_valid,
            #                   data_test = data_test,
            #                   method = method,
            #                   hyperparameters = hyperparameters,
            #                   max_epoch = max_epoch, SGDstep = SGDstep, SCALE = SCALE)