RAAM/SimpleNet.m at master · gregwayne/RAAM · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
classdef SimpleNet

    properties

        W1; % First Layer Weight
        W2;

        R0;
        X0;
        X1;
        R1;
        X2;
        R2;
        D0;
        D1;
        D2;

        N0;
        N1;
        N2;

        t;
        F;
        Fr;
        eta;

        elig;

    end

    methods

        function obj = SimpleNet(N0,N1,N2,g,eta)

            obj.N0      = N0;
            obj.N1      = N1;
            obj.N2      = N2;

            obj.W1      = g*randn(N1,N0)/sqrt(N0);
            obj.W2      = g*randn(N2,N1)/sqrt(N1);

            obj.X1      = zeros(N1,1);
            obj.R1      = zeros(N1,1);
            obj.X2      = zeros(N2,1);
            obj.R2      = zeros(N2,1);

            obj.t       = 2;
            obj.F       = eye(N1,N1);
            obj.Fr      = zeros(N1,1);
            obj.eta     = eta;

            obj.elig    = zeros(N2,N1);

        end

        function obj = FProp(obj,X0)

            obj.X0      = X0;
            obj.R0      = tanh(obj.X0);

            obj.X1      = obj.W1*obj.R0;
            obj.R1      = tanh(obj.X1);

            obj.X2      = obj.W2*obj.R1;
            obj.R2      = obj.X2; % Option for a Transfer Function

        end

        function obj = BProp(obj,E2)

            obj.D2  = E2;
            E1      = obj.W2'*obj.D2;
            obj.D1  = (1-obj.R1.^2).*E1;
            E0      = obj.W1'*obj.D1;
            obj.D0  = (1-obj.R0.^2).*E0;

        end

        function obj = Minimize(obj,X0,target,eta_min,iters,constraint_on,constraint)

            for iter=1:iters

                obj = obj.FProp([1;X0]);
                obj = obj.BProp(target-obj.R2);
                if constraint_on
                    X0  = LinearThreshold(X0 + eta_min*obj.D0(2:end) + eta_min*(constraint-X0));
                else
                    X0  = LinearThreshold(X0 + eta_min*obj.D0(2:end));
                end

            end

        end

        function X2 = FastProp(obj,X0)

            R0          = tanh(X0);
            X1          = obj.W1*R0;
            R1          = tanh(X1);
            X2          = obj.W2*R1;

        end

        function obj = Fisher(obj)

            eps_t       = 1/obj.t;
            meps_t      = 1-eps_t;

            obj.t       = obj.t + 1;
            F           = obj.F;
            r           = obj.R1;
            Fr          = F*r;
            obj.Fr      = Fr;

            obj.F       = (meps_t)^(-1)*(F - eps_t*Fr*Fr'/(meps_t + eps_t*r'*Fr));
            obj.t       = obj.t + 1;

        end

        function obj = ErrorLearn(obj,delta,fisher_on)

            if fisher_on
                obj         = obj.Fisher();
            end
            obj.elig    = obj.elig + obj.eta*delta*obj.Fr'/(obj.R1'*obj.Fr);

        end

        function obj = Tag(obj)

            obj.W2      = obj.W2 + obj.elig;

        end

        function obj = Untag(obj)

            obj.elig    = 0*obj.elig;

        end

    end

end

function y = LinearThreshold(x)

    y = ((x > -1).*(x < 1)).*x - (x < -1) + (x > 1);

end