Mercurial > pylearn
annotate algorithms/daa.py @ 487:94a4c5b7293b
DAA code more generic:
Can now choose activation function and reconstruction cost.
author | Joseph Turian <turian@gmail.com> |
---|---|
date | Tue, 28 Oct 2008 02:21:50 -0400 |
parents | 5ccb1662f9f6 |
children | e06666ac32d5 |
rev | line source |
---|---|
476
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
1 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
2 import theano |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
3 from theano import tensor as T |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
4 from theano.tensor import nnet as NN |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
5 import numpy as N |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
6 |
487 | 7 from pylearn import cost as cost |
8 | |
476
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
9 class DenoisingAA(T.RModule): |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
10 |
487 | 11 def __init__(self, input = None, regularize = True, tie_weights = True, |
12 activation_function=NN.sigmoid, reconstruction_cost_function=cost.cross_entropy): | |
13 """ | |
14 @param reconstruction_cost: Should return one cost per example (row) | |
15 """ | |
476
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
16 super(DenoisingAA, self).__init__() |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
17 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
18 # MODEL CONFIGURATION |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
19 self.regularize = regularize |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
20 self.tie_weights = tie_weights |
487 | 21 self.activation_function = activation_function |
22 self.reconstruction_cost_function = reconstruction_cost_function | |
476
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
23 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
24 # ACQUIRE/MAKE INPUT |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
25 if not input: |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
26 input = T.matrix('input') |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
27 self.input = theano.External(input) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
28 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
29 # HYPER-PARAMETERS |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
30 self.lr = theano.Member(T.scalar()) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
31 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
32 # PARAMETERS |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
33 self.w1 = theano.Member(T.matrix()) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
34 if not tie_weights: |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
35 self.w2 = theano.Member(T.matrix()) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
36 else: |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
37 self.w2 = self.w1.T |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
38 self.b1 = theano.Member(T.vector()) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
39 self.b2 = theano.Member(T.vector()) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
40 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
41 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
42 # REGULARIZATION COST |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
43 self.regularization = self.build_regularization() |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
44 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
45 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
46 ### NOISELESS ### |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
47 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
48 # HIDDEN LAYER |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
49 self.hidden_activation = T.dot(self.input, self.w1) + self.b1 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
50 self.hidden = self.hid_activation_function(self.hidden_activation) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
51 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
52 # RECONSTRUCTION LAYER |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
53 self.output_activation = T.dot(self.hidden, self.w2) + self.b2 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
54 self.output = self.out_activation_function(self.output_activation) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
55 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
56 # RECONSTRUCTION COST |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
57 self.reconstruction_costs = self.build_reconstruction_costs(self.output) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
58 self.reconstruction_cost = T.mean(self.reconstruction_costs) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
59 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
60 # TOTAL COST |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
61 self.cost = self.reconstruction_cost |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
62 if self.regularize: |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
63 self.cost = self.cost + self.regularization |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
64 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
65 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
66 ### WITH NOISE ### |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
67 self.corrupted_input = self.build_corrupted_input() |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
68 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
69 # HIDDEN LAYER |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
70 self.nhidden_activation = T.dot(self.corrupted_input, self.w1) + self.b1 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
71 self.nhidden = self.hid_activation_function(self.nhidden_activation) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
72 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
73 # RECONSTRUCTION LAYER |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
74 self.noutput_activation = T.dot(self.nhidden, self.w2) + self.b2 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
75 self.noutput = self.out_activation_function(self.noutput_activation) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
76 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
77 # RECONSTRUCTION COST |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
78 self.nreconstruction_costs = self.build_reconstruction_costs(self.noutput) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
79 self.nreconstruction_cost = T.mean(self.nreconstruction_costs) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
80 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
81 # TOTAL COST |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
82 self.ncost = self.nreconstruction_cost |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
83 if self.regularize: |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
84 self.ncost = self.ncost + self.regularization |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
85 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
86 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
87 # GRADIENTS AND UPDATES |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
88 if self.tie_weights: |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
89 self.params = self.w1, self.b1, self.b2 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
90 else: |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
91 self.params = self.w1, self.w2, self.b1, self.b2 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
92 gradients = T.grad(self.ncost, self.params) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
93 updates = dict((p, p - self.lr * g) for p, g in zip(self.params, gradients)) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
94 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
95 # INTERFACE METHODS |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
96 self.update = theano.Method(self.input, self.ncost, updates) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
97 self.compute_cost = theano.Method(self.input, self.cost) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
98 self.noisify = theano.Method(self.input, self.corrupted_input) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
99 self.reconstruction = theano.Method(self.input, self.output) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
100 self.representation = theano.Method(self.input, self.hidden) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
101 self.reconstruction_through_noise = theano.Method(self.input, [self.corrupted_input, self.noutput]) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
102 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
103 def _instance_initialize(self, obj, input_size = None, hidden_size = None, seed = None, **init): |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
104 if (input_size is None) ^ (hidden_size is None): |
486
5ccb1662f9f6
Small change to error message
Joseph Turian <turian@gmail.com>
parents:
485
diff
changeset
|
105 raise ValueError("Must specify input_size and hidden_size or neither.") |
476
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
106 super(DenoisingAA, self)._instance_initialize(obj, **init) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
107 if seed is not None: |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
108 R = N.random.RandomState(seed) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
109 else: |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
110 R = N.random |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
111 if input_size is not None: |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
112 sz = (input_size, hidden_size) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
113 inf = 1/N.sqrt(input_size) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
114 hif = 1/N.sqrt(hidden_size) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
115 obj.w1 = R.uniform(size = sz, low = -inf, high = inf) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
116 if not self.tie_weights: |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
117 obj.w2 = R.uniform(size = list(reversed(sz)), low = -inf, high = inf) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
118 obj.b1 = N.zeros(hidden_size) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
119 obj.b2 = N.zeros(input_size) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
120 if seed is not None: |
482 | 121 obj.seed(seed) |
476
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
122 obj.__hide__ = ['params'] |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
123 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
124 def build_regularization(self): |
487 | 125 """ |
126 @todo: Why do we need this function? | |
127 """ | |
476
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
128 return T.zero() # no regularization! |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
129 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
130 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
131 class SigmoidXEDenoisingAA(DenoisingAA): |
487 | 132 """ |
133 @todo: Merge this into the above. | |
134 """ | |
476
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
135 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
136 def build_corrupted_input(self): |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
137 self.noise_level = theano.Member(T.scalar()) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
138 return self.random.binomial(T.shape(self.input), 1, 1 - self.noise_level) * self.input |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
139 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
140 def hid_activation_function(self, activation): |
487 | 141 return self.activation_function(activation) |
476
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
142 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
143 def out_activation_function(self, activation): |
487 | 144 return self.activation_function(activation) |
476
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
145 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
146 def build_reconstruction_costs(self, output): |
487 | 147 return self.reconstruction_cost_function(self.input, output) |
476
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
148 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
149 def build_regularization(self): |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
150 self.l2_coef = theano.Member(T.scalar()) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
151 if self.tie_weights: |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
152 return self.l2_coef * T.sum(self.w1 * self.w1) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
153 else: |
485
e8c37244b54f
Small bugfix in regularization
Joseph Turian <turian@gmail.com>
parents:
482
diff
changeset
|
154 return self.l2_coef * (T.sum(self.w1 * self.w1) + T.sum(self.w2 * self.w2)) |
476
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
155 |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
156 def _instance_initialize(self, obj, input_size = None, hidden_size = None, seed = None, **init): |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
157 init.setdefault('noise_level', 0) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
158 init.setdefault('l2_coef', 0) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
159 super(SigmoidXEDenoisingAA, self)._instance_initialize(obj, input_size, hidden_size, seed, **init) |
8fcd0f3d9a17
added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff
changeset
|
160 |