Mercurial > pylearn

annotate denoising_aa.py @ 210:ffd50efefb70

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
work in progress denoising auto-encoder
author Yoshua Bengio <bengioy@iro.umontreal.ca>
date Sat, 17 May 2008 00:01:47 -0400
parents
children bd728c83faff
rev   line source
210
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1 """
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
2 A denoising auto-encoder
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
3 """
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
4
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
5 import theano
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
6 from theano.formula import *
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
7 from learner import *
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
8 from theano import tensor as t
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
9 from nnet_ops import *
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
10 import math
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
11 from misc import *
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
12 from theano.tensor_random import binomial
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
13
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
14 def hiding_corruption_formula(seed,average_fraction_hidden):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
15 """
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
16 Return a formula for the corruption process, in which a random
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
17 subset of the input numbers are hidden (mapped to 0).
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
18
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
19 @param seed: seed of the random generator
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
20 @type seed: anything that numpy.random.RandomState accepts
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
21
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
22 @param average_fraction_hidden: the probability with which each
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
23 input number is hidden (set to 0).
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
24 @type average_fraction_hidden: 0 <= real number <= 1
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
25 """
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
26 class HidingCorruptionFormula(Formulas):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
27 x = t.matrix()
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
28 corrupted_x = x * binomial(seed,x,1,fraction_sampled)
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
29
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
30 return HidingCorruptionFormula()
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
31
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
32 def squash_affine_formula(squash_function=sigmoid):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
33 """
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
34 By convention prefix the parameters by _
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
35 """
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
36 class SquashAffineFormula(Formulas):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
37 x = t.matrix() # of dimensions minibatch_size x n_inputs
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
38 _b = t.row() # of dimensions 1 x n_outputs
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
39 _W = t.matrix() # of dimensions n_inputs x n_outputs
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
40 a = _b + t.dot(x,_W) # of dimensions minibatch_size x n_outputs
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
41 y = squash_function(a)
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
42 return SquashAffineFormula()
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
43
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
44 def gradient_descent_update_formula():
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
45 class GradientDescentUpdateFormula(Formula):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
46 param = t.matrix()
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
47 learning_rate = t.scalar()
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
48 cost = t.column() # cost of each example in a minibatch
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
49 param_update = t.add_inplace(param, -learning_rate*t.sgrad(cost))
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
50 return gradient_descent_update_formula()
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
51
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
52 def probabilistic_classifier_loss_formula():
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
53 class ProbabilisticClassifierLossFormula(Formulas):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
54 a = t.matrix() # of dimensions minibatch_size x n_classes, pre-softmax output
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
55 target_class = t.ivector() # dimension (minibatch_size)
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
56 nll, probability_predictions = crossentropy_softmax_1hot(a, target_class)
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
57 return ProbabilisticClassifierLossFormula()
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
58
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
59 def binomial_cross_entropy_formula():
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
60 class BinomialCrossEntropyFormula(Formulas):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
61 a = t.matrix() # pre-sigmoid activations, minibatch_size x dim
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
62 p = sigmoid(a) # model prediction
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
63 q = t.matrix() # target binomial probabilities, minibatch_size x dim
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
64 # using the identity softplus(a) - softplus(-a) = a,
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
65 # we obtain that q log(p) + (1-q) log(1-p) = q a - softplus(a)
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
66 nll = -t.sum(q*a - softplus(-a))
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
67
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
68 def squash_affine_autoencoder_formula(hidden_squash=t.tanh,
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
69 reconstruction_squash=sigmoid,
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
70 share_weights=True,
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
71 reconstruction_nll_formula=binomial_cross_entropy_formula(),
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
72 update_formula=gradient_descent_update_formula):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
73 if share_weights:
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
74 autoencoder = squash_affine_formula(hidden_squash).rename(a='code_a') + \
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
75 squash_affine_formula(reconstruction_squash).rename(x='hidden',y='reconstruction',_b='_c') + \
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
76 reconstruction_nll_formula
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
77 else:
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
78 autoencoder = squash_affine_formula(hidden_squash).rename(a='code_a',_W='_W1') + \
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
79 squash_affine_formula(reconstruction_squash).rename(x='hidden',y='reconstruction',_b='_c',_W='_W2') + \
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
80 reconstruction_nll_formula
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
81 autoencoder = autoencoder + [update_formula().rename(cost = 'nll',
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
82 param = p)
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
83 for p in autoencoder.get_all('_.*')]
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
84 return autoencoder
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
85
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
86
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
87 # @todo: try other corruption formulae. The above is the default one.
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
88 # not quite used in the ICML paper... (had a fixed number of 0s).
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
89
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
90 class DenoisingAutoEncoder(LearningAlgorithm):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
91
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
92 def __init__(self,n_inputs,n_hidden_per_layer,
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
93 learning_rate=0.1,
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
94 max_n_epochs=100,
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
95 L1_regularizer=0,
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
96 init_range=1.,
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
97 corruption_formula = hiding_corruption_formula(),
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
98 autoencoder = squash_affine_autoencoder_formula(),
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
99 minibatch_size=None,linker = "c|py"):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
100 for name,val in locals().items():
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
101 if val is not self: self.__setattribute__(name,val)
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
102 self.denoising_autoencoder_formula = corruption_formula + autoencoder.rename(x='corrupted_x')
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
103
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
104 def __call__(self, training_set=None):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
105 model = DenoisingAutoEncoderModel(self)
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
106 if training_set:
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
107
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
108 def compile(self, inputs, outputs):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
109 return theano.function(inputs,outputs,unpack_single=False,linker=self.linker)
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
110
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
111 class DenoisingAutoEncoderModel(LearnerModel):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
112 def __init__(self,learning_algorithm,params):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
113 self.learning_algorithm=learning_algorithm
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
114 self.params=params
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
115 v = learning_algorithm.v
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
116 self.update_fn = learning_algorithm.compile(learning_algorithm.denoising_autoencoder_formula.inputs,
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
117 learning_algorithm.denoising_autoencoder_formula.outputs)
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
118
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
119 def update(self, training_set, train_stats_collector=None):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
120
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
121
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
122 # old stuff
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
123
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
124 # self._learning_rate = t.scalar('learning_rate') # this is the symbol
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
125 # self.L1_regularizer = L1_regularizer
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
126 # self._L1_regularizer = t.scalar('L1_regularizer')
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
127 # self._input = t.matrix('input') # n_examples x n_inputs
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
128 # self._W = t.matrix('W')
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
129 # self._b = t.row('b')
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
130 # self._c = t.row('b')
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
131 # self._regularization_term = self._L1_regularizer * t.sum(t.abs(self._W))
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
132 # self._corrupted_input = corruption_process(self._input)
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
133 # self._hidden = t.tanh(self._b + t.dot(self._input, self._W.T))
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
134 # self._reconstruction_activations =self._c+t.dot(self._hidden,self._W)
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
135 # self._nll,self._output = crossentropy_softmax_1hot(Print("output_activations")(self._output_activations),self._target_vector)
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
136 # self._output_class = t.argmax(self._output,1)
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
137 # self._class_error = t.neq(self._output_class,self._target_vector)
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
138 # self._minibatch_criterion = self._nll + self._regularization_term / t.shape(self._input)[0]
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
139 # OnlineGradientTLearner.__init__(self)
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
140
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
141 # def attributeNames(self):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
142 # return ["parameters","b1","W2","b2","W2", "L2_regularizer","regularization_term"]
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
143
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
144 # def parameterAttributes(self):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
145 # return ["b1","W1", "b2", "W2"]
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
146
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
147 # def updateMinibatchInputFields(self):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
148 # return ["input","target"]
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
149
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
150 # def updateEndOutputAttributes(self):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
151 # return ["regularization_term"]
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
152
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
153 # def lossAttribute(self):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
154 # return "minibatch_criterion"
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
155
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
156 # def defaultOutputFields(self, input_fields):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
157 # output_fields = ["output", "output_class",]
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
158 # if "target" in input_fields:
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
159 # output_fields += ["class_error", "nll"]
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
160 # return output_fields
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
161
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
162 # def allocate(self,minibatch):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
163 # minibatch_n_inputs = minibatch["input"].shape[1]
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
164 # if not self._n_inputs:
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
165 # self._n_inputs = minibatch_n_inputs
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
166 # self.b1 = numpy.zeros((1,self._n_hidden))
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
167 # self.b2 = numpy.zeros((1,self._n_outputs))
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
168 # self.forget()
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
169 # elif self._n_inputs!=minibatch_n_inputs:
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
170 # # if the input changes dimension on the fly, we resize and forget everything
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
171 # self.forget()
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
172
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
173 # def forget(self):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
174 # if self._n_inputs:
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
175 # r = self._init_range/math.sqrt(self._n_inputs)
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
176 # self.W1 = numpy.random.uniform(low=-r,high=r,
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
177 # size=(self._n_hidden,self._n_inputs))
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
178 # r = self._init_range/math.sqrt(self._n_hidden)
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
179 # self.W2 = numpy.random.uniform(low=-r,high=r,
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
180 # size=(self._n_outputs,self._n_hidden))
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
181 # self.b1[:]=0
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
182 # self.b2[:]=0
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
183 # self._n_epochs=0
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
184
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
185 # def isLastEpoch(self):
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
186 # self._n_epochs +=1
ffd50efefb70 work in progress denoising auto-encoder
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
187 # return self._n_epochs>=self._max_n_epochs