Mercurial > pylearn

comparison sandbox/denoising_aa.py @ 409:cf22ebfc90eb

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Moved denoising AA to sandbox
author Joseph Turian <turian@gmail.com>
date Thu, 10 Jul 2008 17:33:28 -0400
parents denoising_aa.py@eded3cb54930
children
comparison
equal deleted inserted replaced
407:b9f545594207 409:cf22ebfc90eb
1 """
2 A denoising auto-encoder
3
4 @warning: You should use this interface. It is not complete and is not functional.
5 Instead, use::
6 ssh://projects@lgcm.iro.umontreal.ca/repos/denoising_aa
7 """
8
9 import theano
10 from theano.formula import *
11 from learner import *
12 from theano import tensor as t
13 from nnet_ops import *
14 import math
15 from misc import *
16 from misc_theano import *
17 from theano.tensor_random import binomial
18
19 def hiding_corruption_formula(seed,average_fraction_hidden):
20 """
21 Return a formula for the corruption process, in which a random
22 subset of the input numbers are hidden (mapped to 0).
23
24 @param seed: seed of the random generator
25 @type seed: anything that numpy.random.RandomState accepts
26
27 @param average_fraction_hidden: the probability with which each
28 input number is hidden (set to 0).
29 @type average_fraction_hidden: 0 <= real number <= 1
30 """
31 class HidingCorruptionFormula(Formulas):
32 x = t.matrix()
33 corrupted_x = x * binomial(seed,x,1,fraction_sampled)
34
35 return HidingCorruptionFormula()
36
37 def squash_affine_formula(squash_function=sigmoid):
38 """
39 Simply does: squash_function(b + xW)
40 By convention prefix the parameters by _
41 """
42 class SquashAffineFormula(Formulas):
43 x = t.matrix() # of dimensions minibatch_size x n_inputs
44 _b = t.row() # of dimensions 1 x n_outputs
45 _W = t.matrix() # of dimensions n_inputs x n_outputs
46 a = _b + t.dot(x,_W) # of dimensions minibatch_size x n_outputs
47 y = squash_function(a)
48 return SquashAffineFormula()
49
50 def gradient_descent_update_formula():
51 class GradientDescentUpdateFormula(Formula):
52 param = t.matrix()
53 learning_rate = t.scalar()
54 cost = t.column() # cost of each example in a minibatch
55 param_update = t.add_inplace(param, -learning_rate*t.sgrad(cost))
56 return gradient_descent_update_formula()
57
58 def probabilistic_classifier_loss_formula():
59 class ProbabilisticClassifierLossFormula(Formulas):
60 a = t.matrix() # of dimensions minibatch_size x n_classes, pre-softmax output
61 target_class = t.ivector() # dimension (minibatch_size)
62 nll, probability_predictions = crossentropy_softmax_1hot(a, target_class) # defined in nnet_ops.py
63 return ProbabilisticClassifierLossFormula()
64
65 def binomial_cross_entropy_formula():
66 class BinomialCrossEntropyFormula(Formulas):
67 a = t.matrix() # pre-sigmoid activations, minibatch_size x dim
68 p = sigmoid(a) # model prediction
69 q = t.matrix() # target binomial probabilities, minibatch_size x dim
70 # using the identity softplus(a) - softplus(-a) = a,
71 # we obtain that q log(p) + (1-q) log(1-p) = q a - softplus(a)
72 nll = -t.sum(q*a - softplus(-a))
73 # next line was missing... hope it's all correct above
74 return BinomialCrossEntropyFormula()
75
76 def squash_affine_autoencoder_formula(hidden_squash=t.tanh,
77 reconstruction_squash=sigmoid,
78 share_weights=True,
79 reconstruction_nll_formula=binomial_cross_entropy_formula(),
80 update_formula=gradient_descent_update_formula):
81 if share_weights:
82 autoencoder = squash_affine_formula(hidden_squash).rename(a='code_a') + \
83 squash_affine_formula(reconstruction_squash).rename(x='hidden',y='reconstruction',_b='_c') + \
84 reconstruction_nll_formula
85 else:
86 autoencoder = squash_affine_formula(hidden_squash).rename(a='code_a',_W='_W1') + \
87 squash_affine_formula(reconstruction_squash).rename(x='hidden',y='reconstruction',_b='_c',_W='_W2') + \
88 reconstruction_nll_formula
89 autoencoder = autoencoder + [update_formula().rename(cost = 'nll',
90 param = p)
91 for p in autoencoder.get_all('_.*')]
92 return autoencoder
93
94
95 # @todo: try other corruption formulae. The above is the default one.
96 # not quite used in the ICML paper... (had a fixed number of 0s).
97
98 class DenoisingAutoEncoder(LearningAlgorithm):
99
100 def __init__(self,n_inputs,n_hidden_per_layer,
101 learning_rate=0.1,
102 max_n_epochs=100,
103 L1_regularizer=0,
104 init_range=1.,
105 corruption_formula = hiding_corruption_formula(),
106 autoencoder = squash_affine_autoencoder_formula(),
107 minibatch_size=None,linker = "c|py"):
108 for name,val in locals().items():
109 if val is not self: self.__setattribute__(name,val)
110 self.denoising_autoencoder_formula = corruption_formula + autoencoder.rename(x='corrupted_x')
111
112 def __call__(self, training_set=None):
113 """ Allocate and optionnaly train a model
114
115 @TODO enables passing in training and valid sets, instead of cutting one set in 80/20
116 """
117 model = DenoisingAutoEncoderModel(self)
118 if training_set:
119 print 'DenoisingAutoEncoder(): what do I do if training_set????'
120 # copied from old mlp_factory_approach:
121 if len(trainset) == sys.maxint:
122 raise NotImplementedError('Learning from infinite streams is not supported')
123 nval = int(self.validation_portion * len(trainset))
124 nmin = len(trainset) - nval
125 assert nmin >= 0
126 minset = trainset[:nmin] #real training set for minimizing loss
127 valset = trainset[nmin:] #validation set for early stopping
128 best = model
129 for stp in self.early_stopper():
130 model.update(
131 minset.minibatches([input, target], minibatch_size=min(32,
132 len(trainset))))
133 #print 'mlp.__call__(), we did an update'
134 if stp.set_score:
135 stp.score = model(valset, ['loss_01'])
136 if (stp.score < stp.best_score):
137 best = copy.copy(model)
138 model = best
139 # end of the copy from mlp_factory_approach
140
141 return model
142
143
144 def compile(self, inputs, outputs):
145 return theano.function(inputs,outputs,unpack_single=False,linker=self.linker)
146
147 class DenoisingAutoEncoderModel(LearnerModel):
148 def __init__(self,learning_algorithm,params):
149 self.learning_algorithm=learning_algorithm
150 self.params=params
151 v = learning_algorithm.v
152 self.update_fn = learning_algorithm.compile(learning_algorithm.denoising_autoencoder_formula.inputs,
153 learning_algorithm.denoising_autoencoder_formula.outputs)
154
155 def update(self, training_set, train_stats_collector=None):
156
157 print 'dont update you crazy frog!'
158
159 # old stuff
160
161 # self._learning_rate = t.scalar('learning_rate') # this is the symbol
162 # self.L1_regularizer = L1_regularizer
163 # self._L1_regularizer = t.scalar('L1_regularizer')
164 # self._input = t.matrix('input') # n_examples x n_inputs
165 # self._W = t.matrix('W')
166 # self._b = t.row('b')
167 # self._c = t.row('b')
168 # self._regularization_term = self._L1_regularizer * t.sum(t.abs(self._W))
169 # self._corrupted_input = corruption_process(self._input)
170 # self._hidden = t.tanh(self._b + t.dot(self._input, self._W.T))
171 # self._reconstruction_activations =self._c+t.dot(self._hidden,self._W)
172 # self._nll,self._output = crossentropy_softmax_1hot(Print("output_activations")(self._output_activations),self._target_vector)
173 # self._output_class = t.argmax(self._output,1)
174 # self._class_error = t.neq(self._output_class,self._target_vector)
175 # self._minibatch_criterion = self._nll + self._regularization_term / t.shape(self._input)[0]
176 # OnlineGradientTLearner.__init__(self)
177
178 # def attributeNames(self):
179 # return ["parameters","b1","W2","b2","W2", "L2_regularizer","regularization_term"]
180
181 # def parameterAttributes(self):
182 # return ["b1","W1", "b2", "W2"]
183
184 # def updateMinibatchInputFields(self):
185 # return ["input","target"]
186
187 # def updateEndOutputAttributes(self):
188 # return ["regularization_term"]
189
190 # def lossAttribute(self):
191 # return "minibatch_criterion"
192
193 # def defaultOutputFields(self, input_fields):
194 # output_fields = ["output", "output_class",]
195 # if "target" in input_fields:
196 # output_fields += ["class_error", "nll"]
197 # return output_fields
198
199 # def allocate(self,minibatch):
200 # minibatch_n_inputs = minibatch["input"].shape[1]
201 # if not self._n_inputs:
202 # self._n_inputs = minibatch_n_inputs
203 # self.b1 = numpy.zeros((1,self._n_hidden))
204 # self.b2 = numpy.zeros((1,self._n_outputs))
205 # self.forget()
206 # elif self._n_inputs!=minibatch_n_inputs:
207 # # if the input changes dimension on the fly, we resize and forget everything
208 # self.forget()
209
210 # def forget(self):
211 # if self._n_inputs:
212 # r = self._init_range/math.sqrt(self._n_inputs)
213 # self.W1 = numpy.random.uniform(low=-r,high=r,
214 # size=(self._n_hidden,self._n_inputs))
215 # r = self._init_range/math.sqrt(self._n_hidden)
216 # self.W2 = numpy.random.uniform(low=-r,high=r,
217 # size=(self._n_outputs,self._n_hidden))
218 # self.b1[:]=0
219 # self.b2[:]=0
220 # self._n_epochs=0
221
222 # def isLastEpoch(self):
223 # self._n_epochs +=1
224 # return self._n_epochs>=self._max_n_epochs