Mercurial > pylearn

comparison nnet_ops.py @ 69:8c2607f387e6

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
added softplus, elaborated sigmoid
author James Bergstra <bergstrj@iro.umontreal.ca>
date Mon, 21 Apr 2008 15:23:49 -0400
parents 315eb36ff954
children 76e5c0f37165
comparison
equal deleted inserted replaced
68:315eb36ff954 69:8c2607f387e6
1 import theano 1 import theano
2 from theano import tensor, gof, scalar 2 from theano import tensor, gof, scalar
3 import numpy 3 import numpy
4 4
5 class ScalarSigmoid(scalar.UnaryScalarOp): 5 ############
6 #
7 # SCALAR OPS
8 #
9
10 class ScalarSigmoid(scalar.FloatUnaryScalarOp):
11 @staticmethod
12 def st_impl(x):
13 if x < -30.0:
14 return 0.0
15 if x > 30.0:
16 return 1.0
17 return 1.0 / (1.0 + numpy.exp(-x))
6 def impl(self, x): 18 def impl(self, x):
7 return 1.0 / (1 + numpy.exp(-x)) 19 return ScalarSigmoid.st_impl(x)
8 def grad(self, (x,), (gz,)): 20 def grad(self, (x,), (gz,)):
9 return gz * scalar_sigmoid(x) * (1.0 - scalar_sigmoid(x)), 21 y = scalar_sigmoid(x)
10 def c_foreach(self, (x,), (z,)): 22 return [gz * y * (1.0 - y)]
11 return "%(z)s = 1.0 / (1 + exp(-%(x)s));" % locals() 23 def c_foreach(self, (x,), (z,), sub):
24 if 'float' in self.inputs[0].dtype:
25 return """%(z)s =
26 %(x)s < -30.0
27 ? 0.0
28 : %(x)s > 30.0
29 ? 1.0
30 : 1.0 /(1.0+exp(-%(x)s));""" % locals()
31 raise NotImplementedError('only floatingpoint is implemented')
12 scalar_sigmoid = gof.op.constructor(ScalarSigmoid) 32 scalar_sigmoid = gof.op.constructor(ScalarSigmoid)
13 Sigmoid, sigmoid, SigmoidInplace, sigmoid_inplace \ 33 Sigmoid, sigmoid, SigmoidInplace, sigmoid_inplace =\
14 = theano.tensor.broadcast(ScalarSigmoid, 'Sigmoid') 34 tensor.broadcast(ScalarSigmoid, 'Sigmoid')
15 35
36 class ScalarSoftplus(scalar.FloatUnaryScalarOp):
37 @staticmethod
38 def static_impl(x):
39 if x < -30.0:
40 return 0.0
41 if x > 30.0:
42 return x
43 return numpy.log1p(numpy.exp(x))
44 def impl(self, x):
45 return ScalarSoftplus.static_impl(x)
46 def grad(self, (x,), (gz,)):
47 return [gz * scalar_sigmoid(x)]
48 def c_foreach(self, (x,), (z,), sub):
49 if 'float' in self.inputs[0].dtype:
50 return """%(z)s =
51 %(x)s < -30.0
52 ? 0.0
53 : %(x)s > 30.0
54 ? %(x)s
55 : log1p(exp(%(x)s));""" % locals()
56 raise NotImplementedError('only floating point x is implemented')
57 scalar_softplus = gof.op.constructor(ScalarSoftplus)
58 Softplus, softplus, SoftplusInplace, softplus_inplace =\
59 tensor.broadcast(ScalarSoftplus, 'Softplus')
60
61
62 ############
63 #
64 # TENSOR OPS
65 #
16 66
17 67
18 class CrossentropySoftmax1Hot(gof.op.Op): 68 class CrossentropySoftmax1Hot(gof.op.Op):
19 """A special compound Op for the output of neural-net classifiers. 69 """A special compound Op for the output of neural-net classifiers.
20 70
215 + log(sum); 265 + log(sum);
216 //mat_at(y,i,0) = -log( mat_at(s,i,t[i])); //less accurate? 266 //mat_at(y,i,0) = -log( mat_at(s,i,t[i])); //less accurate?
217 //mat_at(y,i,0) = - mat_at(x,i,t[i]) - mat_at(b,0,t[i]) + (discount_max ? maxi : 0.0) + log(sum); 267 //mat_at(y,i,0) = - mat_at(x,i,t[i]) - mat_at(b,0,t[i]) + (discount_max ? maxi : 0.0) + log(sum);
218 } 268 }
219 """ % dict(locals(), **sub) 269 """ % dict(locals(), **sub)
220
221
222 270
223 crossentropy_softmax_1hot = gof.op.constructor(CrossentropySoftmax1Hot) 271 crossentropy_softmax_1hot = gof.op.constructor(CrossentropySoftmax1Hot)
224 272
225 class CrossentropySoftmax1HotDx (gof.op.Op): 273 class CrossentropySoftmax1HotDx (gof.op.Op):
226 nin=3 274 nin=3
297 %(fail)s; 345 %(fail)s;
298 } 346 }
299 dx_i[y_i * Sdx] -= dnll_i; 347 dx_i[y_i * Sdx] -= dnll_i;
300 } 348 }
301 """ % dict(locals(), **sub) 349 """ % dict(locals(), **sub)
350