# HG changeset patch
# User James Bergstra <bergstrj@iro.umontreal.ca>
# Date 1208805829 14400
# Node ID 8c2607f387e61ec543df1bbc758ef73cb99a17d5
# Parent  315eb36ff9542f1e3bcbe06989b279e7794c8061
added softplus, elaborated sigmoid

diff -r 315eb36ff954 -r 8c2607f387e6 _nnet_ops.py
--- a/_nnet_ops.py	Fri Apr 18 03:49:17 2008 -0400
+++ b/_nnet_ops.py	Mon Apr 21 15:23:49 2008 -0400
@@ -11,6 +11,11 @@
     def test_elemwise(self):
         TT.verify_grad(self, Sigmoid, [numpy.random.rand(3,4)])
 
+class T_softplus(unittest.TestCase):
+    def setUp(self):
+        numpy.random.seed(9999)
+    def test_elemwise(self):
+        TT.verify_grad(self, Softplus, [numpy.random.rand(3,4)])
 
 class T_CrossentropySoftmax1Hot(unittest.TestCase):
     def setUp(self):
diff -r 315eb36ff954 -r 8c2607f387e6 nnet_ops.py
--- a/nnet_ops.py	Fri Apr 18 03:49:17 2008 -0400
+++ b/nnet_ops.py	Mon Apr 21 15:23:49 2008 -0400
@@ -2,17 +2,67 @@
 from theano import tensor, gof, scalar
 import numpy
 
-class ScalarSigmoid(scalar.UnaryScalarOp):
+############
+#
+# SCALAR OPS
+#
+
+class ScalarSigmoid(scalar.FloatUnaryScalarOp):
+    @staticmethod
+    def st_impl(x):
+        if x < -30.0:
+            return 0.0
+        if x > 30.0:
+            return 1.0 
+        return 1.0 / (1.0 + numpy.exp(-x))
     def impl(self, x):
-        return 1.0 / (1 + numpy.exp(-x))
+        return ScalarSigmoid.st_impl(x)
     def grad(self, (x,), (gz,)):
-        return gz * scalar_sigmoid(x) * (1.0 - scalar_sigmoid(x)),
-    def c_foreach(self, (x,), (z,)): 
-        return "%(z)s = 1.0 / (1 + exp(-%(x)s));" % locals()
+        y = scalar_sigmoid(x)
+        return [gz * y * (1.0 - y)]
+    def c_foreach(self, (x,), (z,), sub):
+        if 'float' in self.inputs[0].dtype:
+            return """%(z)s =
+                %(x)s < -30.0 
+                ? 0.0 
+                : %(x)s > 30.0 
+                   ? 1.0
+                   : 1.0 /(1.0+exp(-%(x)s));""" % locals()
+        raise NotImplementedError('only floatingpoint is implemented')
 scalar_sigmoid = gof.op.constructor(ScalarSigmoid)
-Sigmoid, sigmoid, SigmoidInplace, sigmoid_inplace \
-        = theano.tensor.broadcast(ScalarSigmoid, 'Sigmoid')
+Sigmoid, sigmoid, SigmoidInplace, sigmoid_inplace =\
+        tensor.broadcast(ScalarSigmoid, 'Sigmoid')
 
+class ScalarSoftplus(scalar.FloatUnaryScalarOp):
+    @staticmethod
+    def static_impl(x):
+        if x < -30.0:
+            return 0.0
+        if x > 30.0:
+            return x
+        return numpy.log1p(numpy.exp(x))
+    def impl(self, x):
+        return ScalarSoftplus.static_impl(x)
+    def grad(self, (x,), (gz,)):
+        return [gz * scalar_sigmoid(x)]
+    def c_foreach(self, (x,), (z,), sub):
+        if 'float' in self.inputs[0].dtype:
+            return """%(z)s =
+                %(x)s < -30.0 
+                ? 0.0 
+                : %(x)s > 30.0 
+                   ? %(x)s
+                   : log1p(exp(%(x)s));""" % locals()
+        raise NotImplementedError('only floating point x is implemented')
+scalar_softplus = gof.op.constructor(ScalarSoftplus)
+Softplus, softplus, SoftplusInplace, softplus_inplace =\
+        tensor.broadcast(ScalarSoftplus, 'Softplus')
+
+
+############
+#
+# TENSOR OPS
+#
 
 
 class CrossentropySoftmax1Hot(gof.op.Op):
@@ -218,8 +268,6 @@
         }
         """ % dict(locals(), **sub)
 
-
-
 crossentropy_softmax_1hot = gof.op.constructor(CrossentropySoftmax1Hot)
 
 class CrossentropySoftmax1HotDx (gof.op.Op):
@@ -299,3 +347,4 @@
             dx_i[y_i * Sdx] -= dnll_i;
         }
         """ % dict(locals(), **sub)
+