# HG changeset patch
# User delallea@valhalla.apstat.com
# Date 1222969303 14400
# Node ID 6e7509acb1c03c37e0118a3ae57f2dd75a3e96a9
# Parent  ce6b4fd3ab2960c4aefcdaba85be6277b5acba0b# Parent  739612d316a4502521d9ecad89dca35062de9c9c
Merged

diff -r ce6b4fd3ab29 -r 6e7509acb1c0 _test_xlogx.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/_test_xlogx.py	Thu Oct 02 13:41:43 2008 -0400
@@ -0,0 +1,27 @@
+from xlogx import xlogx
+
+import unittest
+from theano import compile
+from theano import gradient
+
+from theano.tensor import as_tensor
+import theano._test_tensor as TT
+
+import random
+import numpy.random
+
+class T_XlogX(unittest.TestCase):
+    def test0(self):
+        x = as_tensor([1, 0])
+        y = xlogx(x)
+        y = compile.eval_outputs([y])
+        self.failUnless(numpy.all(y == numpy.asarray([0, 0.])))
+    def test1(self):
+        class Dummy(object):
+            def make_node(self, a):
+                return [xlogx(a)[:,2]]
+        TT.verify_grad(self, Dummy(), [numpy.random.rand(3,4)])
+
+
+if __name__ == '__main__':
+    unittest.main()
diff -r ce6b4fd3ab29 -r 6e7509acb1c0 cost.py
--- a/cost.py	Thu Sep 04 13:48:47 2008 -0400
+++ b/cost.py	Thu Oct 02 13:41:43 2008 -0400
@@ -6,6 +6,7 @@
 """
 
 import theano.tensor as T
+from xlogx import xlogx
 
 def quadratic(target, output, axis=1):
     return T.mean(T.sqr(target - output), axis)
@@ -16,3 +17,12 @@
     @warning: OUTPUT and TARGET are reversed in nnet_ops.binary_crossentropy
     """
     return -T.mean(target * T.log(output) + (1 - target) * T.log(1 - output), axis=axis)
+
+def KL_divergence(target, output):
+    """
+    @note: We do not compute the mean, because if target and output have
+    different shapes then the result will be garbled.
+    """
+    return -(target * T.log(output) + (1 - target) * T.log(1 - output)) \
+            + (xlogx(target) + xlogx(1 - target))
+#    return cross_entropy(target, output, axis) - cross_entropy(target, target, axis)
diff -r ce6b4fd3ab29 -r 6e7509acb1c0 dataset.py
--- a/dataset.py	Thu Sep 04 13:48:47 2008 -0400
+++ b/dataset.py	Thu Oct 02 13:41:43 2008 -0400
@@ -605,7 +605,7 @@
         @type fieldname: any type 
 
         @param values: bits near the beginning or end of the dataset 
-        @type values: list of minibatches (returned by minibatch_nowrap) 
+        @type values: list of minibatches (returned by minibatches_nowrap) 
 
         @return: the concatenation (stacking) of the values 
         @rtype: something suitable as a minibatch field 
diff -r ce6b4fd3ab29 -r 6e7509acb1c0 xlogx.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/xlogx.py	Thu Oct 02 13:41:43 2008 -0400
@@ -0,0 +1,28 @@
+
+import theano
+from theano import tensor, scalar
+import numpy
+
+class XlogX(scalar.UnaryScalarOp):
+    """
+    Compute X * log(X), with special case 0 log(0) = 0.
+    """
+    @staticmethod
+    def st_impl(x):
+        if x == 0.0:
+            return 0.0
+        return x * numpy.log(x)
+    def impl(self, x):
+        return XlogX.st_impl(x)
+    def grad(self, (x,), (gz,)):
+        return [gz * (1 + scalar.log(x))]
+    def c_code(self, node, name, (x,), (z,), sub):
+        if node.inputs[0].type in [scalar.float32, scalar.float64]:
+            return """%(z)s =
+                %(x)s == 0.0
+                ? 0.0
+                : %(x)s * log(%(x)s);""" % locals()
+        raise NotImplementedError('only floatingpoint is implemented')
+scalar_xlogx  = XlogX(scalar.upgrade_to_float, name='scalar_xlogx')
+xlogx = tensor.Elemwise(scalar_xlogx, name='xlogx')
+