# HG changeset patch
# User Joseph Turian <turian@gmail.com>
# Date 1220553990 14400
# Node ID d99fefbc9324bd2af09373e4aa201828bc640291
# Parent  117e5b09cf318b9a961af4d2130d77a4e677e286
Added a KL-divergence.

diff -r 117e5b09cf31 -r d99fefbc9324 cost.py
--- a/cost.py	Thu Sep 04 14:46:17 2008 -0400
+++ b/cost.py	Thu Sep 04 14:46:30 2008 -0400
@@ -6,6 +6,7 @@
 """
 
 import theano.tensor as T
+from xlogx import xlogx
 
 def quadratic(target, output, axis=1):
     return T.mean(T.sqr(target - output), axis)
@@ -16,3 +17,12 @@
     @warning: OUTPUT and TARGET are reversed in nnet_ops.binary_crossentropy
     """
     return -T.mean(target * T.log(output) + (1 - target) * T.log(1 - output), axis=axis)
+
+def KL_divergence(target, output):
+    """
+    @note: We do not compute the mean, because if target and output have
+    different shapes then the result will be garbled.
+    """
+    return -(target * T.log(output) + (1 - target) * T.log(1 - output)) \
+            + (xlogx(target) + xlogx(1 - target))
+#    return cross_entropy(target, output, axis) - cross_entropy(target, target, axis)