--- a/pylearn/formulas/costs.py	Wed Oct 13 15:05:16 2010 -0400
+++ b/pylearn/formulas/costs.py	Thu Oct 14 13:33:33 2010 -0400
@@ -1,10 +1,23 @@
 """
-Common training criteria.
+
+TODO: make sur stabilization optimization are done.
+TODO: make test
+TODO: check that this work for nd tensor.
 """
+
+#"""
+#Common training criteria.
+#"""
 import theano
 import theano.tensor as T
+
 from tags import tags
 
+__authors__   = "Frederic Bastien, Nicolas Boulanger-Lewandowski, .."
+__copyright__ = "(c) 2010, Universite de Montreal"
+__license__   = "3-clause BSD License"
+__contact__   = "theano-user <theano-users@googlegroups.com>"
+
 @tags('cost','binary','cross-entropy')
 def binary_crossentropy(output, target):
     """ Compute the crossentropy of binary output wrt binary target.
@@ -16,66 +29,141 @@
     :param output: Binary output or prediction :math:`\in[0,1]`
     :type target: Theano variable
     :param target: Binary target usually :math:`\in\{0,1\}`
+
+    :note: no stabilization optimization needed for a generic output variable
     """
     return -(target * T.log(output) + (1.0 - target) * T.log(1.0 - output))
 
 
+@tags('cost','binary','cross-entropy', 'sigmoid')
+def sigmoid_crossentropy(output, target):
+    """ crossentropy of a sigmoid activation
+
+    .. math::
+                L_{CE} \equiv t\log(\sigma(a)) + (1-t)\log(1-\sigma(a))
+
+    :type output: Theano variable
+    :param output: Output before activation
+    :type target: Theano variable
+    :param target: Target
+
+    :note: no stabilization done. 
+    """
+    return target * (- T.log(1.0 + T.exp(-output))) + (1.0 - target) * (- T.log(1.0 + T.exp(output)))
+
+@tags('cost','binary','cross-entropy', 'tanh')
+def tanh_crossentropy(output, target):
+    """ crossentropy of a tanh activation
+
+    .. math::
+                L_{CE} \equiv t\log(\\frac{1+\\tanh(a)}2) + (1-t)\log(\\frac{1-\\tanh(a)}2)
+
+    :type output: Theano variable
+    :param output: Output before activation
+    :type target: Theano variable
+    :param target: Target
+
+    :note: no stabilization done. 
+    """
+    return sigmoid_crossentropy(2.0*output, target)
+
+@tags('cost','binary','cross-entropy', 'tanh', 'abs')
+def abstanh_crossentropy(output, target):
+    """ crossentropy of a absolute value tanh activation
+
+    .. math::
+                L_{CE} \equiv t\log(\\frac{1+\\tanh(|a|)}2) + (1-t)\log(\\frac{1-\\tanh(|a|)}2)
+
+    :type output: Theano variable
+    :param output: Output before activation
+    :type target: Theano variable
+    :param target: Target
+
+    :note: no stabilization done. 
+    """
+    return tanh_crossentropy(T.abs_(output), target)
+
+@tags('cost','binary','cross-entropy', 'tanh', 'normalized')
+def normtanh_crossentropy(output, target):
+    """ crossentropy of a "normalized" tanh activation (LeCun)
+
+    .. math::
+                L_{CE} \equiv t\log(\\frac{1+\\tanh(0.6666a)}2) + (1-t)\log(\\frac{1-\\tanh(0.6666a)}2)
+
+    :type output: Theano variable
+    :param output: Output before activation
+    :type target: Theano variable
+    :param target: Target
+
+    :note: no stabilization done. 
+    """
+    return tanh_crossentropy(0.6666*output, target)
+
+@tags('cost','binary','cross-entropy', 'tanh', 'normalized', 'abs')
+def absnormtanh_cross_entropy(output, target):
+    """ crossentropy of a "absolute normalized" tanh activation
+
+    .. math::
+                L_{CE} \equiv t\log(\\frac{1+\\tanh(0.6666*|a|)}2) + (1-t)\log(\\frac{1-\\tanh(0.6666*|a|)}2)
+
+    :type output: Theano variable
+    :param output: Output before activation
+    :type target: Theano variable
+    :param target: Target
+
+    :note: no stabilization done. 
+    """
+    return normtanh_crossentropy(T.abs_(output), target)
+
+def cross_entropy(output_act, output, target, act=None):
+    """ Execute the cross entropy with a sum on the last dimension and a mean on the first dimension.
+
+    If act is in 'sigmoid', 'tanh', 'tanhnorm', 'abstanh', 'abstanhnorm' we 
+    call the specialized version.
+    
+    .. math::
+        mean(sum(sqr(output-target),axis=-1),axis=0)
+
+    :type output_act: Theano variable
+    :param output_act: Output after activation
+    :type output: Theano variable
+    :param output: Output before activation
+    :type target:Theano variable
+    :param target: Target
+    :type act: str or None
+    :param act: The type of activation used
+    """
+    if act in ['sigmoid','tanh','tanhnorm','abstanh','abstanhnorm']:
+        if act == 'sigmoid':
+            return sigmoid_crossentropy(output, target)
+        if act == 'tanh':
+            return tanh_crossentropy(output, target)
+        if act == 'tanhnorm':
+            return normtanh_crossentropy(output, target)
+        if act == 'abstanh':
+            return abstanh_crossentropy(output, target)
+        if act == 'abstanhnorm':
+            return absnormtanh_cross_entropy(output, target)
+    elif act is None:
+        XE = target * T.log(output_act) + (1 - target) * T.log(1 - output_act)
+        return -T.mean(T.sum(XE, axis=-1),axis=0)
+    else:
+        raise Exception("cross_entropy() Expected parameter act to be in ['sigmoid','tanh','tanhnorm','abstanh','abstanhnorm', None]")
+
+def quadratic_cost(output, target):
+    """ The quadratic cost of output again target with a sum on the last dimension and a mean on the first dimension.
+    
+    .. math::
+        mean(sum(sqr(output-target),axis=-1),axis=0)
+
+    :type output: Theano variable
+    :param output: The value that we want to compare again target
+    :type target:Theano variable
+    :param target: The value that we consider correct
+    """
+    return T.mean(T.sum(T.sqr(output - target), axis=-1),axis=0)
+
+
 # This file seems like it has some overlap with theano.tensor.nnet.  Which functions should go
 # in which file?
 
-@tags('cost','binary','cross-entropy', 'sigmoid')
-def sigmoid_crossentropy(output_act, target):
-    """ Stable crossentropy of a sigmoid activation
-
-    .. math::
-                L_{CE} \equiv t\log(\sigma(a)) + (1-t)\log(1-\sigma(a))
-
-    :type output_act: Theano variable
-    :param output: Activation
-    :type target: Theano variable
-    :param target: Binary target usually :math:`\in\{0,1\}`
-    """
-    return target * (- T.log(1.0 + T.exp(-output_act))) + (1.0 - target) * (- T.log(1.0 + T.exp(output_act)))
-
-@tags('cost','binary','cross-entropy', 'tanh')
-def tanh_crossentropy(output_act, target):
-    """ Stable crossentropy of a tanh activation
-
-    .. math::
-                L_{CE} \equiv t\log(\\frac{1+\\tanh(a)}2) + (1-t)\log(\\frac{1-\\tanh(a)}2)
-
-    :type output_act: Theano variable
-    :param output: Activation
-    :type target: Theano variable
-    :param target: Binary target usually :math:`\in\{0,1\}`
-    """
-    return sigmoid_crossentropy(2.0*output_act, target)
-
-@tags('cost','binary','cross-entropy', 'tanh', 'abs')
-def abstanh_crossentropy(output_act, target):
-    """ Stable crossentropy of a absolute value tanh activation
-
-    .. math::
-                L_{CE} \equiv t\log(\\frac{1+\\tanh(|a|)}2) + (1-t)\log(\\frac{1-\\tanh(|a|)}2)
-
-    :type output_act: Theano variable
-    :param output: Activation
-    :type target: Theano variable
-    :param target: Binary target usually :math:`\in\{0,1\}`
-    """
-    return tanh_crossentropy(T.abs_(output_act), target)
-
-@tags('cost','binary','cross-entropy', 'tanh', "normalized")
-def normtanh_crossentropy(output_act, target):
-    """ Stable crossentropy of a "normalized" tanh activation (LeCun)
-
-    .. math::
-                L_{CE} \equiv t\log(\\frac{1+\\tanh(0.6666a)}2) + (1-t)\log(\\frac{1-\\tanh(0.6666a)}2)
-
-    :type output_act: Theano variable
-    :param output: Activation
-    :type target: Theano variable
-    :param target: Binary target usually :math:`\in\{0,1\}`
-    """
-    return tanh_crossentropy(0.6666*output_act, target)
-