# HG changeset patch
# User Joseph Turian <turian@gmail.com>
# Date 1215496653 14400
# Node ID e4473d9697d768221faf7e5239aa7d49c4fd65c5
# Parent  42cc94cf6c122490c7c7c82c20c8be15e0a8a1ab
Added xent loss

diff -r 42cc94cf6c12 -r e4473d9697d7 nnet_ops.py
--- a/nnet_ops.py	Mon Jul 07 02:15:14 2008 -0400
+++ b/nnet_ops.py	Tue Jul 08 01:57:33 2008 -0400
@@ -379,4 +379,3 @@
 def crossentropy_softmax_1hot(x, y_idx, **kwargs):
     b = tensor.zeros_like(x[0,:])
     return crossentropy_softmax_1hot_with_bias(x, b, y_idx, **kwargs)
-
diff -r 42cc94cf6c12 -r e4473d9697d7 sparse_random_autoassociator/globals.py
--- a/sparse_random_autoassociator/globals.py	Mon Jul 07 02:15:14 2008 -0400
+++ b/sparse_random_autoassociator/globals.py	Tue Jul 08 01:57:33 2008 -0400
@@ -2,11 +2,12 @@
 Global variables.
 """
 
-INPUT_DIMENSION = 20
-HIDDEN_DIMENSION = 5
+INPUT_DIMENSION = 1000
+HIDDEN_DIMENSION = 100
 LEARNING_RATE = 0.1
 LR = LEARNING_RATE
 SEED = 666
-ZERO_SAMPLE_SIZE = 5
-MARGIN = 0.1
+ZERO_SAMPLE_SIZE = 50
+#ZERO_SAMPLE_SIZE = 250
+MARGIN = 0.25
 #MARGIN = 0.0
diff -r 42cc94cf6c12 -r e4473d9697d7 sparse_random_autoassociator/graph.py
--- a/sparse_random_autoassociator/graph.py	Mon Jul 07 02:15:14 2008 -0400
+++ b/sparse_random_autoassociator/graph.py	Tue Jul 08 01:57:33 2008 -0400
@@ -31,6 +31,15 @@
 #zeroloss = hingeloss(-t.max(-(ynonzero - xnonzero)) - yzero - MARGIN)
 loss = t.sum(nonzeroloss) + t.sum(zeroloss)
 
+def binary_crossentropy(output, target):
+    """
+    Compute the crossentropy of binary output wrt binary target.
+    @note: We do not sum, crossentropy is computed by component.
+    @todo: Rewrite as a scalar, and then broadcast to tensor.
+    """
+    return -(target * t.log(output) + (1 - target) * t.log(1 - output))
+#loss = t.sum(binary_crossentropy(ynonzero, xnonzero)) + t.sum(binary_crossentropy(yzero, t.constant(0)))
+
 (gw1nonzero, gb1, gw2nonzero, gw2zero, gb2nonzero, gb2zero) = t.grad(loss, [w1nonzero, b1, w2nonzero, w2zero, b2nonzero, b2zero])
 
 import theano.compile
diff -r 42cc94cf6c12 -r e4473d9697d7 sparse_random_autoassociator/main.py
--- a/sparse_random_autoassociator/main.py	Mon Jul 07 02:15:14 2008 -0400
+++ b/sparse_random_autoassociator/main.py	Tue Jul 08 01:57:33 2008 -0400
@@ -20,6 +20,8 @@
     xnonzero magnitude (this may be a limitation). Hence, all nonzeroes
     are equally important to exceed the maximum yzero.
 
+    (Alternately, there is a commented out binary xent loss.)
+
     LIMITATIONS:
        - Only does pure stochastic gradient (batchsize = 1).
        - Loss is irrespective of the xnonzero magnitude.