# HG changeset patch
# User James Bergstra <bergstrj@iro.umontreal.ca>
# Date 1224705413 14400
# Node ID bd937e845bbb6c68e646822a67d16c03711e501c
# Parent  4335309f4924b1774f89165dd920f20a97e4cfc3
new stuff: algorithms/logistic_regression, datasets/MNIST

diff -r 4335309f4924 -r bd937e845bbb algorithms/__init__.py
diff -r 4335309f4924 -r bd937e845bbb algorithms/_test_logistic_regression.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/algorithms/_test_logistic_regression.py	Wed Oct 22 15:56:53 2008 -0400
@@ -0,0 +1,60 @@
+from logistic_regression import *
+import sys, time
+
+if __name__ == '__main__':
+    pprint.assign(nnet_ops.crossentropy_softmax_1hot_with_bias_dx, printing.FunctionPrinter('xsoftmaxdx'))
+    pprint.assign(nnet_ops.crossentropy_softmax_argmax_1hot_with_bias, printing.FunctionPrinter('nll', 'softmax', 'argmax'))
+    if 1:
+        lrc = Module_Nclass()
+
+        print '================'
+        print lrc.update.pretty()
+        print '================'
+        print lrc.update.pretty(mode = theano.Mode('py', 'fast_run'))
+        print '================'
+#         print lrc.update.pretty(mode = compile.FAST_RUN.excluding('inplace'))
+#         print '================'
+
+#        sys.exit(0)
+
+        lr = lrc.make(10, 2, mode=theano.Mode('c|py', 'fast_run'))
+        #lr = lrc.make(10, 2, mode=compile.FAST_RUN.excluding('fast_run'))
+        #lr = lrc.make(10, 2, mode=theano.Mode('py', 'merge')) #'FAST_RUN')
+
+        data_x = N.random.randn(5, 10)
+        data_y = (N.random.randn(5) > 0)
+
+        t = time.time()
+        for i in xrange(10000):
+            lr.lr = 0.02
+            xe = lr.update(data_x, data_y) 
+            #if i % 100 == 0:
+            #    print i, xe
+
+        print 'training time:', time.time() - t
+        print 'final error', xe
+
+        #print
+        #print 'TRAINED MODEL:'
+        #print lr
+
+    if 0:
+        lrc = Module()
+
+        lr = lrc.make(10, mode=theano.Mode('c|py', 'merge')) #'FAST_RUN')
+
+        data_x = N.random.randn(5, 10)
+        data_y = (N.random.randn(5, 1) > 0)
+
+        for i in xrange(10000):
+            xe = lr.update(data_x, data_y)
+            if i % 100 == 0:
+                print i, xe
+
+        print
+        print 'TRAINED MODEL:'
+        print lr
+
+
+
+
diff -r 4335309f4924 -r bd937e845bbb algorithms/logistic_regression.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/algorithms/logistic_regression.py	Wed Oct 22 15:56:53 2008 -0400
@@ -0,0 +1,95 @@
+import theano
+from theano import tensor as T
+from theano.tensor import nnet_ops
+from theano.compile import module
+from theano import printing, pprint
+from theano import compile
+
+import numpy as N
+
+
+class Module_Nclass(module.FancyModule):
+    class __instance_type__(module.FancyModuleInstance):
+        def initialize(self, n_in, n_out, rng=N.random):
+            #self.component is the LogisticRegressionTemplate instance that built this guy.
+
+            self.w = rng.randn(n_in, n_out)
+            self.b = rng.randn(n_out)
+            self.lr = 0.01
+            self.__hide__ = ['params']
+
+    def __init__(self, x=None, targ=None, w=None, b=None, lr=None):
+        super(Module_Nclass, self).__init__() #boilerplate
+
+        self.x = x if x is not None else T.matrix()
+        self.targ = targ if targ is not None else T.lvector()
+
+        self.w = w if w is not None else module.Member(T.dmatrix())
+        self.b = b if b is not None else module.Member(T.dvector())
+        self.lr = lr if lr is not None else module.Member(T.dscalar())
+
+        self.params = [p for p in [self.w, self.b] if p.owner is None]
+
+        xent, y = nnet_ops.crossentropy_softmax_1hot(
+                T.dot(self.x, self.w) + self.b, self.targ)
+        sum_xent = T.sum(xent)
+
+        self.y = y
+        self.sum_xent = sum_xent
+
+        #define the apply method
+        self.pred = T.argmax(T.dot(self.x, self.w) + self.b, axis=1)
+        self.apply = module.Method([self.x], self.pred)
+
+        if self.params:
+            gparams = T.grad(sum_xent, self.params)
+
+            self.update = module.Method([self.x, self.targ], sum_xent,
+                    updates = dict((p, p - self.lr * g) for p, g in zip(self.params, gparams)))
+
+class Module(module.FancyModule):
+    class __instance_type__(module.FancyModuleInstance):
+        def initialize(self, n_in):
+            #self.component is the LogisticRegressionTemplate instance that built this guy.
+
+            self.w = N.random.randn(n_in,1)
+            self.b = N.random.randn(1)
+            self.lr = 0.01
+            self.__hide__ = ['params']
+
+    def __init__(self, x=None, targ=None, w=None, b=None, lr=None):
+        super(Module, self).__init__() #boilerplate
+
+        self.x = x if x is not None else T.matrix()
+        self.targ = targ if targ is not None else T.lcol()
+
+        self.w = w if w is not None else module.Member(T.dmatrix())
+        self.b = b if b is not None else module.Member(T.dvector())
+        self.lr = lr if lr is not None else module.Member(T.dscalar())
+
+        self.params = [p for p in [self.w, self.b] if p.owner is None]
+
+        y = nnet_ops.sigmoid(T.dot(self.x, self.w))
+        xent = -self.targ * T.log(y) - (1.0 - self.targ) * T.log(1.0 - y)
+        sum_xent = T.sum(xent)
+
+        self.y = y
+        self.xent = xent
+        self.sum_xent = sum_xent
+
+        #define the apply method
+        self.pred = (T.dot(self.x, self.w) + self.b) > 0.0
+        self.apply = module.Method([self.x], self.pred)
+
+        #if this module has any internal parameters, define an update function for them
+        if self.params:
+            gparams = T.grad(sum_xent, self.params)
+            self.update = module.Method([self.x, self.targ], sum_xent,
+                                        updates = dict((p, p - self.lr * g) for p, g in zip(self.params, gparams)))
+
+
+
+class Learner(object):
+    """TODO: Encapsulate the algorithm for finding an optimal regularization coefficient"""
+    pass
+
diff -r 4335309f4924 -r bd937e845bbb amat.py
--- a/amat.py	Tue Oct 21 16:32:06 2008 -0400
+++ b/amat.py	Wed Oct 22 15:56:53 2008 -0400
@@ -2,18 +2,15 @@
 
 import sys, numpy, array
 
-path_MNIST = '/u/bergstrj/pub/data/mnist.amat'
-
-
 class AMat:
     """DataSource to access a plearn amat file as a periodic unrandomized stream.
 
     Attributes:
 
-    input -- minibatch of input
-    target -- minibatch of target
-    weight -- minibatch of weight
-    extra -- minitbatch of extra
+    input -- all columns of input
+    target -- all columns of target
+    weight -- all columns of weight
+    extra -- all columns of extra
 
     all -- the entire data contents of the amat file
     n_examples -- the number of training examples in the file
diff -r 4335309f4924 -r bd937e845bbb datasets/MNIST.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/datasets/MNIST.py	Wed Oct 22 15:56:53 2008 -0400
@@ -0,0 +1,31 @@
+"""
+Various routines to load/access MNIST data.
+"""
+from __future__ import absolute_import
+
+import numpy
+
+from ..amat import AMat
+
+default_path = '/u/bergstrj/pub/data/mnist.amat'
+"""the location of a file containing mnist data in .amat format"""
+
+
+def head(n=10, path=None):
+    """Load the first MNIST examples.
+
+    Returns two matrices: x, y.  x has N rows of 784 columns.  Each row of x represents the
+    28x28 grey-scale pixels in raster order.  y is a vector of N integers.  Each element y[i]
+    is the label of the i'th row of x.
+    
+    """
+    path = path if path is not None else default_path
+
+    dat = AMat(path=path, head=n)
+
+    return dat.input, numpy.asarray(dat.target, dtype='int64').reshape(dat.target.shape[0])
+
+def all(path=None):
+    return head(n=None, path=path)
+
+
diff -r 4335309f4924 -r bd937e845bbb datasets/__init__.py