Mercurial > pylearn

changeset 470:bd937e845bbb

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
new stuff: algorithms/logistic_regression, datasets/MNIST
author James Bergstra <bergstrj@iro.umontreal.ca>
date Wed, 22 Oct 2008 15:56:53 -0400
parents 4335309f4924
children 45b3eb429c15
files algorithms/__init__.py algorithms/_test_logistic_regression.py algorithms/logistic_regression.py amat.py datasets/MNIST.py datasets/__init__.py
diffstat 4 files changed, 190 insertions(+), 7 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/algorithms/_test_logistic_regression.py	Wed Oct 22 15:56:53 2008 -0400
@@ -0,0 +1,60 @@
+from logistic_regression import *
+import sys, time
+
+if __name__ == '__main__':
+    pprint.assign(nnet_ops.crossentropy_softmax_1hot_with_bias_dx, printing.FunctionPrinter('xsoftmaxdx'))
+    pprint.assign(nnet_ops.crossentropy_softmax_argmax_1hot_with_bias, printing.FunctionPrinter('nll', 'softmax', 'argmax'))
+    if 1:
+        lrc = Module_Nclass()
+
+        print '================'
+        print lrc.update.pretty()
+        print '================'
+        print lrc.update.pretty(mode = theano.Mode('py', 'fast_run'))
+        print '================'
+#         print lrc.update.pretty(mode = compile.FAST_RUN.excluding('inplace'))
+#         print '================'
+
+#        sys.exit(0)
+
+        lr = lrc.make(10, 2, mode=theano.Mode('c|py', 'fast_run'))
+        #lr = lrc.make(10, 2, mode=compile.FAST_RUN.excluding('fast_run'))
+        #lr = lrc.make(10, 2, mode=theano.Mode('py', 'merge')) #'FAST_RUN')
+
+        data_x = N.random.randn(5, 10)
+        data_y = (N.random.randn(5) > 0)
+
+        t = time.time()
+        for i in xrange(10000):
+            lr.lr = 0.02
+            xe = lr.update(data_x, data_y) 
+            #if i % 100 == 0:
+            #    print i, xe
+
+        print 'training time:', time.time() - t
+        print 'final error', xe
+
+        #print
+        #print 'TRAINED MODEL:'
+        #print lr
+
+    if 0:
+        lrc = Module()
+
+        lr = lrc.make(10, mode=theano.Mode('c|py', 'merge')) #'FAST_RUN')
+
+        data_x = N.random.randn(5, 10)
+        data_y = (N.random.randn(5, 1) > 0)
+
+        for i in xrange(10000):
+            xe = lr.update(data_x, data_y)
+            if i % 100 == 0:
+                print i, xe
+
+        print
+        print 'TRAINED MODEL:'
+        print lr
+
+
+
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/algorithms/logistic_regression.py	Wed Oct 22 15:56:53 2008 -0400
@@ -0,0 +1,95 @@
+import theano
+from theano import tensor as T
+from theano.tensor import nnet_ops
+from theano.compile import module
+from theano import printing, pprint
+from theano import compile
+
+import numpy as N
+
+
+class Module_Nclass(module.FancyModule):
+    class __instance_type__(module.FancyModuleInstance):
+        def initialize(self, n_in, n_out, rng=N.random):
+            #self.component is the LogisticRegressionTemplate instance that built this guy.
+
+            self.w = rng.randn(n_in, n_out)
+            self.b = rng.randn(n_out)
+            self.lr = 0.01
+            self.__hide__ = ['params']
+
+    def __init__(self, x=None, targ=None, w=None, b=None, lr=None):
+        super(Module_Nclass, self).__init__() #boilerplate
+
+        self.x = x if x is not None else T.matrix()
+        self.targ = targ if targ is not None else T.lvector()
+
+        self.w = w if w is not None else module.Member(T.dmatrix())
+        self.b = b if b is not None else module.Member(T.dvector())
+        self.lr = lr if lr is not None else module.Member(T.dscalar())
+
+        self.params = [p for p in [self.w, self.b] if p.owner is None]
+
+        xent, y = nnet_ops.crossentropy_softmax_1hot(
+                T.dot(self.x, self.w) + self.b, self.targ)
+        sum_xent = T.sum(xent)
+
+        self.y = y
+        self.sum_xent = sum_xent
+
+        #define the apply method
+        self.pred = T.argmax(T.dot(self.x, self.w) + self.b, axis=1)
+        self.apply = module.Method([self.x], self.pred)
+
+        if self.params:
+            gparams = T.grad(sum_xent, self.params)
+
+            self.update = module.Method([self.x, self.targ], sum_xent,
+                    updates = dict((p, p - self.lr * g) for p, g in zip(self.params, gparams)))
+
+class Module(module.FancyModule):
+    class __instance_type__(module.FancyModuleInstance):
+        def initialize(self, n_in):
+            #self.component is the LogisticRegressionTemplate instance that built this guy.
+
+            self.w = N.random.randn(n_in,1)
+            self.b = N.random.randn(1)
+            self.lr = 0.01
+            self.__hide__ = ['params']
+
+    def __init__(self, x=None, targ=None, w=None, b=None, lr=None):
+        super(Module, self).__init__() #boilerplate
+
+        self.x = x if x is not None else T.matrix()
+        self.targ = targ if targ is not None else T.lcol()
+
+        self.w = w if w is not None else module.Member(T.dmatrix())
+        self.b = b if b is not None else module.Member(T.dvector())
+        self.lr = lr if lr is not None else module.Member(T.dscalar())
+
+        self.params = [p for p in [self.w, self.b] if p.owner is None]
+
+        y = nnet_ops.sigmoid(T.dot(self.x, self.w))
+        xent = -self.targ * T.log(y) - (1.0 - self.targ) * T.log(1.0 - y)
+        sum_xent = T.sum(xent)
+
+        self.y = y
+        self.xent = xent
+        self.sum_xent = sum_xent
+
+        #define the apply method
+        self.pred = (T.dot(self.x, self.w) + self.b) > 0.0
+        self.apply = module.Method([self.x], self.pred)
+
+        #if this module has any internal parameters, define an update function for them
+        if self.params:
+            gparams = T.grad(sum_xent, self.params)
+            self.update = module.Method([self.x, self.targ], sum_xent,
+                                        updates = dict((p, p - self.lr * g) for p, g in zip(self.params, gparams)))
+
+
+
+class Learner(object):
+    """TODO: Encapsulate the algorithm for finding an optimal regularization coefficient"""
+    pass
+
--- a/amat.py	Tue Oct 21 16:32:06 2008 -0400
+++ b/amat.py	Wed Oct 22 15:56:53 2008 -0400
@@ -2,18 +2,15 @@
 
 import sys, numpy, array
 
-path_MNIST = '/u/bergstrj/pub/data/mnist.amat'
-
-
 class AMat:
     """DataSource to access a plearn amat file as a periodic unrandomized stream.
 
     Attributes:
 
-    input -- minibatch of input
-    target -- minibatch of target
-    weight -- minibatch of weight
-    extra -- minitbatch of extra
+    input -- all columns of input
+    target -- all columns of target
+    weight -- all columns of weight
+    extra -- all columns of extra
 
     all -- the entire data contents of the amat file
     n_examples -- the number of training examples in the file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/datasets/MNIST.py	Wed Oct 22 15:56:53 2008 -0400
@@ -0,0 +1,31 @@
+"""
+Various routines to load/access MNIST data.
+"""
+from __future__ import absolute_import
+
+import numpy
+
+from ..amat import AMat
+
+default_path = '/u/bergstrj/pub/data/mnist.amat'
+"""the location of a file containing mnist data in .amat format"""
+
+
+def head(n=10, path=None):
+    """Load the first MNIST examples.
+
+    Returns two matrices: x, y.  x has N rows of 784 columns.  Each row of x represents the
+    28x28 grey-scale pixels in raster order.  y is a vector of N integers.  Each element y[i]
+    is the label of the i'th row of x.
+    
+    """
+    path = path if path is not None else default_path
+
+    dat = AMat(path=path, head=n)
+
+    return dat.input, numpy.asarray(dat.target, dtype='int64').reshape(dat.target.shape[0])
+
+def all(path=None):
+    return head(n=None, path=path)
+
+