--- a/pylearn/examples/daa.py	Thu Aug 19 12:01:30 2010 -0400
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,206 +0,0 @@
-import math, sys, time, copy, cPickle, shutil, functools
-import numpy 
-
-import theano
-from theano import tensor
-from theano.compile import module
-from theano.sandbox.softsign import softsign
-from theano import tensor as T, sparse as S
-from theano.tensor import nnet, as_tensor
-from theano.tensor.randomstreams import RandomStreams
-from theano.printing import Print
-from theano.compile.mode import Mode
-
-from ..algorithms import cost, minimizer, logistic_regression, sgd, stopper
-from ..io import filetensor
-from ..datasets import MNIST
-
-
-class AbstractFunction(Exception): """Override me"""
-
-#
-# default corruption function for BasicDAA
-# to extend
-#
-def corrupt_random_zeros(x, rng, p_zero):
-    mask = rng.binomial(T.shape(x), 1, 1.0 - p_zero)
-    return mask * x
-
-#
-# BasicDAA
-# ========
-#
-# Re-usable module, could be in pylearn.algorithms.daa
-#
-class BasicDAA(theano.Module):
-    def __init__(self, n_visible, n_hidden, seed,
-            w=None, 
-            vbias=None, 
-            hbias=None,
-            sigmoid=nnet.sigmoid,
-            corrupt=functools.partial(corrupt_random_zeros, p_zero=0.1),
-            reconstruction_cost=cost.cross_entropy,
-            w_scale = None
-            ):
-        """
-        :param w_scale: should be a floating-point value or None.  The weights are initialized
-        to a random value on the interval (-1,1) and scaled by this value.
-        None means the default of 1/sqrt(max(n_visible, n_hidden))
-
-
-        """
-        super(BasicDAA, self).__init__()
-
-        self.n_visible = n_visible
-        self.n_hidden = n_hidden
-        self.random = RandomStreams()
-        self.seed = seed
-        self.w_scale = w_scale if w_scale is not None else 1.0 / math.sqrt(max(n_visible,n_hidden))
-        self.sigmoid = sigmoid
-        self.corrupt = corrupt
-        self.reconstruction_cost = reconstruction_cost
-
-        self.w = tensor.dmatrix() if w is None else w
-        self.vbias = tensor.dvector() if vbias is None else vbias
-        self.hbias = tensor.dvector() if hbias is None else hbias
-
-        self.params = [self.w, self.vbias, self.hbias]
-
-    def _instance_initialize(self, obj):
-        #consider offering override parameters for seed, n_visible, n_hidden, w_scale
-        super(BasicDAA, self)._instance_initialize(obj)
-        rng = numpy.random.RandomState(self.seed)
-        s = rng.randint(2**30)
-        obj.w = (rng.rand(self.n_visible, self.n_hidden) * 2.0 - 1.0) * self.w_scale
-        obj.vbias = numpy.zeros(self.n_visible)
-        obj.hbias = numpy.zeros(self.n_hidden)
-        obj.random.initialize(int(s))
-
-    def hidden_act(self, visible):
-        return theano.dot(visible, self.w) + self.hbias
-
-    def hidden(self, visible):
-        return self.sigmoid(self.hidden_act(visible))
-
-    def visible_act(self, hidden):
-        return theano.dot(hidden, self.w.T) + self.vbias
-
-    def visible(self, hidden):
-        return self.sigmoid(self.visible_act(hidden))
-
-    def reconstruction(self, visible):
-        return self.visible(self.hidden(self.corrupt(visible, self.random)))
-
-    def daa_cost(self, visible):
-        return self.reconstruction_cost(visible, self.reconstruction(visible))
-
-    def l2_cost(self):
-        return self.w.norm(2)
-
-
-
-#
-# StackedDAA
-# ==========
-#
-# Hacky experiment type code, which would *not* be in pylearn.
-#
-# This Module can be extended / parametrized so many ways that I think this code is best cut &
-# pasted.
-#
-class StackedDAA(theano.Module):
-    def __init__(self, layer_widths, seed, finetune_lr=1e-3, pretrain_lr=1e-4):
-        super(StackedDAA, self).__init__()
-        input = theano.tensor.dmatrix()
-
-        #the parameters of this function, required for the minimizer
-        self.params = [] 
-        daa_widths = layer_widths[:-1]
-
-        #create the stack of DAA modules, and the self.params list
-        self.daa_list = []
-        daa_input = input
-        for i in xrange(len(daa_widths)-1):
-            self.daa_list.append(BasicDAA(daa_widths[i], daa_widths[i+1], seed+i))
-            daa_input = self.daa_list[-1].hidden(daa_input)
-            self.params.extend(self.daa_list[-1].params)
-
-        #put a logistic regression module on top for classification
-        self.classif = logistic_regression.LogRegN(input=daa_input,
-                n_in=layer_widths[-2], n_out = layer_widths[-1])
-        self.params.extend(self.classif.params)
-
-        #set up the fine-tuning function (minimizer.step)
-        FineTuneMinimizer = sgd.sgd_minimizer(stepsize=finetune_lr)
-        self.finetune = FineTuneMinimizer([input, self.classif.target], self.classif.unregularized_cost, self.params)
-
-        #set up the pre-training function
-        pretrain_cost, pretrain_input, pretrain_params = reduce(
-                lambda (c,i,p), daa: (c + daa.daa_cost(i), daa.hidden(i), p + daa.params),
-                self.daa_list,
-                (0.0, input, []))
-        PreTrainMinimizer = sgd.sgd_minimizer(stepsize=pretrain_lr)
-        self.pretrain = PreTrainMinimizer([input], pretrain_cost, pretrain_params)
-        
-    def _instance_initialize(self, obj):
-        #consider offering override parameters for seed, n_visible, n_hidden, w_scale
-        super(StackedDAA, self)._instance_initialize(obj)
-
-        #ugh... why do i need to do this???
-        for daa in obj.daa_list:
-            daa.initialize()
-        obj.classif.initialize()
-        obj.finetune.initialize()
-        obj.pretrain.initialize()
-
-#
-# DRIVER
-# ======
-#
-# This learning algorithm is the sort of thing that we've put in 'experiment' functions, that
-# can be run using dbdict.  
-#
-def demo_random(layer_widths=[3,4,5]):
-    sdaa = StackedDAA(layer_widths, seed=666).make(mode='FAST_COMPILE')
-
-    # create some training data
-    rng = numpy.random.RandomState(7832)
-    input_data = rng.randn(10,3)
-    targ_data = rng.binomial(1,.5, size=10)
-
-    print 'Pre-training ...'
-    for i in xrange(5):
-        print sdaa.pretrain.step_cost(input_data)
-
-    print 'Fine-tuning ...'
-    for i in xrange(5):
-        print sdaa.finetune.step_cost(input_data, targ_data)
-
-
-def demo_mnist(layer_widths=[784,500,500]):
-    sdaa = StackedDAA(layer_widths, seed=666).make()
-
-    mnist = MNIST.full()
-    batchsize=10
-    n_pretrain_batches=10000
-    n_finetune_batches=10000
-
-    t0 = time.time()
-    print 'Pre-training ...'
-    for i in xrange(n_pretrain_batches):
-        ii = (i*batchsize) % len(mnist.train.x)
-        x = mnist.train.x[ii:ii+batchsize]
-        c = sdaa.pretrain.step_cost(x)
-        if not i % 100:
-            print i, n_pretrain_batches, time.time() - t0, c
-
-    t1 = time.time()
-    print 'Fine-tuning ...'
-    for i in xrange(n_finetune_batches):
-        ii = (i*batchsize) % len(mnist.train.x)
-        x = mnist.train.x[ii:ii+batchsize]
-        y = mnist.train.y[ii:ii+batchsize]
-        c = sdaa.finetune.step_cost(x, y)
-        if not i % 100:
-            print i, n_finetune_batches, time.time() - t1, c
-

--- a/pylearn/examples/linear_classifier.py	Thu Aug 19 12:01:30 2010 -0400
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,224 +0,0 @@
-#! /usr/bin/env python
-"""
-T. Bertin-Mahieux (2008) University of Montreal
-bertinmt@iro.umontreal.ca
-
-linear_classifier.py
-Simple script that creates a linear_classifier, and
-learns the parameters using backpropagation.
-
-This is to illustrate how to use theano/pylearn.
-Anyone who knows how to make this script simpler/clearer is welcome to
-make the modifications.
-"""
-
-
-import os
-import sys
-import time
-import copy
-import pickle
-import numpy
-import numpy as N
-import numpy.random as NR
-from pylearn import cost
-import theano
-from theano import tensor as T
-
-
-def cost_function(*args,**kwargs) :
-    """ default cost function, quadratic """
-    return cost.quadratic(*args,**kwargs)
-
-
-class modelgraph() :
-    """ class that contains the graph of the model """
-    lr = T.scalar()                              # learning rate
-    inputs = T.matrix()                          # inputs (one example per line)
-    true_outputs = T.matrix()                    # outputs (one example per line)
-    W = T.matrix()                               # weights input * W + b= output
-    b = T.vector()                               # bias
-    outputs = T.dot(inputs,W) + b                # output, one per line
-    costs = cost_function(true_outputs,outputs)  # costs
-    g_W = T.grad(costs,W)                        # gradient of W
-    g_b = T.grad(costs,b)                        # gradient of b
-    new_W = T.sub_inplace(W, lr * g_W)           # update inplace of W
-    new_b = T.sub_inplace(b, lr * g_b)           # update inplace of b
-
-
-class model() :
-    """ 
-    The model! 
-    Contains needed matrices, needed functions, and a link to the model graph.
-    """
-
-    def __init__(self,input_size,output_size) :
-        """ init matrix and bias, creates the graph, create a dict of compiled functions """
-        # graph
-        self.graph = modelgraph()
-        # weights and bias, saved in self.params
-        seed = 666
-        r = NR.RandomState(seed)
-        W = r.uniform(size = [input_size, output_size], low = -1/N.sqrt(input_size), high = 1/N.sqrt(input_size))
-        b = numpy.zeros((output_size, ))
-        self.params = [W,b]
-        # dictionary of compiled functions
-        self.func_dict = dict()
-        # keep some init_infos (may not be necessary)
-        self.init_params = [input_size,output_size]
-
-
-    def update(self,lr,true_inputs,true_outputs) :
-        """ does an update of the model, one gradient descent """
-        # do we already have the proper theano function?
-        if self.func_dict.has_key('update_func') :
-            self.func_dict['update_func'](lr,true_inputs,true_outputs,self.params[0],self.params[1])
-            return
-        else :
-            # create the theano function, tell him what are the inputs and outputs)
-            func = theano.function([self.graph.lr,self.graph.inputs,self.graph.true_outputs,
-                                self.graph.W, self.graph.b],
-                               [self.graph.new_W,self.graph.new_b])
-            # add function to dictionary, so we don't compile it again
-            self.func_dict['update_func'] = func
-            # use this function
-            func(lr,true_inputs,true_outputs,self.params[0],self.params[1])
-            return
-    
-    def costs(self,true_inputs,true_outputs) :
-        """ get the costs for given examples, don't update """
-        # do we already have the proper theano function?
-        if self.func_dict.has_key('costs_func') :
-            return self.func_dict['costs_func'](true_inputs,true_outputs,self.params[0],self.params[1])
-        else :
-            # create the theano function, tell him what are the inputs and outputs)
-            func = theano.function([self.graph.inputs,self.graph.true_outputs,self.graph.W,self.graph.b],
-                               [self.graph.costs])
-            # add function to dictionary, se we don't compile it again
-            self.func_dict['costs_func'] = func
-            # use this function
-            return func(true_inputs,true_outputs,self.params[0],self.params[1])
-
-    def outputs(self,true_inputs) :
-        """ get the output for a set of examples (could be called 'predict') """
-        # do we already have the proper theano function?
-        if self.func_dict.has_key('outputs_func') :
-            return self.func_dict['outputs_func'](true_inputs,self.params[0],self.params[1])
-        else :
-            # create the theano function, tell him what are the inputs and outputs)
-            func = theano.function([self.graph.inputs, self.graph.W, self.graph.b],
-                               [self.graph.outputs])
-            # add function to dictionary, se we don't compile it again
-            self.func_dict['outputs_func'] = func
-            # use this function
-            return func(true_inputs,self.params[0],self.params[1])
-
-    def __getitem__(self,inputs) :
-        """ for simplicity, we can use the model this way: predictions = model[inputs] """
-        return self.outputs(inputs)
-
-    def __getstate__(self) :
-        """
-        To save/copy the model, used by pickle.dump() and by copy.deepcopy().
-        @return a dictionnary with the params (matrix + bias)
-        """
-        d = dict()
-        d['params'] = self.params
-        d['init_params'] = self.init_params
-        return d
-        
-    def __setstate__(self,d) :
-        """
-        Get the dictionary created by __getstate__(), use it to recreate the model.
-        """
-        self.params = d['params']
-        self.init_params = d['init_params']
-        self.graph = modelgraph() # we did not save the model graph 
-
-    def __str__(self) :
-        """ returns a string representing the model """
-        res = "Linear regressor, input size =",str(self.init_params[0])
-        res += ", output size =", str(self.init_params[1])
-        return res
-
-    def __equal__(self,other) :
-        """ 
-        Compares the model based on the params.
-        @return True if the params are the same, False otherwise
-        """
-        # class
-        if not isinstance(other,model) :
-            return False
-        # input size
-        if self.params[0].shape[0] != other.params[0].shape[0] :
-            return False
-        # output size
-        if self.params[0].shape[1] != other.params[0].shape[1] :
-            return False
-        # actual values
-        if not (self.params[0] == other.params[0]).all():
-            return False
-        if not (self.params[1] == other.params[1]).all():
-            return False
-        # all good
-        return True
-
-
-def die_with_usage() :
-    """ help menu """
-    print 'simple script to illustrate how to use theano/pylearn'
-    print 'to launch:'
-    print '  python linear_classifier.py -launch'
-    sys.exit(0)
-
-
-
-#************************************************************
-# main
-
-if __name__ == '__main__' :
-
-    if len(sys.argv) < 2 :
-        die_with_usage()
-
-    # print create data
-    inputs = numpy.array([[.1,.2],
-                          [.2,.8],
-                          [.9,.3],
-                          [.6,.5]])
-    outputs = numpy.array([[0],
-                           [0],
-                           [1],
-                           [1]])
-    assert inputs.shape[0] == outputs.shape[0]
-
-    # create model
-    m = model(2,1)
-    
-    # predict
-    print 'prediction before training:'
-    print m[inputs]
-
-    # update it for 100 iterations
-    for k in range(50) :
-        m.update(.1,inputs,outputs)
-
-     # predict
-    print 'prediction after training:'
-    print m[inputs]
-
-    # show points
-    import pylab as P
-    colors = outputs.flatten().tolist()
-    x = inputs[:,0]
-    y = inputs[:,1]
-    P.plot(x[numpy.where(outputs==0)[0]],y[numpy.where(outputs==0)[0]],'r+')
-    P.plot(x[numpy.where(outputs==1)[0]],y[numpy.where(outputs==1)[0]],'b+')
-    # decision line
-    p1 = (.5 - m.params[1] * 1.) / m.params[0][1,0] # abs = 0
-    p2 = (.5 - m.params[1] * 1.) / m.params[0][0,0] # ord = 0
-    P.plot((0,p2[0],2*p2[0]),(p1[0],0,-p1[0]),'g-')
-    # show
-    P.axis([-1,2,-1,2])
-    P.show()
-    

--- a/pylearn/examples/theano_update.py	Thu Aug 19 12:01:30 2010 -0400
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,56 +0,0 @@
-import theano
-from theano import tensor
-
-import numpy
-
-# Two scalar symbolic variables
-a = tensor.scalar()
-b = tensor.scalar()
-
-# Definition of output symbolic variable
-c = a * b
-# Definition of the function computing it
-fprop = theano.function([a,b], [c])
-
-# Initialize numerical variables
-a_val = numpy.array(12.)
-b_val = numpy.array(2.)
-print 'a_val =', a_val
-print 'b_val =', b_val
-
-# Numerical value of output is returned by the call to "fprop"
-c_val = fprop(a_val, b_val)
-print 'c_val =', c_val
-
-
-# Definition of simple update (increment by one)
-new_b = b + 1
-update = theano.function([b], [new_b])
-
-# New numerical value of b is returned by the call to "update"
-b_val = update(b_val)
-print 'new b_val =', b_val
-# We can use the new value in "fprop"
-c_val = fprop(a_val, b_val)
-print 'c_val =', c_val
-
-
-# Definition of in-place update (increment by one)
-re_new_b = tensor.add_inplace(b, 1.)
-re_update = theano.function([b], [re_new_b])
-
-# "re_update" can be used the same way as "update"
-b_val = re_update(b_val)
-print 'new b_val =', b_val
-# We can use the new value in "fprop"
-c_val = fprop(a_val, b_val)
-print 'c_val =', c_val
-
-# It is not necessary to keep the return value when the update is done in place
-re_update(b_val)
-print 'new b_val =', b_val
-c_val = fprop(a_val, b_val)
-print 'c_val =', c_val
-
-
-