Mercurial > pylearn

annotate mlp.py @ 186:562f308873f0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
added ManualNNet
author James Bergstra <bergstrj@iro.umontreal.ca>
date Tue, 13 May 2008 20:10:03 -0400
parents 25d0a0c713da
children ebbb0e749565
rev   line source
132
f6505ec32dc3 Updated documentation slightly
Joseph Turian <turian@gmail.com>
parents: 129
diff changeset
1 """
f6505ec32dc3 Updated documentation slightly
Joseph Turian <turian@gmail.com>
parents: 129
diff changeset
2 A straightforward classicial feedforward
f6505ec32dc3 Updated documentation slightly
Joseph Turian <turian@gmail.com>
parents: 129
diff changeset
3 one-hidden-layer neural net, with L2 regularization.
f6505ec32dc3 Updated documentation slightly
Joseph Turian <turian@gmail.com>
parents: 129
diff changeset
4 This is one of the simplest example of L{Learner}, and illustrates
f6505ec32dc3 Updated documentation slightly
Joseph Turian <turian@gmail.com>
parents: 129
diff changeset
5 the use of theano.
f6505ec32dc3 Updated documentation slightly
Joseph Turian <turian@gmail.com>
parents: 129
diff changeset
6 """
111
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
7
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
8 from learner import *
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
9 from theano import tensor as t
118
d0a1bd0378c6 Finished draft of OneHiddenLayerNNetClassifier to debut learner.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 111
diff changeset
10 from nnet_ops import *
133
b4657441dd65 Corrected typos
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 132
diff changeset
11 import math
175
e9a95e19e6f8 Added a Print Op
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 155
diff changeset
12 from misc import *
111
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
13
186
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
14 def function(inputs, outputs, linker='c&py'):
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
15 return theano.function(inputs, outputs, unpack_single=False,linker=linker)
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
16
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
17 def randshape(*shape): return (numpy.random.rand(*shape) -0.5) * 0.001
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
18
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
19 class ManualNNet(object):
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
20 def __init__(self, ninputs, nhid, nclass, lr, nepochs,
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
21 linker='c&yp',
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
22 hidden_layer=None):
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
23 class Vars:
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
24 def __init__(self, lr):
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
25 lr = t.constant(lr)
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
26 input = t.matrix('input') # n_examples x n_inputs
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
27 target = t.ivector('target') # n_examples x 1
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
28 W2 = t.matrix('W2')
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
29 b2 = t.vector('b2')
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
30
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
31 if hidden_layer:
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
32 hidden, hidden_params, hidden_ivals = hidden_layer(input)
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
33 else:
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
34 W1 = t.matrix('W1')
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
35 b1 = t.vector('b1')
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
36 hidden = t.tanh(b1 + t.dot(input, W1))
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
37 hidden_params = [W1, b1]
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
38 hidden_ivals = [randshape(ninputs, nhid), randshape(nhid)]
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
39
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
40 params = [W2, b2] + hidden_params
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
41 ivals = [randshape(nhid, nclass), randshape(nclass)]\
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
42 + hidden_ivals
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
43 nll, predictions = crossentropy_softmax_1hot( b2 + t.dot(hidden, W2), target)
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
44 output_class = t.argmax(predictions,1)
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
45 loss_01 = t.neq(output_class, target)
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
46 g_params = t.grad(nll, params)
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
47 new_params = [t.sub_inplace(p, lr * gp) for p,gp in zip(params, g_params)]
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
48 self.__dict__.update(locals()); del self.self
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
49 self.nhid = nhid
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
50 self.nclass = nclass
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
51 self.nepochs = nepochs
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
52 self.v = Vars(lr)
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
53 self.params = None
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
54
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
55 def update(self, trainset):
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
56 params = self.v.ivals
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
57 update_fn = function(
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
58 [self.v.input, self.v.target] + self.v.params,
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
59 [self.v.nll] + self.v.new_params)
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
60 for i in xrange(self.nepochs):
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
61 for input, target in trainset.minibatches(['input', 'target'],
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
62 minibatch_size=min(32, len(trainset))):
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
63 dummy = update_fn(input, target[:,0], *params)
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
64 if 0: print dummy[0] #the nll
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
65 return self.use
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
66 __call__ = update
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
67
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
68 def use(self, dset,
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
69 output_fieldnames=['output_class'],
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
70 test_stats_collector=None,
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
71 copy_inputs=False,
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
72 put_stats_in_output_dataset=True,
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
73 output_attributes=[]):
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
74 inputs = [self.v.input, self.v.target] + self.v.params
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
75 fn = function(inputs, [getattr(self.v, name) for name in output_fieldnames])
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
76 target = dset.fields()['target'] if ('target' in dset.fields()) else numpy.zeros((1,1),dtype='int64')
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
77 return ApplyFunctionDataSet(dset,
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
78 lambda input, target: fn(input, target[:,0], *self.v.ivals),
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
79 output_fieldnames)
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
80
562f308873f0 added ManualNNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 183
diff changeset
81
129
4c2280edcaf5 Fixed typos in learner.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 126
diff changeset
82 class OneHiddenLayerNNetClassifier(OnlineGradientTLearner):
111
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
83 """
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
84 Implement a straightforward classicial feedforward
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
85 one-hidden-layer neural net, with L2 regularization.
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
86
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
87 The predictor parameters are obtained by minibatch/online gradient descent.
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
88 Training can proceed sequentially (with multiple calls to update with
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
89 different disjoint subsets of the training sets).
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
90
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
91 Hyper-parameters:
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
92 - L2_regularizer
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
93 - learning_rate
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
94 - n_hidden
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
95
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
96 For each (input_t,output_t) pair in a minibatch,::
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
97
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
98 output_activations_t = b2+W2*tanh(b1+W1*input_t)
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
99 output_t = softmax(output_activations_t)
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
100 output_class_t = argmax(output_activations_t)
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
101 class_error_t = 1_{output_class_t != target_t}
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
102 nll_t = -log(output_t[target_t])
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
103
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
104 and the training criterion is::
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
105
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
106 loss = L2_regularizer*(||W1||^2 + ||W2||^2) + sum_t nll_t
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
107
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
108 The parameters are [b1,W1,b2,W2] and are obtained by minimizing the loss by
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
109 stochastic minibatch gradient descent::
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
110
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
111 parameters[i] -= learning_rate * dloss/dparameters[i]
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
112
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
113 The fields and attributes expected and produced by use and update are the following:
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
114
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
115 - Input and output fields (example-wise quantities):
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
116
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
117 - 'input' (always expected by use and update)
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
118 - 'target' (optionally expected by use and always by update)
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
119 - 'output' (optionally produced by use)
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
120 - 'output_class' (optionally produced by use)
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
121 - 'class_error' (optionally produced by use)
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
122 - 'nll' (optionally produced by use)
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
123
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
124 - optional attributes (optionally expected as input_dataset attributes)
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
125 (warning, this may be dangerous, the 'use' method will use those provided in the
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
126 input_dataset rather than those learned during 'update'; currently no support
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
127 for providing these to update):
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
128
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
129 - 'L2_regularizer'
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
130 - 'b1'
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
131 - 'W1'
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
132 - 'b2'
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
133 - 'W2'
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
134 - 'parameters' = [b1, W1, b2, W2]
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
135 - 'regularization_term'
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
136
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
137 """
183
25d0a0c713da did some debugging of test_mlp
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 182
diff changeset
138 def __init__(self,n_hidden,n_classes,learning_rate,max_n_epochs,L2_regularizer=0,init_range=1.,n_inputs=None,minibatch_size=None,linker='c|py'):
133
b4657441dd65 Corrected typos
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 132
diff changeset
139 self._n_inputs = n_inputs
121
2ca8dccba270 debugging mlp.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 118
diff changeset
140 self._n_outputs = n_classes
2ca8dccba270 debugging mlp.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 118
diff changeset
141 self._n_hidden = n_hidden
2ca8dccba270 debugging mlp.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 118
diff changeset
142 self._init_range = init_range
133
b4657441dd65 Corrected typos
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 132
diff changeset
143 self._max_n_epochs = max_n_epochs
b4657441dd65 Corrected typos
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 132
diff changeset
144 self._minibatch_size = minibatch_size
121
2ca8dccba270 debugging mlp.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 118
diff changeset
145 self.learning_rate = learning_rate # this is the float
134
3f4e5c9bdc5e Fixes to ApplyFunctionDataSet and other things to make learner and mlp work
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 133
diff changeset
146 self.L2_regularizer = L2_regularizer
121
2ca8dccba270 debugging mlp.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 118
diff changeset
147 self._learning_rate = t.scalar('learning_rate') # this is the symbol
2ca8dccba270 debugging mlp.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 118
diff changeset
148 self._input = t.matrix('input') # n_examples x n_inputs
183
25d0a0c713da did some debugging of test_mlp
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 182
diff changeset
149 self._target = t.lmatrix('target') # n_examples x 1
134
3f4e5c9bdc5e Fixes to ApplyFunctionDataSet and other things to make learner and mlp work
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 133
diff changeset
150 self._target_vector = self._target[:,0]
121
2ca8dccba270 debugging mlp.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 118
diff changeset
151 self._L2_regularizer = t.scalar('L2_regularizer')
2ca8dccba270 debugging mlp.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 118
diff changeset
152 self._W1 = t.matrix('W1')
2ca8dccba270 debugging mlp.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 118
diff changeset
153 self._W2 = t.matrix('W2')
2ca8dccba270 debugging mlp.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 118
diff changeset
154 self._b1 = t.row('b1')
2ca8dccba270 debugging mlp.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 118
diff changeset
155 self._b2 = t.row('b2')
126
4efe6d36c061 minor edits
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 121
diff changeset
156 self._regularization_term = self._L2_regularizer * (t.sum(self._W1*self._W1) + t.sum(self._W2*self._W2))
121
2ca8dccba270 debugging mlp.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 118
diff changeset
157 self._output_activations =self._b2+t.dot(t.tanh(self._b1+t.dot(self._input,self._W1.T)),self._W2.T)
180
2698c0feeb54 mlp seems to work!
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 179
diff changeset
158 self._nll,self._output = crossentropy_softmax_1hot(self._output_activations,self._target_vector)
155
ae5651a3696b new argmax calling convention
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 134
diff changeset
159 self._output_class = t.argmax(self._output,1)
134
3f4e5c9bdc5e Fixes to ApplyFunctionDataSet and other things to make learner and mlp work
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 133
diff changeset
160 self._class_error = t.neq(self._output_class,self._target_vector)
121
2ca8dccba270 debugging mlp.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 118
diff changeset
161 self._minibatch_criterion = self._nll + self._regularization_term / t.shape(self._input)[0]
183
25d0a0c713da did some debugging of test_mlp
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 182
diff changeset
162 OnlineGradientTLearner.__init__(self, linker = linker)
121
2ca8dccba270 debugging mlp.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 118
diff changeset
163
111
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
164 def attributeNames(self):
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
165 return ["parameters","b1","W2","b2","W2", "L2_regularizer","regularization_term"]
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
166
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
167 def parameterAttributes(self):
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
168 return ["b1","W1", "b2", "W2"]
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
169
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
170 def updateMinibatchInputFields(self):
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
171 return ["input","target"]
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
172
180
2698c0feeb54 mlp seems to work!
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 179
diff changeset
173 def updateMinibatchInputAttributes(self):
2698c0feeb54 mlp seems to work!
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 179
diff changeset
174 return OnlineGradientTLearner.updateMinibatchInputAttributes(self)+["L2_regularizer"]
2698c0feeb54 mlp seems to work!
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 179
diff changeset
175
111
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
176 def updateEndOutputAttributes(self):
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
177 return ["regularization_term"]
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
178
118
d0a1bd0378c6 Finished draft of OneHiddenLayerNNetClassifier to debut learner.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 111
diff changeset
179 def lossAttribute(self):
d0a1bd0378c6 Finished draft of OneHiddenLayerNNetClassifier to debut learner.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 111
diff changeset
180 return "minibatch_criterion"
d0a1bd0378c6 Finished draft of OneHiddenLayerNNetClassifier to debut learner.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 111
diff changeset
181
111
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
182 def defaultOutputFields(self, input_fields):
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
183 output_fields = ["output", "output_class",]
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
184 if "target" in input_fields:
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
185 output_fields += ["class_error", "nll"]
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
186 return output_fields
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
187
182
4afb41e61fcf strange bug in linker obtained by 'python test_mlp.py'
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 180
diff changeset
188 def updateMinibatch(self,minibatch):
4afb41e61fcf strange bug in linker obtained by 'python test_mlp.py'
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 180
diff changeset
189 MinibatchUpdatesTLearner.updateMinibatch(self,minibatch)
183
25d0a0c713da did some debugging of test_mlp
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 182
diff changeset
190 #print self.nll
182
4afb41e61fcf strange bug in linker obtained by 'python test_mlp.py'
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 180
diff changeset
191
111
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
192 def allocate(self,minibatch):
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
193 minibatch_n_inputs = minibatch["input"].shape[1]
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
194 if not self._n_inputs:
118
d0a1bd0378c6 Finished draft of OneHiddenLayerNNetClassifier to debut learner.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 111
diff changeset
195 self._n_inputs = minibatch_n_inputs
134
3f4e5c9bdc5e Fixes to ApplyFunctionDataSet and other things to make learner and mlp work
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 133
diff changeset
196 self.b1 = numpy.zeros((1,self._n_hidden))
3f4e5c9bdc5e Fixes to ApplyFunctionDataSet and other things to make learner and mlp work
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 133
diff changeset
197 self.b2 = numpy.zeros((1,self._n_outputs))
111
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
198 self.forget()
118
d0a1bd0378c6 Finished draft of OneHiddenLayerNNetClassifier to debut learner.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 111
diff changeset
199 elif self._n_inputs!=minibatch_n_inputs:
d0a1bd0378c6 Finished draft of OneHiddenLayerNNetClassifier to debut learner.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 111
diff changeset
200 # if the input changes dimension on the fly, we resize and forget everything
111
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
201 self.forget()
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
202
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
203 def forget(self):
118
d0a1bd0378c6 Finished draft of OneHiddenLayerNNetClassifier to debut learner.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 111
diff changeset
204 if self._n_inputs:
d0a1bd0378c6 Finished draft of OneHiddenLayerNNetClassifier to debut learner.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 111
diff changeset
205 r = self._init_range/math.sqrt(self._n_inputs)
d0a1bd0378c6 Finished draft of OneHiddenLayerNNetClassifier to debut learner.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 111
diff changeset
206 self.W1 = numpy.random.uniform(low=-r,high=r,
d0a1bd0378c6 Finished draft of OneHiddenLayerNNetClassifier to debut learner.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 111
diff changeset
207 size=(self._n_hidden,self._n_inputs))
d0a1bd0378c6 Finished draft of OneHiddenLayerNNetClassifier to debut learner.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 111
diff changeset
208 r = self._init_range/math.sqrt(self._n_hidden)
d0a1bd0378c6 Finished draft of OneHiddenLayerNNetClassifier to debut learner.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 111
diff changeset
209 self.W2 = numpy.random.uniform(low=-r,high=r,
d0a1bd0378c6 Finished draft of OneHiddenLayerNNetClassifier to debut learner.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 111
diff changeset
210 size=(self._n_outputs,self._n_hidden))
d0a1bd0378c6 Finished draft of OneHiddenLayerNNetClassifier to debut learner.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 111
diff changeset
211 self.b1[:]=0
d0a1bd0378c6 Finished draft of OneHiddenLayerNNetClassifier to debut learner.py
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 111
diff changeset
212 self.b2[:]=0
133
b4657441dd65 Corrected typos
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 132
diff changeset
213 self._n_epochs=0
111
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
214
133
b4657441dd65 Corrected typos
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 132
diff changeset
215 def isLastEpoch(self):
b4657441dd65 Corrected typos
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 132
diff changeset
216 self._n_epochs +=1
b4657441dd65 Corrected typos
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 132
diff changeset
217 return self._n_epochs>=self._max_n_epochs
111
88257dfedf8c Added another work in progress, for mlp's
bengioy@bengiomac.local
parents:
diff changeset
218
180
2698c0feeb54 mlp seems to work!
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 179
diff changeset
219 def debug_updateMinibatch(self,minibatch):
178
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 155
diff changeset
220 # make sure all required fields are allocated and initialized
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 155
diff changeset
221 self.allocate(minibatch)
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 155
diff changeset
222 input_attributes = self.names2attributes(self.updateMinibatchInputAttributes())
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 155
diff changeset
223 input_fields = minibatch(*self.updateMinibatchInputFields())
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 155
diff changeset
224 print 'input attributes', input_attributes
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 155
diff changeset
225 print 'input fields', input_fields
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 155
diff changeset
226 results = self.update_minibatch_function(*(input_attributes+input_fields))
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 155
diff changeset
227 print 'output attributes', self.updateMinibatchOutputAttributes()
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 155
diff changeset
228 print 'results', results
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 155
diff changeset
229 self.setAttributes(self.updateMinibatchOutputAttributes(),
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 155
diff changeset
230 results)
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 155
diff changeset
231
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 155
diff changeset
232 if 0:
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 155
diff changeset
233 print 'n0', self.names2OpResults(self.updateMinibatchOutputAttributes()+ self.updateMinibatchInputFields())
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 155
diff changeset
234 print 'n1', self.names2OpResults(self.updateMinibatchOutputAttributes())
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 155
diff changeset
235 print 'n2', self.names2OpResults(self.updateEndInputAttributes())
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 155
diff changeset
236 print 'n3', self.names2OpResults(self.updateEndOutputAttributes())
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 155
diff changeset
237