Mercurial > pylearn

annotate mlp_factory_approach.py @ 189:8f58abb943d4

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
many changes to NeuralNet
author James Bergstra <bergstrj@iro.umontreal.ca>
date Wed, 14 May 2008 14:50:07 -0400
parents ebbb0e749565
children aa7a3ecbcc90
rev   line source
187
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
1 import dataset
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
2 import theano
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
3 import theano.tensor as t
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
4 import numpy
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
5 import nnet_ops
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
6
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
7 def _randshape(*shape):
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
8 return (numpy.random.rand(*shape) -0.5) * 0.001
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
9
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
10 class NeuralNet(object):
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
11
189
8f58abb943d4 many changes to NeuralNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 187
diff changeset
12 class _Model(object):
187
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
13 def __init__(self, nnet, params):
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
14 self.nnet = nnet
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
15 self.params = params
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
16
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
17 def update(self, trainset, stopper=None):
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
18 """Update this model from more training data."""
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
19 v = self.nnet.v
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
20 params = self.params
189
8f58abb943d4 many changes to NeuralNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 187
diff changeset
21 update_fn = self.nnet._fn([v.input, v.target] + v.params, [v.nll] + v.new_params)
187
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
22 if stopper is not None:
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
23 raise NotImplementedError()
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
24 else:
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
25 for i in xrange(100):
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
26 for input, target in trainset.minibatches(['input', 'target'],
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
27 minibatch_size=min(32, len(trainset))):
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
28 dummy = update_fn(input, target[:,0], *params)
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
29 if 0: print dummy[0] #the nll
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
30
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
31 def __call__(self, testset,
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
32 output_fieldnames=['output_class'],
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
33 test_stats_collector=None,
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
34 copy_inputs=False,
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
35 put_stats_in_output_dataset=True,
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
36 output_attributes=[]):
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
37 """Apply this model (as a function) to new data"""
189
8f58abb943d4 many changes to NeuralNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 187
diff changeset
38 v = self.nnet.v
8f58abb943d4 many changes to NeuralNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 187
diff changeset
39 outputs = [getattr(self.nnet.v, name) for name in output_fieldnames]
8f58abb943d4 many changes to NeuralNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 187
diff changeset
40 if 'target' in testset:
8f58abb943d4 many changes to NeuralNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 187
diff changeset
41 fn = self.nnet._fn([v.input, v.target] + v.params, outputs)
187
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
42 return dataset.ApplyFunctionDataSet(testset,
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
43 lambda input, target: fn(input, target[:,0], *self.params),
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
44 output_fieldnames)
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
45 else:
189
8f58abb943d4 many changes to NeuralNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 187
diff changeset
46 fn = self.nnet._fn([v.input] + v.params, outputs)
187
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
47 return dataset.ApplyFunctionDataSet(testset,
189
8f58abb943d4 many changes to NeuralNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 187
diff changeset
48 lambda input: fn(input, *self.params),
187
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
49 output_fieldnames)
189
8f58abb943d4 many changes to NeuralNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 187
diff changeset
50 def _fn(self, inputs, outputs):
8f58abb943d4 many changes to NeuralNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 187
diff changeset
51 #it is possible for this function to implement function caching
8f58abb943d4 many changes to NeuralNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 187
diff changeset
52 #... but not necessarily desirable.
8f58abb943d4 many changes to NeuralNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 187
diff changeset
53 #- caching ruins the possibility of multi-threaded learning
8f58abb943d4 many changes to NeuralNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 187
diff changeset
54 #- caching demands more efficiency in the face of resizing inputs
8f58abb943d4 many changes to NeuralNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 187
diff changeset
55 #- caching makes it really hard to borrow references to function outputs
8f58abb943d4 many changes to NeuralNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 187
diff changeset
56 return theano.function(inputs, outputs, unpack_single=False, linker=self.linker)
187
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
57
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
58 def __init__(self, ninputs, nhid, nclass, lr, nepochs,
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
59 l2coef=0.0,
189
8f58abb943d4 many changes to NeuralNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 187
diff changeset
60 linker='c&py',
187
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
61 hidden_layer=None):
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
62 class Vars:
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
63 def __init__(self, lr, l2coef):
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
64 lr = t.constant(lr)
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
65 l2coef = t.constant(l2coef)
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
66 input = t.matrix('input') # n_examples x n_inputs
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
67 target = t.ivector('target') # n_examples x 1
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
68 W2 = t.matrix('W2')
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
69 b2 = t.vector('b2')
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
70
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
71 if hidden_layer:
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
72 hid, hid_params, hid_ivals, hid_regularization = hidden_layer(input)
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
73 else:
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
74 W1 = t.matrix('W1')
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
75 b1 = t.vector('b1')
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
76 hid = t.tanh(b1 + t.dot(input, W1))
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
77 hid_params = [W1, b1]
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
78 hid_regularization = l2coef * t.sum(W1*W1)
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
79 hid_ivals = lambda : [_randshape(ninputs, nhid), _randshape(nhid)]
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
80
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
81 params = [W2, b2] + hid_params
189
8f58abb943d4 many changes to NeuralNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 187
diff changeset
82 activations = b2 + t.dot(hid, W2)
8f58abb943d4 many changes to NeuralNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 187
diff changeset
83 nll, predictions = nnet_ops.crossentropy_softmax_1hot(activations, target)
187
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
84 regularization = l2coef * t.sum(W2*W2) + hid_regularization
189
8f58abb943d4 many changes to NeuralNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 187
diff changeset
85 output_class = t.argmax(activations,1)
187
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
86 loss_01 = t.neq(output_class, target)
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
87 g_params = t.grad(nll + regularization, params)
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
88 new_params = [t.sub_inplace(p, lr * gp) for p,gp in zip(params, g_params)]
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
89 self.__dict__.update(locals()); del self.self
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
90 self.nhid = nhid
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
91 self.nclass = nclass
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
92 self.nepochs = nepochs
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
93 self.v = Vars(lr, l2coef)
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
94 self.params = None
189
8f58abb943d4 many changes to NeuralNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 187
diff changeset
95 self.linker = linker
187
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
96
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
97 def __call__(self, trainset=None, iparams=None):
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
98 if iparams is None:
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
99 iparams = [_randshape(self.nhid, self.nclass), _randshape(self.nclass)]\
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
100 + self.v.hid_ivals()
189
8f58abb943d4 many changes to NeuralNet
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 187
diff changeset
101 rval = NeuralNet._Model(self, iparams)
187
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
102 if trainset:
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
103 rval.update(trainset)
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
104 return rval
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
105
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
106
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
107 if __name__ == '__main__':
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
108 training_set1 = dataset.ArrayDataSet(numpy.array([[0, 0, 0],
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
109 [0, 1, 1],
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
110 [1, 0, 1],
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
111 [1, 1, 1]]),
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
112 {'input':slice(2),'target':2})
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
113 training_set2 = dataset.ArrayDataSet(numpy.array([[0, 0, 0],
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
114 [0, 1, 1],
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
115 [1, 0, 0],
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
116 [1, 1, 1]]),
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
117 {'input':slice(2),'target':2})
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
118 test_data = dataset.ArrayDataSet(numpy.array([[0, 0, 0],
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
119 [0, 1, 1],
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
120 [1, 0, 0],
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
121 [1, 1, 1]]),
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
122 {'input':slice(2)})
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
123
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
124 learn_algo = NeuralNet(2, 10, 3, .1, 1000)
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
125
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
126 model1 = learn_algo(training_set1)
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
127
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
128 model2 = learn_algo(training_set2)
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
129
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
130 n_match = 0
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
131 for o1, o2 in zip(model1(test_data), model2(test_data)):
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
132 n_match += (o1 == o2)
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
133
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
134 print n_match, numpy.sum(training_set1.fields()['target'] ==
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
135 training_set2.fields()['target'])
ebbb0e749565 added mlp_factory_approach
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
136