Mercurial > pylearn

annotate pylearn/algorithms/stacker.py @ 1531:88f361283a19 tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Fix url/name to pylearn2.
author Frederic Bastien <nouiz@nouiz.org>
date Mon, 09 Sep 2013 10:08:05 -0400
parents f4729745bb58
children
rev   line source
516
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 503
diff changeset
1
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 503
diff changeset
2 # for example in examples:
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 503
diff changeset
3 # repr = example
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 503
diff changeset
4 # for layer in stacked.layers:
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 503
diff changeset
5 # layer.update(repr)
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 503
diff changeset
6 # repr = layer.representation(repr)
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
7
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
8 import theano
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
9 from theano import tensor as T
665
070a7d68d3a1 Changed RModule to deprecated.RModule .... temporarily until they are fixed with
desjagui@atchoum.iro.umontreal.ca
parents: 635
diff changeset
10 from theano.tensor.deprecated import rmodule
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
11 import sys
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
12 import numpy as N
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
13
665
070a7d68d3a1 Changed RModule to deprecated.RModule .... temporarily until they are fixed with
desjagui@atchoum.iro.umontreal.ca
parents: 635
diff changeset
14 class Stacker(rmodule.RModule):
489
bb6bdd3b7ff3 Another bit of documentation
Joseph Turian <turian@gmail.com>
parents: 476
diff changeset
15 """
498
2be795cc5c3a More documentation + todo
Joseph Turian <turian@gmail.com>
parents: 489
diff changeset
16 @note: Assumes some names in the layers: input, cost, lr, and update
489
bb6bdd3b7ff3 Another bit of documentation
Joseph Turian <turian@gmail.com>
parents: 476
diff changeset
17 @todo: Maybe compile functions on demand, rather than immediately.
bb6bdd3b7ff3 Another bit of documentation
Joseph Turian <turian@gmail.com>
parents: 476
diff changeset
18 """
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
19
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
20 def __init__(self, submodules, input = None, regularize = False):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
21 super(Stacker, self).__init__()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
22
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
23 current = input
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
24 layers = []
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
25 for i, (submodule, outname) in enumerate(submodules):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
26 layer = submodule(current, regularize = regularize)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
27 layers.append(layer)
785
12f587e37ee3 * fixed test_stacker.py
desjagui@atchoum.iro.umontreal.ca
parents: 665
diff changeset
28 current = getattr(layer, outname)
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
29 self.layers = layers
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
30
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
31 self.input = self.layers[0].input
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
32 self.output = current
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
33
516
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 503
diff changeset
34 representation = []
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
35 local_update = []
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
36 global_update = []
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
37 to_update = []
516
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 503
diff changeset
38 for layer, (submodule, outname) in zip(layers, submodules):
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
39 u = layer.update
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
40 u.resolve_all()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
41 to_update += u.updates.keys()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
42 # the input is the whole deep model's input instead of the layer's own
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
43 # input (which is previous_layer[outname])
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
44 inputs = [self.input] + u.inputs[1:]
785
12f587e37ee3 * fixed test_stacker.py
desjagui@atchoum.iro.umontreal.ca
parents: 665
diff changeset
45 method = theano.Method(inputs, u.outputs, u.updates)
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
46 local_update.append(method)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
47 global_update.append(
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
48 theano.Method(inputs,
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
49 u.outputs,
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
50 # we update the params of the previous layers too but wrt
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
51 # this layer's cost
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
52 dict((param, param - layer.lr * T.grad(layer.cost, param))
785
12f587e37ee3 * fixed test_stacker.py
desjagui@atchoum.iro.umontreal.ca
parents: 665
diff changeset
53 for param in to_update)))
12f587e37ee3 * fixed test_stacker.py
desjagui@atchoum.iro.umontreal.ca
parents: 665
diff changeset
54 representation.append(theano.Method(self.input, getattr(layer,outname)))
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
55
498
2be795cc5c3a More documentation + todo
Joseph Turian <turian@gmail.com>
parents: 489
diff changeset
56 # @todo: Add diagnostics
2be795cc5c3a More documentation + todo
Joseph Turian <turian@gmail.com>
parents: 489
diff changeset
57 # self.diagnose_from_input = Method([self.input], self.layers[0].diagnose.outputs + self.layers[1].diagnose.outputs ...
2be795cc5c3a More documentation + todo
Joseph Turian <turian@gmail.com>
parents: 489
diff changeset
58
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
59 self.local_update = local_update
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
60 self.global_update = global_update
516
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 503
diff changeset
61 self.representation = representation
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
62 self.update = self.global_update[-1]
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
63 self.compute = theano.Method(self.input, self.output)
785
12f587e37ee3 * fixed test_stacker.py
desjagui@atchoum.iro.umontreal.ca
parents: 665
diff changeset
64
12f587e37ee3 * fixed test_stacker.py
desjagui@atchoum.iro.umontreal.ca
parents: 665
diff changeset
65 # takes method from last layer (usually ll.classify), copies it to self.,
12f587e37ee3 * fixed test_stacker.py
desjagui@atchoum.iro.umontreal.ca
parents: 665
diff changeset
66 # while converting its input to deal with the global "model" input
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
67 ll = self.layers[-1]
785
12f587e37ee3 * fixed test_stacker.py
desjagui@atchoum.iro.umontreal.ca
parents: 665
diff changeset
68 for name, method in ll.__dict__['local_attr'].iteritems():
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
69 if isinstance(method, theano.Method) and not hasattr(self, name):
785
12f587e37ee3 * fixed test_stacker.py
desjagui@atchoum.iro.umontreal.ca
parents: 665
diff changeset
70 if not isinstance(method.inputs, (list,dict)):
12f587e37ee3 * fixed test_stacker.py
desjagui@atchoum.iro.umontreal.ca
parents: 665
diff changeset
71 method.inputs = [method.inputs]
818
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 785
diff changeset
72 inputs = []
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 785
diff changeset
73 for x in method.inputs:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 785
diff changeset
74 if x is ll.input:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 785
diff changeset
75 inputs += [self.input]
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 785
diff changeset
76 else:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 785
diff changeset
77 inputs += [x]
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 785
diff changeset
78 #backport
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 785
diff changeset
79 #inputs = [self.input if x is ll.input else x for x in method.inputs]
785
12f587e37ee3 * fixed test_stacker.py
desjagui@atchoum.iro.umontreal.ca
parents: 665
diff changeset
80 m = theano.Method(inputs, method.outputs, method.updates)
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
81 setattr(self, name, m)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
82
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
83 def _instance_initialize(self, obj, nunits = None, lr = 0.01, seed = None, **kwargs):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
84 super(Stacker, self)._instance_initialize(obj, **kwargs)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
85 if seed is not None:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
86 R = N.random.RandomState(seed)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
87 else:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
88 R = N.random
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
89 for layer in obj.layers:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
90 if layer.lr is None:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
91 layer.lr = lr
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
92 if nunits:
503
c7ce66b4e8f4 Extensions to algorithms, and some cleanup (by defining linear_output result).
Joseph Turian <turian@gmail.com>
parents: 498
diff changeset
93 obj.input_dimension = nunits[0]
c7ce66b4e8f4 Extensions to algorithms, and some cleanup (by defining linear_output result).
Joseph Turian <turian@gmail.com>
parents: 498
diff changeset
94 obj.output_dimension = nunits[-1]
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
95 if len(nunits) != len(obj.layers) + 1:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
96 raise ValueError('You should give exactly one more unit numbers as there are layers.')
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
97 for ni, no, layer in zip(nunits[:-1], nunits[1:], obj.layers):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
98 if seed is not None:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
99 layer.initialize(ni, no, seed = R.random_integers(sys.maxint - 1))
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
100 else:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
101 layer.initialize(ni, no)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
102 if seed is not None:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
103 obj.seed(seed)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
104
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
105 def _instance_flops_approx(self, obj):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
106 rval = 0
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
107 for layer in obj.layers:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
108 rval += layer.flops_approx()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
109 return rval
635
89bc88affef0 Reverting out of changeset 633,634. Hopefully i did this write !
projects@lgcm
parents: 633
diff changeset
110