Mercurial > pylearn

annotate algorithms/stacker.py @ 523:111e547ffa7b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
modified to use the new implecement of ops and use the new interface to theano.function
author Frederic Bastien <bastienf@iro.umontreal.ca>
date Fri, 14 Nov 2008 16:39:59 -0500
parents 2b0e10ac6929
children
rev   line source
516
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 503
diff changeset
1
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 503
diff changeset
2 # for example in examples:
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 503
diff changeset
3 # repr = example
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 503
diff changeset
4 # for layer in stacked.layers:
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 503
diff changeset
5 # layer.update(repr)
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 503
diff changeset
6 # repr = layer.representation(repr)
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
7
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
8 import theano
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
9 from theano import tensor as T
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
10 import sys
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
11 import numpy as N
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
12
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
13 class Stacker(T.RModule):
489
bb6bdd3b7ff3 Another bit of documentation
Joseph Turian <turian@gmail.com>
parents: 476
diff changeset
14 """
498
2be795cc5c3a More documentation + todo
Joseph Turian <turian@gmail.com>
parents: 489
diff changeset
15 @note: Assumes some names in the layers: input, cost, lr, and update
489
bb6bdd3b7ff3 Another bit of documentation
Joseph Turian <turian@gmail.com>
parents: 476
diff changeset
16 @todo: Maybe compile functions on demand, rather than immediately.
bb6bdd3b7ff3 Another bit of documentation
Joseph Turian <turian@gmail.com>
parents: 476
diff changeset
17 """
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
18
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
19 def __init__(self, submodules, input = None, regularize = False):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
20 super(Stacker, self).__init__()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
21
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
22 current = input
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
23 layers = []
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
24 for i, (submodule, outname) in enumerate(submodules):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
25 layer = submodule(current, regularize = regularize)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
26 layers.append(layer)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
27 current = layer[outname]
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
28 self.layers = layers
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
29
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
30 self.input = self.layers[0].input
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
31 self.output = current
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
32
516
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 503
diff changeset
33 representation = []
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
34 local_update = []
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
35 global_update = []
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
36 to_update = []
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
37 all_kits = []
516
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 503
diff changeset
38 for layer, (submodule, outname) in zip(layers, submodules):
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
39 u = layer.update
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
40 u.resolve_all()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
41 to_update += u.updates.keys()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
42 all_kits += u.kits
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
43 # the input is the whole deep model's input instead of the layer's own
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
44 # input (which is previous_layer[outname])
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
45 inputs = [self.input] + u.inputs[1:]
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
46 method = theano.Method(inputs, u.outputs, u.updates, u.kits)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
47 local_update.append(method)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
48 global_update.append(
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
49 theano.Method(inputs,
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
50 u.outputs,
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
51 # we update the params of the previous layers too but wrt
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
52 # this layer's cost
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
53 dict((param, param - layer.lr * T.grad(layer.cost, param))
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
54 for param in to_update),
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
55 list(all_kits)))
516
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 503
diff changeset
56 representation.append(theano.Method(self.input, layer[outname]))
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
57
498
2be795cc5c3a More documentation + todo
Joseph Turian <turian@gmail.com>
parents: 489
diff changeset
58 # @todo: Add diagnostics
2be795cc5c3a More documentation + todo
Joseph Turian <turian@gmail.com>
parents: 489
diff changeset
59 # self.diagnose_from_input = Method([self.input], self.layers[0].diagnose.outputs + self.layers[1].diagnose.outputs ...
2be795cc5c3a More documentation + todo
Joseph Turian <turian@gmail.com>
parents: 489
diff changeset
60
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
61 self.local_update = local_update
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
62 self.global_update = global_update
516
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 503
diff changeset
63 self.representation = representation
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
64 self.update = self.global_update[-1]
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
65 self.compute = theano.Method(self.input, self.output)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
66 ll = self.layers[-1]
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
67 for name, method in ll.components_map():
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
68 if isinstance(method, theano.Method) and not hasattr(self, name):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
69 m = method.dup()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
70 m.resolve_all()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
71 m.inputs = [self.input if x is ll.input else x for x in m.inputs]
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
72 setattr(self, name, m)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
73
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
74 def _instance_initialize(self, obj, nunits = None, lr = 0.01, seed = None, **kwargs):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
75 super(Stacker, self)._instance_initialize(obj, **kwargs)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
76 if seed is not None:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
77 R = N.random.RandomState(seed)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
78 else:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
79 R = N.random
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
80 for layer in obj.layers:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
81 if layer.lr is None:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
82 layer.lr = lr
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
83 if nunits:
503
c7ce66b4e8f4 Extensions to algorithms, and some cleanup (by defining linear_output result).
Joseph Turian <turian@gmail.com>
parents: 498
diff changeset
84 obj.input_dimension = nunits[0]
c7ce66b4e8f4 Extensions to algorithms, and some cleanup (by defining linear_output result).
Joseph Turian <turian@gmail.com>
parents: 498
diff changeset
85 obj.output_dimension = nunits[-1]
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
86 if len(nunits) != len(obj.layers) + 1:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
87 raise ValueError('You should give exactly one more unit numbers as there are layers.')
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
88 for ni, no, layer in zip(nunits[:-1], nunits[1:], obj.layers):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
89 if seed is not None:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
90 layer.initialize(ni, no, seed = R.random_integers(sys.maxint - 1))
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
91 else:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
92 layer.initialize(ni, no)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
93 if seed is not None:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
94 obj.seed(seed)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
95
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
96 def _instance_flops_approx(self, obj):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
97 rval = 0
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
98 for layer in obj.layers:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
99 rval += layer.flops_approx()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
100 return rval
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
101