Mercurial > pylearn

annotate algorithms/stacker.py @ 495:7560817a07e8

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
nnet_ops => nnet
author Joseph Turian <turian@gmail.com>
date Tue, 28 Oct 2008 12:09:39 -0400
parents bb6bdd3b7ff3
children 2be795cc5c3a
rev   line source
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
1
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
2 import theano
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
3 from theano import tensor as T
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
4 import sys
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
5 import numpy as N
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
6
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
7 class Stacker(T.RModule):
489
bb6bdd3b7ff3 Another bit of documentation
Joseph Turian <turian@gmail.com>
parents: 476
diff changeset
8 """
bb6bdd3b7ff3 Another bit of documentation
Joseph Turian <turian@gmail.com>
parents: 476
diff changeset
9 @todo: Maybe compile functions on demand, rather than immediately.
bb6bdd3b7ff3 Another bit of documentation
Joseph Turian <turian@gmail.com>
parents: 476
diff changeset
10 """
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
11
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
12 def __init__(self, submodules, input = None, regularize = False):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
13 super(Stacker, self).__init__()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
14
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
15 current = input
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
16 layers = []
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
17 for i, (submodule, outname) in enumerate(submodules):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
18 layer = submodule(current, regularize = regularize)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
19 layers.append(layer)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
20 current = layer[outname]
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
21 self.layers = layers
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
22
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
23 self.input = self.layers[0].input
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
24 self.output = current
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
25
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
26 local_update = []
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
27 global_update = []
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
28 to_update = []
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
29 all_kits = []
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
30 for layer in layers:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
31 u = layer.update
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
32 u.resolve_all()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
33 to_update += u.updates.keys()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
34 all_kits += u.kits
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
35 # the input is the whole deep model's input instead of the layer's own
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
36 # input (which is previous_layer[outname])
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
37 inputs = [self.input] + u.inputs[1:]
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
38 method = theano.Method(inputs, u.outputs, u.updates, u.kits)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
39 local_update.append(method)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
40 global_update.append(
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
41 theano.Method(inputs,
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
42 u.outputs,
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
43 # we update the params of the previous layers too but wrt
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
44 # this layer's cost
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
45 dict((param, param - layer.lr * T.grad(layer.cost, param))
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
46 for param in to_update),
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
47 list(all_kits)))
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
48
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
49 self.local_update = local_update
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
50 self.global_update = global_update
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
51 self.update = self.global_update[-1]
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
52 self.compute = theano.Method(self.input, self.output)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
53 ll = self.layers[-1]
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
54 for name, method in ll.components_map():
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
55 if isinstance(method, theano.Method) and not hasattr(self, name):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
56 m = method.dup()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
57 m.resolve_all()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
58 m.inputs = [self.input if x is ll.input else x for x in m.inputs]
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
59 setattr(self, name, m)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
60
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
61 def _instance_initialize(self, obj, nunits = None, lr = 0.01, seed = None, **kwargs):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
62 super(Stacker, self)._instance_initialize(obj, **kwargs)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
63 if seed is not None:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
64 R = N.random.RandomState(seed)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
65 else:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
66 R = N.random
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
67 for layer in obj.layers:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
68 if layer.lr is None:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
69 layer.lr = lr
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
70 if nunits:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
71 if len(nunits) != len(obj.layers) + 1:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
72 raise ValueError('You should give exactly one more unit numbers as there are layers.')
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
73 for ni, no, layer in zip(nunits[:-1], nunits[1:], obj.layers):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
74 if seed is not None:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
75 layer.initialize(ni, no, seed = R.random_integers(sys.maxint - 1))
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
76 else:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
77 layer.initialize(ni, no)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
78 if seed is not None:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
79 obj.seed(seed)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
80
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
81 def _instance_flops_approx(self, obj):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
82 rval = 0
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
83 for layer in obj.layers:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
84 rval += layer.flops_approx()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
85 return rval
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
86