Mercurial > pylearn

annotate pylearn/algorithms/aa.py @ 1469:c41fdf8c35b8

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fix about floatX=float32 to remove error in the build bot.
author Frederic Bastien <nouiz@nouiz.org>
date Wed, 27 Apr 2011 11:34:36 -0400
parents ba65e95d1221
children
rev   line source
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
1
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
2 import theano
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
3 from theano import tensor as T
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
4 from theano.tensor import nnet as NN
1469
c41fdf8c35b8 fix about floatX=float32 to remove error in the build bot.
Frederic Bastien <nouiz@nouiz.org>
parents: 784
diff changeset
5 floatX = theano.config.floatX
c41fdf8c35b8 fix about floatX=float32 to remove error in the build bot.
Frederic Bastien <nouiz@nouiz.org>
parents: 784
diff changeset
6
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
7 import numpy as N
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
8
592
8d0b73c7d768 Minor cosmetic changes
Olivier Delalleau <delallea@iro>
parents: 537
diff changeset
9 class AutoEncoder(theano.Module):
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
10
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
11 def __init__(self, input = None, regularize = True, tie_weights = True):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
12 super(AutoEncoder, self).__init__()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
13
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
14 # MODEL CONFIGURATION
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
15 self.regularize = regularize
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
16 self.tie_weights = tie_weights
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
17
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
18 # ACQUIRE/MAKE INPUT
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
19 if not input:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
20 input = T.matrix('input')
784
ba65e95d1221 removed manual call to Member and Variable as this is deprecated in theano.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 592
diff changeset
21 self.input = input
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
22
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
23 # HYPER-PARAMETERS
784
ba65e95d1221 removed manual call to Member and Variable as this is deprecated in theano.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 592
diff changeset
24 self.lr = T.scalar()
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
25
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
26 # PARAMETERS
784
ba65e95d1221 removed manual call to Member and Variable as this is deprecated in theano.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 592
diff changeset
27 self.w1 = T.matrix()
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
28 if not tie_weights:
784
ba65e95d1221 removed manual call to Member and Variable as this is deprecated in theano.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 592
diff changeset
29 self.w2 = T.matrix()
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
30 else:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
31 self.w2 = self.w1.T
784
ba65e95d1221 removed manual call to Member and Variable as this is deprecated in theano.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 592
diff changeset
32 self.b1 = T.vector()
ba65e95d1221 removed manual call to Member and Variable as this is deprecated in theano.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 592
diff changeset
33 self.b2 = T.vector()
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
34
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
35 # HIDDEN LAYER
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
36 self.hidden_activation = T.dot(input, self.w1) + self.b1
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
37 self.hidden = self.build_hidden()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
38
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
39 # RECONSTRUCTION LAYER
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
40 self.output_activation = T.dot(self.hidden, self.w2) + self.b2
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
41 self.output = self.build_output()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
42
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
43 # RECONSTRUCTION COST
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
44 self.reconstruction_cost = self.build_reconstruction_cost()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
45
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
46 # REGULARIZATION COST
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
47 self.regularization = self.build_regularization()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
48
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
49 # TOTAL COST
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
50 self.cost = self.reconstruction_cost
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
51 if self.regularize:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
52 self.cost = self.cost + self.regularization
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
53
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
54 # GRADIENTS AND UPDATES
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
55 if self.tie_weights:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
56 self.params = self.w1, self.b1, self.b2
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
57 else:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
58 self.params = self.w1, self.w2, self.b1, self.b2
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
59 gradients = T.grad(self.cost, self.params)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
60 updates = dict((p, p - self.lr * g) for p, g in zip(self.params, gradients))
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
61
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
62 # INTERFACE METHODS
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
63 self.update = theano.Method(input, self.cost, updates)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
64 self.reconstruction = theano.Method(input, self.output)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
65 self.representation = theano.Method(input, self.hidden)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
66
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
67 def _instance_initialize(self, obj, input_size = None, hidden_size = None, seed = None, **init):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
68 if (input_size is None) ^ (hidden_size is None):
592
8d0b73c7d768 Minor cosmetic changes
Olivier Delalleau <delallea@iro>
parents: 537
diff changeset
69 raise ValueError("Must specify hidden_size and input_size or neither.")
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
70 super(AutoEncoder, self)._instance_initialize(obj, **init)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
71 if seed is not None:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
72 R = N.random.RandomState(seed)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
73 else:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
74 R = N.random
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
75 if input_size is not None:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
76 sz = (input_size, hidden_size)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
77 range = 1/N.sqrt(input_size)
1469
c41fdf8c35b8 fix about floatX=float32 to remove error in the build bot.
Frederic Bastien <nouiz@nouiz.org>
parents: 784
diff changeset
78 if floatX=='float32':
c41fdf8c35b8 fix about floatX=float32 to remove error in the build bot.
Frederic Bastien <nouiz@nouiz.org>
parents: 784
diff changeset
79 range = N.float32(range)
c41fdf8c35b8 fix about floatX=float32 to remove error in the build bot.
Frederic Bastien <nouiz@nouiz.org>
parents: 784
diff changeset
80 obj.w1 = N.asarray(R.uniform(size = sz, low = -range, high = range),
c41fdf8c35b8 fix about floatX=float32 to remove error in the build bot.
Frederic Bastien <nouiz@nouiz.org>
parents: 784
diff changeset
81 dtype=floatX)
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
82 if not self.tie_weights:
1469
c41fdf8c35b8 fix about floatX=float32 to remove error in the build bot.
Frederic Bastien <nouiz@nouiz.org>
parents: 784
diff changeset
83 obj.w2 = N.asarray(R.uniform(size = list(reversed(sz)), low = -range, high = range),
c41fdf8c35b8 fix about floatX=float32 to remove error in the build bot.
Frederic Bastien <nouiz@nouiz.org>
parents: 784
diff changeset
84 dtype=floatX)
c41fdf8c35b8 fix about floatX=float32 to remove error in the build bot.
Frederic Bastien <nouiz@nouiz.org>
parents: 784
diff changeset
85 obj.b1 = N.zeros(hidden_size, dtype=floatX)
c41fdf8c35b8 fix about floatX=float32 to remove error in the build bot.
Frederic Bastien <nouiz@nouiz.org>
parents: 784
diff changeset
86 obj.b2 = N.zeros(input_size, dtype=floatX)
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
87
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
88 def build_regularization(self):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
89 return T.zero() # no regularization!
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
90
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
91
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
92 class SigmoidXEAutoEncoder(AutoEncoder):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
93
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
94 def build_hidden(self):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
95 return NN.sigmoid(self.hidden_activation)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
96
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
97 def build_output(self):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
98 return NN.sigmoid(self.output_activation)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
99
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
100 def build_reconstruction_cost(self):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
101 self.reconstruction_cost_matrix = self.input * T.log(self.output) + (1.0 - self.input) * T.log(1.0 - self.output)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
102 self.reconstruction_costs = -T.sum(self.reconstruction_cost_matrix, axis=1)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
103 return T.sum(self.reconstruction_costs)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
104
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
105 def build_regularization(self):
784
ba65e95d1221 removed manual call to Member and Variable as this is deprecated in theano.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 592
diff changeset
106 self.l2_coef = T.scalar()
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
107 if self.tie_weights:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
108 return self.l2_coef * T.sum(self.w1 * self.w1)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
109 else:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
110 return self.l2_coef * T.sum(self.w1 * self.w1) + T.sum(self.w2 * self.w2)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
111
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
112 def _instance_initialize(self, obj, input_size = None, hidden_size = None, **init):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
113 init.setdefault('l2_coef', 0)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
114 super(SigmoidXEAutoEncoder, self)._instance_initialize(obj, input_size, hidden_size, **init)