Mercurial > pylearn

annotate algorithms/daa.py @ 487:94a4c5b7293b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
DAA code more generic: Can now choose activation function and reconstruction cost.
author Joseph Turian <turian@gmail.com>
date Tue, 28 Oct 2008 02:21:50 -0400
parents 5ccb1662f9f6
children e06666ac32d5
rev   line source
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
1
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
2 import theano
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
3 from theano import tensor as T
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
4 from theano.tensor import nnet as NN
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
5 import numpy as N
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
6
487
94a4c5b7293b DAA code more generic:
Joseph Turian <turian@gmail.com>
parents: 486
diff changeset
7 from pylearn import cost as cost
94a4c5b7293b DAA code more generic:
Joseph Turian <turian@gmail.com>
parents: 486
diff changeset
8
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
9 class DenoisingAA(T.RModule):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
10
487
94a4c5b7293b DAA code more generic:
Joseph Turian <turian@gmail.com>
parents: 486
diff changeset
11 def __init__(self, input = None, regularize = True, tie_weights = True,
94a4c5b7293b DAA code more generic:
Joseph Turian <turian@gmail.com>
parents: 486
diff changeset
12 activation_function=NN.sigmoid, reconstruction_cost_function=cost.cross_entropy):
94a4c5b7293b DAA code more generic:
Joseph Turian <turian@gmail.com>
parents: 486
diff changeset
13 """
94a4c5b7293b DAA code more generic:
Joseph Turian <turian@gmail.com>
parents: 486
diff changeset
14 @param reconstruction_cost: Should return one cost per example (row)
94a4c5b7293b DAA code more generic:
Joseph Turian <turian@gmail.com>
parents: 486
diff changeset
15 """
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
16 super(DenoisingAA, self).__init__()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
17
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
18 # MODEL CONFIGURATION
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
19 self.regularize = regularize
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
20 self.tie_weights = tie_weights
487
94a4c5b7293b DAA code more generic:
Joseph Turian <turian@gmail.com>
parents: 486
diff changeset
21 self.activation_function = activation_function
94a4c5b7293b DAA code more generic:
Joseph Turian <turian@gmail.com>
parents: 486
diff changeset
22 self.reconstruction_cost_function = reconstruction_cost_function
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
23
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
24 # ACQUIRE/MAKE INPUT
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
25 if not input:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
26 input = T.matrix('input')
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
27 self.input = theano.External(input)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
28
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
29 # HYPER-PARAMETERS
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
30 self.lr = theano.Member(T.scalar())
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
31
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
32 # PARAMETERS
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
33 self.w1 = theano.Member(T.matrix())
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
34 if not tie_weights:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
35 self.w2 = theano.Member(T.matrix())
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
36 else:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
37 self.w2 = self.w1.T
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
38 self.b1 = theano.Member(T.vector())
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
39 self.b2 = theano.Member(T.vector())
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
40
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
41
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
42 # REGULARIZATION COST
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
43 self.regularization = self.build_regularization()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
44
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
45
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
46 ### NOISELESS ###
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
47
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
48 # HIDDEN LAYER
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
49 self.hidden_activation = T.dot(self.input, self.w1) + self.b1
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
50 self.hidden = self.hid_activation_function(self.hidden_activation)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
51
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
52 # RECONSTRUCTION LAYER
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
53 self.output_activation = T.dot(self.hidden, self.w2) + self.b2
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
54 self.output = self.out_activation_function(self.output_activation)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
55
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
56 # RECONSTRUCTION COST
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
57 self.reconstruction_costs = self.build_reconstruction_costs(self.output)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
58 self.reconstruction_cost = T.mean(self.reconstruction_costs)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
59
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
60 # TOTAL COST
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
61 self.cost = self.reconstruction_cost
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
62 if self.regularize:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
63 self.cost = self.cost + self.regularization
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
64
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
65
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
66 ### WITH NOISE ###
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
67 self.corrupted_input = self.build_corrupted_input()
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
68
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
69 # HIDDEN LAYER
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
70 self.nhidden_activation = T.dot(self.corrupted_input, self.w1) + self.b1
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
71 self.nhidden = self.hid_activation_function(self.nhidden_activation)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
72
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
73 # RECONSTRUCTION LAYER
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
74 self.noutput_activation = T.dot(self.nhidden, self.w2) + self.b2
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
75 self.noutput = self.out_activation_function(self.noutput_activation)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
76
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
77 # RECONSTRUCTION COST
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
78 self.nreconstruction_costs = self.build_reconstruction_costs(self.noutput)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
79 self.nreconstruction_cost = T.mean(self.nreconstruction_costs)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
80
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
81 # TOTAL COST
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
82 self.ncost = self.nreconstruction_cost
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
83 if self.regularize:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
84 self.ncost = self.ncost + self.regularization
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
85
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
86
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
87 # GRADIENTS AND UPDATES
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
88 if self.tie_weights:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
89 self.params = self.w1, self.b1, self.b2
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
90 else:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
91 self.params = self.w1, self.w2, self.b1, self.b2
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
92 gradients = T.grad(self.ncost, self.params)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
93 updates = dict((p, p - self.lr * g) for p, g in zip(self.params, gradients))
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
94
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
95 # INTERFACE METHODS
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
96 self.update = theano.Method(self.input, self.ncost, updates)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
97 self.compute_cost = theano.Method(self.input, self.cost)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
98 self.noisify = theano.Method(self.input, self.corrupted_input)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
99 self.reconstruction = theano.Method(self.input, self.output)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
100 self.representation = theano.Method(self.input, self.hidden)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
101 self.reconstruction_through_noise = theano.Method(self.input, [self.corrupted_input, self.noutput])
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
102
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
103 def _instance_initialize(self, obj, input_size = None, hidden_size = None, seed = None, **init):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
104 if (input_size is None) ^ (hidden_size is None):
486
5ccb1662f9f6 Small change to error message
Joseph Turian <turian@gmail.com>
parents: 485
diff changeset
105 raise ValueError("Must specify input_size and hidden_size or neither.")
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
106 super(DenoisingAA, self)._instance_initialize(obj, **init)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
107 if seed is not None:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
108 R = N.random.RandomState(seed)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
109 else:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
110 R = N.random
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
111 if input_size is not None:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
112 sz = (input_size, hidden_size)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
113 inf = 1/N.sqrt(input_size)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
114 hif = 1/N.sqrt(hidden_size)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
115 obj.w1 = R.uniform(size = sz, low = -inf, high = inf)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
116 if not self.tie_weights:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
117 obj.w2 = R.uniform(size = list(reversed(sz)), low = -inf, high = inf)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
118 obj.b1 = N.zeros(hidden_size)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
119 obj.b2 = N.zeros(input_size)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
120 if seed is not None:
482
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 476
diff changeset
121 obj.seed(seed)
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
122 obj.__hide__ = ['params']
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
123
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
124 def build_regularization(self):
487
94a4c5b7293b DAA code more generic:
Joseph Turian <turian@gmail.com>
parents: 486
diff changeset
125 """
94a4c5b7293b DAA code more generic:
Joseph Turian <turian@gmail.com>
parents: 486
diff changeset
126 @todo: Why do we need this function?
94a4c5b7293b DAA code more generic:
Joseph Turian <turian@gmail.com>
parents: 486
diff changeset
127 """
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
128 return T.zero() # no regularization!
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
129
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
130
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
131 class SigmoidXEDenoisingAA(DenoisingAA):
487
94a4c5b7293b DAA code more generic:
Joseph Turian <turian@gmail.com>
parents: 486
diff changeset
132 """
94a4c5b7293b DAA code more generic:
Joseph Turian <turian@gmail.com>
parents: 486
diff changeset
133 @todo: Merge this into the above.
94a4c5b7293b DAA code more generic:
Joseph Turian <turian@gmail.com>
parents: 486
diff changeset
134 """
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
135
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
136 def build_corrupted_input(self):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
137 self.noise_level = theano.Member(T.scalar())
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
138 return self.random.binomial(T.shape(self.input), 1, 1 - self.noise_level) * self.input
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
139
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
140 def hid_activation_function(self, activation):
487
94a4c5b7293b DAA code more generic:
Joseph Turian <turian@gmail.com>
parents: 486
diff changeset
141 return self.activation_function(activation)
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
142
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
143 def out_activation_function(self, activation):
487
94a4c5b7293b DAA code more generic:
Joseph Turian <turian@gmail.com>
parents: 486
diff changeset
144 return self.activation_function(activation)
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
145
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
146 def build_reconstruction_costs(self, output):
487
94a4c5b7293b DAA code more generic:
Joseph Turian <turian@gmail.com>
parents: 486
diff changeset
147 return self.reconstruction_cost_function(self.input, output)
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
148
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
149 def build_regularization(self):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
150 self.l2_coef = theano.Member(T.scalar())
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
151 if self.tie_weights:
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
152 return self.l2_coef * T.sum(self.w1 * self.w1)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
153 else:
485
e8c37244b54f Small bugfix in regularization
Joseph Turian <turian@gmail.com>
parents: 482
diff changeset
154 return self.l2_coef * (T.sum(self.w1 * self.w1) + T.sum(self.w2 * self.w2))
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
155
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
156 def _instance_initialize(self, obj, input_size = None, hidden_size = None, seed = None, **init):
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
157 init.setdefault('noise_level', 0)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
158 init.setdefault('l2_coef', 0)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
159 super(SigmoidXEDenoisingAA, self)._instance_initialize(obj, input_size, hidden_size, seed, **init)
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
diff changeset
160