Mercurial > pylearn

comparison algorithms/daa.py @ 476:8fcd0f3d9a17

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
added a few algorithms
author Olivier Breuleux <breuleuo@iro.umontreal.ca>
date Mon, 27 Oct 2008 17:26:00 -0400
parents
children b15dad843c8c
comparison
equal deleted inserted replaced
475:11e0357f06f4 476:8fcd0f3d9a17
1
2 import theano
3 from theano import tensor as T
4 from theano.tensor import nnet as NN
5 import numpy as N
6
7 class DenoisingAA(T.RModule):
8
9 def __init__(self, input = None, regularize = True, tie_weights = True):
10 super(DenoisingAA, self).__init__()
11
12 # MODEL CONFIGURATION
13 self.regularize = regularize
14 self.tie_weights = tie_weights
15
16 # ACQUIRE/MAKE INPUT
17 if not input:
18 input = T.matrix('input')
19 self.input = theano.External(input)
20
21 # HYPER-PARAMETERS
22 self.lr = theano.Member(T.scalar())
23
24 # PARAMETERS
25 self.w1 = theano.Member(T.matrix())
26 if not tie_weights:
27 self.w2 = theano.Member(T.matrix())
28 else:
29 self.w2 = self.w1.T
30 self.b1 = theano.Member(T.vector())
31 self.b2 = theano.Member(T.vector())
32
33
34 # REGULARIZATION COST
35 self.regularization = self.build_regularization()
36
37
38 ### NOISELESS ###
39
40 # HIDDEN LAYER
41 self.hidden_activation = T.dot(self.input, self.w1) + self.b1
42 self.hidden = self.hid_activation_function(self.hidden_activation)
43
44 # RECONSTRUCTION LAYER
45 self.output_activation = T.dot(self.hidden, self.w2) + self.b2
46 self.output = self.out_activation_function(self.output_activation)
47
48 # RECONSTRUCTION COST
49 self.reconstruction_costs = self.build_reconstruction_costs(self.output)
50 self.reconstruction_cost = T.mean(self.reconstruction_costs)
51
52 # TOTAL COST
53 self.cost = self.reconstruction_cost
54 if self.regularize:
55 self.cost = self.cost + self.regularization
56
57
58 ### WITH NOISE ###
59 self.corrupted_input = self.build_corrupted_input()
60
61 # HIDDEN LAYER
62 self.nhidden_activation = T.dot(self.corrupted_input, self.w1) + self.b1
63 self.nhidden = self.hid_activation_function(self.nhidden_activation)
64
65 # RECONSTRUCTION LAYER
66 self.noutput_activation = T.dot(self.nhidden, self.w2) + self.b2
67 self.noutput = self.out_activation_function(self.noutput_activation)
68
69 # RECONSTRUCTION COST
70 self.nreconstruction_costs = self.build_reconstruction_costs(self.noutput)
71 self.nreconstruction_cost = T.mean(self.nreconstruction_costs)
72
73 # TOTAL COST
74 self.ncost = self.nreconstruction_cost
75 if self.regularize:
76 self.ncost = self.ncost + self.regularization
77
78
79 # GRADIENTS AND UPDATES
80 if self.tie_weights:
81 self.params = self.w1, self.b1, self.b2
82 else:
83 self.params = self.w1, self.w2, self.b1, self.b2
84 gradients = T.grad(self.ncost, self.params)
85 updates = dict((p, p - self.lr * g) for p, g in zip(self.params, gradients))
86
87 # INTERFACE METHODS
88 self.update = theano.Method(self.input, self.ncost, updates)
89 self.compute_cost = theano.Method(self.input, self.cost)
90 self.noisify = theano.Method(self.input, self.corrupted_input)
91 self.reconstruction = theano.Method(self.input, self.output)
92 self.representation = theano.Method(self.input, self.hidden)
93 self.reconstruction_through_noise = theano.Method(self.input, [self.corrupted_input, self.noutput])
94
95 def _instance_initialize(self, obj, input_size = None, hidden_size = None, seed = None, **init):
96 if (input_size is None) ^ (hidden_size is None):
97 raise ValueError("Must specify hidden_size and target_size or neither.")
98 super(DenoisingAA, self)._instance_initialize(obj, **init)
99 if seed is not None:
100 R = N.random.RandomState(seed)
101 else:
102 R = N.random
103 if input_size is not None:
104 sz = (input_size, hidden_size)
105 inf = 1/N.sqrt(input_size)
106 hif = 1/N.sqrt(hidden_size)
107 obj.w1 = R.uniform(size = sz, low = -inf, high = inf)
108 if not self.tie_weights:
109 obj.w2 = R.uniform(size = list(reversed(sz)), low = -inf, high = inf)
110 obj.b1 = N.zeros(hidden_size)
111 obj.b2 = N.zeros(input_size)
112 if seed is not None:
113 self.seed(seed)
114 obj.__hide__ = ['params']
115
116 def build_regularization(self):
117 return T.zero() # no regularization!
118
119
120 class SigmoidXEDenoisingAA(DenoisingAA):
121
122 def build_corrupted_input(self):
123 self.noise_level = theano.Member(T.scalar())
124 return self.random.binomial(T.shape(self.input), 1, 1 - self.noise_level) * self.input
125
126 def hid_activation_function(self, activation):
127 return NN.sigmoid(activation)
128
129 def out_activation_function(self, activation):
130 return NN.sigmoid(activation)
131
132 def build_reconstruction_costs(self, output):
133 reconstruction_cost_matrix = -(self.input * T.log(output) + (1 - self.input) * T.log(1 - output))
134 return T.sum(reconstruction_cost_matrix, axis=1)
135
136 def build_regularization(self):
137 self.l2_coef = theano.Member(T.scalar())
138 if self.tie_weights:
139 return self.l2_coef * T.sum(self.w1 * self.w1)
140 else:
141 return self.l2_coef * T.sum(self.w1 * self.w1) + T.sum(self.w2 * self.w2)
142
143 def _instance_initialize(self, obj, input_size = None, hidden_size = None, seed = None, **init):
144 init.setdefault('noise_level', 0)
145 init.setdefault('l2_coef', 0)
146 super(SigmoidXEDenoisingAA, self)._instance_initialize(obj, input_size, hidden_size, seed, **init)
147