Mercurial > ift6266

annotate scripts/stacked_dae/stacked_convolutional_dae.py @ 138:128507ac4edf

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Initial commit for the stacked convolutional denoising autoencoders
author Owner <salahmeister@gmail.com>
date Sun, 21 Feb 2010 17:30:38 -0600
parents
children
rev   line source
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
1 import numpy
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
2 import theano
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
3 import time
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
4 import theano.tensor as T
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
5 from theano.tensor.shared_randomstreams import RandomStreams
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
6 import theano.sandbox.softsign
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
7
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
8 from theano.tensor.signal import downsample
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
9 from theano.tensor.nnet import conv
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
10 import gzip
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
11 import cPickle
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
12
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
13
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
14 class LogisticRegression(object):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
15
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
16 def __init__(self, input, n_in, n_out):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
17
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
18 self.W = theano.shared( value=numpy.zeros((n_in,n_out),
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
19 dtype = theano.config.floatX) )
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
20
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
21 self.b = theano.shared( value=numpy.zeros((n_out,),
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
22 dtype = theano.config.floatX) )
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
23
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
24 self.p_y_given_x = T.nnet.softmax(T.dot(input, self.W)+self.b)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
25
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
26
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
27 self.y_pred=T.argmax(self.p_y_given_x, axis=1)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
28
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
29 self.params = [self.W, self.b]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
30
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
31 def negative_log_likelihood(self, y):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
32 return -T.mean(T.log(self.p_y_given_x)[T.arange(y.shape[0]),y])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
33
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
34 def MSE(self, y):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
35 return -T.mean(abs((self.p_y_given_x)[T.arange(y.shape[0]),y]-y)**2)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
36
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
37 def errors(self, y):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
38 if y.ndim != self.y_pred.ndim:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
39 raise TypeError('y should have the same shape as self.y_pred',
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
40 ('y', target.type, 'y_pred', self.y_pred.type))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
41
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
42
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
43 if y.dtype.startswith('int'):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
44 return T.mean(T.neq(self.y_pred, y))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
45 else:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
46 raise NotImplementedError()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
47
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
48
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
49 class SigmoidalLayer(object):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
50 def __init__(self, rng, input, n_in, n_out):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
51
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
52 self.input = input
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
53
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
54 W_values = numpy.asarray( rng.uniform( \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
55 low = -numpy.sqrt(6./(n_in+n_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
56 high = numpy.sqrt(6./(n_in+n_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
57 size = (n_in, n_out)), dtype = theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
58 self.W = theano.shared(value = W_values)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
59
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
60 b_values = numpy.zeros((n_out,), dtype= theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
61 self.b = theano.shared(value= b_values)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
62
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
63 self.output = T.tanh(T.dot(input, self.W) + self.b)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
64 self.params = [self.W, self.b]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
65
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
66 class dA_conv(object):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
67
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
68 def __init__(self, corruption_level = 0.1, input = None, shared_W = None,\
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
69 shared_b = None, filter_shape = None, image_shape = None, poolsize = (2,2)):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
70
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
71 theano_rng = RandomStreams()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
72
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
73 fan_in = numpy.prod(filter_shape[1:])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
74 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
75
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
76 center = theano.shared(value = 1, name="center")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
77 scale = theano.shared(value = 2, name="scale")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
78
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
79 if shared_W != None and shared_b != None :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
80 self.W = shared_W
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
81 self.b = shared_b
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
82 else:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
83 initial_W = numpy.asarray( numpy.random.uniform( \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
84 low = -numpy.sqrt(6./(fan_in+fan_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
85 high = numpy.sqrt(6./(fan_in+fan_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
86 size = filter_shape), dtype = theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
87 initial_b = numpy.zeros((filter_shape[0],), dtype= theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
88
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
89
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
90 self.W = theano.shared(value = initial_W, name = "W")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
91 self.b = theano.shared(value = initial_b, name = "b")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
92
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
93
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
94 initial_b_prime= numpy.zeros((filter_shape[1],))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
95
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
96 self.W_prime=T.dtensor4('W_prime')
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
97
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
98 self.b_prime = theano.shared(value = initial_b_prime, name = "b_prime")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
99
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
100 self.x = input
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
101
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
102 self.tilde_x = theano_rng.binomial( self.x.shape, 1, 1 - corruption_level) * self.x
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
103
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
104 conv1_out = conv.conv2d(self.tilde_x, self.W, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
105 filter_shape=filter_shape, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
106 image_shape=image_shape, border_mode='valid')
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
107
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
108
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
109 self.y = T.tanh(conv1_out + self.b.dimshuffle('x', 0, 'x', 'x'))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
110
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
111
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
112 da_filter_shape = [ filter_shape[1], filter_shape[0], filter_shape[2],\
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
113 filter_shape[3] ]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
114 da_image_shape = [ image_shape[0],filter_shape[0],image_shape[2]-filter_shape[2]+1, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
115 image_shape[3]-filter_shape[3]+1 ]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
116 initial_W_prime = numpy.asarray( numpy.random.uniform( \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
117 low = -numpy.sqrt(6./(fan_in+fan_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
118 high = numpy.sqrt(6./(fan_in+fan_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
119 size = da_filter_shape), dtype = theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
120 self.W_prime = theano.shared(value = initial_W_prime, name = "W_prime")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
121
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
122 #import pdb;pdb.set_trace()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
123
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
124 conv2_out = conv.conv2d(self.y, self.W_prime, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
125 filter_shape = da_filter_shape, image_shape = da_image_shape ,\
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
126 border_mode='full')
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
127
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
128 self.z = (T.tanh(conv2_out + self.b_prime.dimshuffle('x', 0, 'x', 'x'))+center) / scale
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
129
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
130 scaled_x = (self.x + center) / scale
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
131
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
132 self.L = - T.sum( scaled_x*T.log(self.z) + (1-scaled_x)*T.log(1-self.z), axis=1 )
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
133
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
134 self.cost = T.mean(self.L)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
135
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
136 self.params = [ self.W, self.b, self.b_prime ]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
137
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
139
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
140 class LeNetConvPoolLayer(object):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
141 def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2,2)):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
142 assert image_shape[1]==filter_shape[1]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
143 self.input = input
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
144
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
145 W_values = numpy.zeros(filter_shape, dtype=theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
146 self.W = theano.shared(value = W_values)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
147
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
148 b_values = numpy.zeros((filter_shape[0],), dtype= theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
149 self.b = theano.shared(value= b_values)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
150
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
151 conv_out = conv.conv2d(input, self.W,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
152 filter_shape=filter_shape, image_shape=image_shape)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
153
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
154
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
155 fan_in = numpy.prod(filter_shape[1:])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
156 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:]) / numpy.prod(poolsize)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
157
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
158 W_bound = numpy.sqrt(6./(fan_in + fan_out))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
159 self.W.value = numpy.asarray(
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
160 rng.uniform(low=-W_bound, high=W_bound, size=filter_shape),
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
161 dtype = theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
162
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
163
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
164 pooled_out = downsample.max_pool2D(conv_out, poolsize, ignore_border=True)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
165
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
166 self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
167 self.params = [self.W, self.b]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
168
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
169
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
170 class SdA():
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
171 def __init__(self, input, n_ins_conv, n_ins_mlp, train_set_x, train_set_y, batch_size, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
172 conv_hidden_layers_sizes, mlp_hidden_layers_sizes, corruption_levels, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
173 rng, n_out, pretrain_lr, finetune_lr):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
174
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
175 self.layers = []
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
176 self.pretrain_functions = []
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
177 self.params = []
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
178 self.conv_n_layers = len(conv_hidden_layers_sizes)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
179 self.mlp_n_layers = len(mlp_hidden_layers_sizes)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
180
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
181 index = T.lscalar() # index to a [mini]batch
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
182 self.x = T.dmatrix('x') # the data is presented as rasterized images
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
183 self.y = T.ivector('y') # the labels are presented as 1D vector of
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
184
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
185
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
186
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
187 for i in xrange( self.conv_n_layers ):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
188
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
189 filter_shape=conv_hidden_layers_sizes[i][0]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
190 image_shape=conv_hidden_layers_sizes[i][1]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
191 max_poolsize=conv_hidden_layers_sizes[i][2]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
192
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
193 if i == 0 :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
194 layer_input=self.x.reshape((batch_size,1,28,28))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
195 else:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
196 layer_input=self.layers[-1].output
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
197
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
198 layer = LeNetConvPoolLayer(rng, input=layer_input, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
199 image_shape=image_shape, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
200 filter_shape=filter_shape,poolsize=max_poolsize)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
201 print 'Convolutional layer '+str(i+1)+' created'
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
202
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
203 self.layers += [layer]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
204 self.params += layer.params
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
205
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
206 da_layer = dA_conv(corruption_level = corruption_levels[0],\
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
207 input = layer_input, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
208 shared_W = layer.W, shared_b = layer.b,\
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
209 filter_shape = filter_shape , image_shape = image_shape )
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
210
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
211
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
212 gparams = T.grad(da_layer.cost, da_layer.params)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
213
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
214 updates = {}
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
215 for param, gparam in zip(da_layer.params, gparams):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
216 updates[param] = param - gparam * pretrain_lr
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
217
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
218
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
219 update_fn = theano.function([index], da_layer.cost, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
220 updates = updates,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
221 givens = {
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
222 self.x : train_set_x[index*batch_size:(index+1)*batch_size]} )
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
223
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
224 self.pretrain_functions += [update_fn]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
225
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
226 for i in xrange( self.mlp_n_layers ):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
227 if i == 0 :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
228 input_size = n_ins_mlp
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
229 else:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
230 input_size = mlp_hidden_layers_sizes[i-1]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
231
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
232 if i == 0 :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
233 if len( self.layers ) == 0 :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
234 layer_input=self.x
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
235 else :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
236 layer_input = self.layers[-1].output.flatten(2)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
237 else:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
238 layer_input = self.layers[-1].output
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
239
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
240 layer = SigmoidalLayer(rng, layer_input, input_size,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
241 mlp_hidden_layers_sizes[i] )
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
242
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
243 self.layers += [layer]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
244 self.params += layer.params
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
245
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
246
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
247 print 'MLP layer '+str(i+1)+' created'
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
248
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
249 self.logLayer = LogisticRegression(input=self.layers[-1].output, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
250 n_in=mlp_hidden_layers_sizes[-1], n_out=n_out)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
251 self.params += self.logLayer.params
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
252
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
253 cost = self.logLayer.negative_log_likelihood(self.y)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
254
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
255 gparams = T.grad(cost, self.params)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
256 updates = {}
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
257
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
258 for param,gparam in zip(self.params, gparams):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
259 updates[param] = param - gparam*finetune_lr
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
260
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
261 self.finetune = theano.function([index], cost,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
262 updates = updates,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
263 givens = {
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
264 self.x : train_set_x[index*batch_size:(index+1)*batch_size],
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
265 self.y : train_set_y[index*batch_size:(index+1)*batch_size]} )
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
266
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
267
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
268 self.errors = self.logLayer.errors(self.y)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
269
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
270
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
271
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
272 def sgd_optimization_mnist( learning_rate=0.1, pretraining_epochs = 2, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
273 pretrain_lr = 0.01, training_epochs = 1000, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
274 dataset='mnist.pkl.gz'):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
275
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
276 f = gzip.open(dataset,'rb')
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
277 train_set, valid_set, test_set = cPickle.load(f)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
278 f.close()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
279
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
280
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
281 def shared_dataset(data_xy):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
282 data_x, data_y = data_xy
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
283 shared_x = theano.shared(numpy.asarray(data_x, dtype=theano.config.floatX))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
284 shared_y = theano.shared(numpy.asarray(data_y, dtype=theano.config.floatX))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
285 return shared_x, T.cast(shared_y, 'int32')
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
286
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
287
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
288 test_set_x, test_set_y = shared_dataset(test_set)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
289 valid_set_x, valid_set_y = shared_dataset(valid_set)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
290 train_set_x, train_set_y = shared_dataset(train_set)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
291
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
292 batch_size = 500 # size of the minibatch
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
293
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
294
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
295 n_train_batches = train_set_x.value.shape[0] / batch_size
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
296 n_valid_batches = valid_set_x.value.shape[0] / batch_size
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
297 n_test_batches = test_set_x.value.shape[0] / batch_size
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
298
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
299 # allocate symbolic variables for the data
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
300 index = T.lscalar() # index to a [mini]batch
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
301 x = T.matrix('x') # the data is presented as rasterized images
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
302 y = T.ivector('y') # the labels are presented as 1d vector of
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
303 # [int] labels
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
304 layer0_input = x.reshape((batch_size,1,28,28))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
305
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
306
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
307 # Setup the convolutional layers with their DAs(add as many as you want)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
308 corruption_levels = [ 0.2, 0.2, 0.2]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
309 rng = numpy.random.RandomState(1234)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
310 ker1=2
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
311 ker2=2
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
312 conv_layers=[]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
313 conv_layers.append([[ker1,1,5,5], [batch_size,1,28,28], [2,2] ])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
314 conv_layers.append([[ker2,ker1,5,5], [batch_size,ker1,12,12], [2,2] ])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
315
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
316 # Setup the MLP layers of the network
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
317 mlp_layers=[500]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
318
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
319 network = SdA(input = layer0_input, n_ins_conv = 28*28, n_ins_mlp = ker2*4*4, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
320 train_set_x = train_set_x, train_set_y = train_set_y, batch_size = batch_size,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
321 conv_hidden_layers_sizes = conv_layers, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
322 mlp_hidden_layers_sizes = mlp_layers, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
323 corruption_levels = corruption_levels , n_out = 10, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
324 rng = rng , pretrain_lr = pretrain_lr , finetune_lr = learning_rate )
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
325
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
326 test_model = theano.function([index], network.errors,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
327 givens = {
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
328 network.x: test_set_x[index*batch_size:(index+1)*batch_size],
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
329 network.y: test_set_y[index*batch_size:(index+1)*batch_size]})
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
330
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
331 validate_model = theano.function([index], network.errors,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
332 givens = {
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
333 network.x: valid_set_x[index*batch_size:(index+1)*batch_size],
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
334 network.y: valid_set_y[index*batch_size:(index+1)*batch_size]})
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
335
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
336
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
337
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
338 start_time = time.clock()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
339 for i in xrange(len(network.layers)-len(mlp_layers)):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
340 for epoch in xrange(pretraining_epochs):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
341 for batch_index in xrange(n_train_batches):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
342 c = network.pretrain_functions[i](batch_index)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
343 print 'pre-training convolution layer %i, epoch %d, cost '%(i,epoch),c
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
344
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
345 patience = 10000 # look as this many examples regardless
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
346 patience_increase = 2. # WAIT THIS MUCH LONGER WHEN A NEW BEST IS
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
347 # FOUND
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
348 improvement_threshold = 0.995 # a relative improvement of this much is
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
349
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
350 validation_frequency = min(n_train_batches, patience/2)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
351
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
352
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
353 best_params = None
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
354 best_validation_loss = float('inf')
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
355 test_score = 0.
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
356 start_time = time.clock()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
357
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
358 done_looping = False
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
359 epoch = 0
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
360
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
361 while (epoch < training_epochs) and (not done_looping):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
362 epoch = epoch + 1
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
363 for minibatch_index in xrange(n_train_batches):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
364
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
365 cost_ij = network.finetune(minibatch_index)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
366 iter = epoch * n_train_batches + minibatch_index
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
367
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
368 if (iter+1) % validation_frequency == 0:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
369
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
370 validation_losses = [validate_model(i) for i in xrange(n_valid_batches)]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
371 this_validation_loss = numpy.mean(validation_losses)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
372 print('epoch %i, minibatch %i/%i, validation error %f %%' % \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
373 (epoch, minibatch_index+1, n_train_batches, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
374 this_validation_loss*100.))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
375
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
376
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
377 # if we got the best validation score until now
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
378 if this_validation_loss < best_validation_loss:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
379
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
380 #improve patience if loss improvement is good enough
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
381 if this_validation_loss < best_validation_loss * \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
382 improvement_threshold :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
383 patience = max(patience, iter * patience_increase)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
384
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
385 # save best validation score and iteration number
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
386 best_validation_loss = this_validation_loss
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
387 best_iter = iter
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
388
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
389 # test it on the test set
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
390 test_losses = [test_model(i) for i in xrange(n_test_batches)]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
391 test_score = numpy.mean(test_losses)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
392 print((' epoch %i, minibatch %i/%i, test error of best '
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
393 'model %f %%') %
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
394 (epoch, minibatch_index+1, n_train_batches,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
395 test_score*100.))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
396
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
397
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
398 if patience <= iter :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
399 done_looping = True
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
400 break
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
401
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
402 end_time = time.clock()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
403 print(('Optimization complete with best validation score of %f %%,'
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
404 'with test performance %f %%') %
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
405 (best_validation_loss * 100., test_score*100.))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
406 print ('The code ran for %f minutes' % ((end_time-start_time)/60.))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
407
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
408
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
409
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
410
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
411
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
412
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
413 if __name__ == '__main__':
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
414 sgd_optimization_mnist()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
415