Mercurial > ift6266

comparison deep/convolutional_dae/stacked_convolutional_dae.py @ 259:3919c71e3091

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
author Arnaud Bergeron <abergeron@gmail.com>
date Wed, 17 Mar 2010 15:24:25 -0400
parents 7e6fecabb656
children 0c0f0b3f6a93
comparison
equal deleted inserted replaced
258:c2fae7b96769 259:3919c71e3091
2 import theano 2 import theano
3 import time 3 import time
4 import sys 4 import sys
5 import theano.tensor as T 5 import theano.tensor as T
6 from theano.tensor.shared_randomstreams import RandomStreams 6 from theano.tensor.shared_randomstreams import RandomStreams
7 import theano.sandbox.softsign 7 #import theano.sandbox.softsign
8 8
9 from theano.tensor.signal import downsample 9 from theano.tensor.signal import downsample
10 from theano.tensor.nnet import conv 10 from theano.tensor.nnet import conv
11 11
12 sys.path.append('../../../')
13
14 from ift6266 import datasets 12 from ift6266 import datasets
15 from ift6266.baseline.log_reg.log_reg import LogisticRegression 13 from ift6266.baseline.log_reg.log_reg import LogisticRegression
16 14
17 batch_size = 100 15 batch_size = 100
18
19 16
20 class SigmoidalLayer(object): 17 class SigmoidalLayer(object):
21 def __init__(self, rng, input, n_in, n_out): 18 def __init__(self, rng, input, n_in, n_out):
22 19
23 self.input = input 20 self.input = input
68 self.x = input 65 self.x = input
69 66
70 self.tilde_x = theano_rng.binomial( self.x.shape, 1, 1 - corruption_level,dtype=theano.config.floatX) * self.x 67 self.tilde_x = theano_rng.binomial( self.x.shape, 1, 1 - corruption_level,dtype=theano.config.floatX) * self.x
71 68
72 conv1_out = conv.conv2d(self.tilde_x, self.W, filter_shape=filter_shape, 69 conv1_out = conv.conv2d(self.tilde_x, self.W, filter_shape=filter_shape,
73 image_shape=image_shape, unroll_kern=4,unroll_batch=4, border_mode='valid') 70 image_shape=image_shape, border_mode='valid')
74
75 71
76 self.y = T.tanh(conv1_out + self.b.dimshuffle('x', 0, 'x', 'x')) 72 self.y = T.tanh(conv1_out + self.b.dimshuffle('x', 0, 'x', 'x'))
77 73
78 74 da_filter_shape = [ filter_shape[1], filter_shape[0],
79 da_filter_shape = [ filter_shape[1], filter_shape[0], filter_shape[2],\ 75 filter_shape[2], filter_shape[3] ]
80 filter_shape[3] ]
81 da_image_shape = [ image_shape[0], filter_shape[0], image_shape[2]-filter_shape[2]+1, image_shape[3]-filter_shape[3]+1 ]
82 initial_W_prime = numpy.asarray( numpy.random.uniform( \ 76 initial_W_prime = numpy.asarray( numpy.random.uniform( \
83 low = -numpy.sqrt(6./(fan_in+fan_out)), \ 77 low = -numpy.sqrt(6./(fan_in+fan_out)), \
84 high = numpy.sqrt(6./(fan_in+fan_out)), \ 78 high = numpy.sqrt(6./(fan_in+fan_out)), \
85 size = da_filter_shape), dtype = theano.config.floatX) 79 size = da_filter_shape), dtype = theano.config.floatX)
86 self.W_prime = theano.shared(value = initial_W_prime, name = "W_prime") 80 self.W_prime = theano.shared(value = initial_W_prime, name = "W_prime")
87 81
88 conv2_out = conv.conv2d(self.y, self.W_prime, 82 conv2_out = conv.conv2d(self.y, self.W_prime,
89 filter_shape = da_filter_shape,\ 83 filter_shape = da_filter_shape,
90 image_shape = da_image_shape, \
91 unroll_kern=4,unroll_batch=4, \
92 border_mode='full') 84 border_mode='full')
93 85
94 self.z = (T.tanh(conv2_out + self.b_prime.dimshuffle('x', 0, 'x', 'x'))+center) / scale 86 self.z = (T.tanh(conv2_out + self.b_prime.dimshuffle('x', 0, 'x', 'x'))+center) / scale
95 87
96 scaled_x = (self.x + center) / scale 88 scaled_x = (self.x + center) / scale
111 103
112 b_values = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX) 104 b_values = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
113 self.b = theano.shared(value=b_values) 105 self.b = theano.shared(value=b_values)
114 106
115 conv_out = conv.conv2d(input, self.W, 107 conv_out = conv.conv2d(input, self.W,
116 filter_shape=filter_shape, image_shape=image_shape, unroll_kern=4,unroll_batch=4) 108 filter_shape=filter_shape, image_shape=image_shape)
117 109
118 110
119 fan_in = numpy.prod(filter_shape[1:]) 111 fan_in = numpy.prod(filter_shape[1:])
120 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:]) / numpy.prod(poolsize) 112 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:]) / numpy.prod(poolsize)
121 113
132 124
133 125
134 class SdA(): 126 class SdA():
135 def __init__(self, input, n_ins_mlp, conv_hidden_layers_sizes, 127 def __init__(self, input, n_ins_mlp, conv_hidden_layers_sizes,
136 mlp_hidden_layers_sizes, corruption_levels, rng, n_out, 128 mlp_hidden_layers_sizes, corruption_levels, rng, n_out,
137 pretrain_lr, finetune_lr): 129 pretrain_lr, finetune_lr, img_shape):
138 130
139 self.layers = [] 131 self.layers = []
140 self.pretrain_functions = [] 132 self.pretrain_functions = []
141 self.params = [] 133 self.params = []
142 self.conv_n_layers = len(conv_hidden_layers_sizes) 134 self.conv_n_layers = len(conv_hidden_layers_sizes)
149 filter_shape=conv_hidden_layers_sizes[i][0] 141 filter_shape=conv_hidden_layers_sizes[i][0]
150 image_shape=conv_hidden_layers_sizes[i][1] 142 image_shape=conv_hidden_layers_sizes[i][1]
151 max_poolsize=conv_hidden_layers_sizes[i][2] 143 max_poolsize=conv_hidden_layers_sizes[i][2]
152 144
153 if i == 0 : 145 if i == 0 :
154 layer_input=self.x.reshape((self.x.shape[0], 1, 32, 32)) 146 layer_input=self.x.reshape((self.x.shape[0], 1) + img_shape)
155 else: 147 else:
156 layer_input=self.layers[-1].output 148 layer_input=self.layers[-1].output
157 149
158 layer = LeNetConvPoolLayer(rng, input=layer_input, 150 layer = LeNetConvPoolLayer(rng, input=layer_input,
159 image_shape=image_shape, 151 image_shape=image_shape,
216 208
217 self.finetune = theano.function([self.x, self.y], cost, updates = updates) 209 self.finetune = theano.function([self.x, self.y], cost, updates = updates)
218 210
219 self.errors = self.logLayer.errors(self.y) 211 self.errors = self.logLayer.errors(self.y)
220 212
221 def sgd_optimization_mnist( learning_rate=0.1, pretraining_epochs = 1, \ 213 def sgd_optimization_mnist(learning_rate=0.1, pretraining_epochs = 1,
222 pretrain_lr = 0.1, training_epochs = 1000, \ 214 pretrain_lr = 0.1, training_epochs = 1000,
223 kernels = [ [4,5,5] , [4,3,3] ], mlp_layers=[500], \ 215 kernels = [[4,5,5], [4,3,3]], mlp_layers=[500],
224 corruption_levels = [ 0.2, 0.2, 0.2], batch_size = batch_size, \ 216 corruption_levels = [0.2, 0.2, 0.2],
225 max_pool_layers = [ [2,2] , [2,2] ], \ 217 batch_size = batch_size, img_shape=(28, 28),
226 dataset=datasets.nist_digits): 218 max_pool_layers = [[2,2], [2,2]],
227 219 dataset=datasets.mnist(5000)):
228 220
229 # allocate symbolic variables for the data 221 # allocate symbolic variables for the data
230 index = T.lscalar() # index to a [mini]batch 222 index = T.lscalar() # index to a [mini]batch
231 x = T.matrix('x') # the data is presented as rasterized images 223 x = T.matrix('x') # the data is presented as rasterized images
232 y = T.ivector('y') # the labels are presented as 1d vector of 224 y = T.ivector('y') # the labels are presented as 1d vector of
233 # [int] labels 225 # [int] labels
234 226
235 layer0_input = x.reshape((x.shape[0],1,32,32)) 227 layer0_input = x.reshape((x.shape[0],1)+img_shape)
236 228
237 rng = numpy.random.RandomState(1234) 229 rng = numpy.random.RandomState(1234)
238 conv_layers=[] 230 conv_layers=[]
239 init_layer = [ [ kernels[0][0],1,kernels[0][1],kernels[0][2] ],\ 231 init_layer = [[kernels[0][0],1,kernels[0][1],kernels[0][2]],
240 [ batch_size , 1, 32, 32 ], 232 None, # do not specify the batch size since it can
241 max_pool_layers[0] ] 233 # change for the last one and then theano will
234 # crash.
235 max_pool_layers[0]]
242 conv_layers.append(init_layer) 236 conv_layers.append(init_layer)
243 237
244 conv_n_out = (32-kernels[0][2]+1)/max_pool_layers[0][0] 238 conv_n_out = (img_shape[0]-kernels[0][2]+1)/max_pool_layers[0][0]
245 239
246 for i in range(1,len(kernels)): 240 for i in range(1,len(kernels)):
247 layer = [ [ kernels[i][0],kernels[i-1][0],kernels[i][1],kernels[i][2] ],\ 241 layer = [[kernels[i][0],kernels[i-1][0],kernels[i][1],kernels[i][2]],
248 [ batch_size, kernels[i-1][0], conv_n_out,conv_n_out ], 242 None, # same comment as for init_layer
249 max_pool_layers[i] ] 243 max_pool_layers[i] ]
250 conv_layers.append(layer) 244 conv_layers.append(layer)
251 conv_n_out = (conv_n_out - kernels[i][2]+1)/max_pool_layers[i][0] 245 conv_n_out = (conv_n_out - kernels[i][2]+1)/max_pool_layers[i][0]
252 print layer [1] 246
253 network = SdA(input = layer0_input, n_ins_mlp = kernels[-1][0]*conv_n_out**2, 247 network = SdA(input = layer0_input, n_ins_mlp = kernels[-1][0]*conv_n_out**2,
254 conv_hidden_layers_sizes = conv_layers, 248 conv_hidden_layers_sizes = conv_layers,
255 mlp_hidden_layers_sizes = mlp_layers, 249 mlp_hidden_layers_sizes = mlp_layers,
256 corruption_levels = corruption_levels , n_out = 62, 250 corruption_levels = corruption_levels, n_out = 62,
257 rng = rng , pretrain_lr = pretrain_lr , 251 rng = rng , pretrain_lr = pretrain_lr,
258 finetune_lr = learning_rate ) 252 finetune_lr = learning_rate, img_shape=img_shape)
259 253
260 test_model = theano.function([network.x, network.y], network.errors) 254 test_model = theano.function([network.x, network.y], network.errors)
261 255
262 start_time = time.clock() 256 start_time = time.clock()
263 for i in xrange(len(network.layers)-len(mlp_layers)): 257 for i in xrange(len(network.layers)-len(mlp_layers)):