ift6266: deep/convolutional_dae/scdae.py annotate

author	Arnaud Bergeron <abergeron@gmail.com>
date	Mon, 22 Mar 2010 13:33:29 -0400
parents
children	20ebc1f2a9fe

rev	line source
276 727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	1 from pynnet import *
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	2 # use hacks also
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	3 from pynnet.utils import *
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	4
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	5 import numpy
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	6 import theano
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	7 import theano.tensor as T
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	8
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	9 from itertools import izip
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	10
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	11 class cdae(LayerStack):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	12 def __init__(self, filter_size, num_filt, num_in, subsampling, corruption,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	13 dtype, img_shape):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	14 LayerStack.__init__(self, [ConvAutoencoder(filter_size=filter_size,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	15 num_filt=num_filt,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	16 num_in=num_in,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	17 noisyness=corruption,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	18 dtype=dtype,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	19 image_shape=img_shape),
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	20 MaxPoolLayer(subsampling)])
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	21
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	22 def build(self, input):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	23 LayerStack.build(self, input)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	24 self.cost = self.layers[0].cost
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	25
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	26 def cdae_out_size(in_size, filt_size, num_filt, num_in, subs):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	27 out = [None] * 3
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	28 out[0] = num_filt
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	29 out[1] = (in_size[1]-filt_size[0]+1)/subs[0]
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	30 out[2] = (in_size[2]-filt_size[1]+1)/subs[1]
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	31 return out
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	32
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	33 def scdae(in_size, num_in, filter_sizes, num_filts,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	34 subsamplings, corruptions, dtype):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	35 layers = []
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	36 old_nfilt = 1
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	37 for fsize, nfilt, subs, corr in izip(filter_sizes, num_filts,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	38 subsamplings, corruptions):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	39 layers.append(cdae(fsize, nfilt, old_nfilt, subs, corr, dtype,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	40 (num_in, in_size[0], in_size[1], in_size[2])))
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	41 in_size = cdae_out_size(in_size, fsize, nfilt, old_nfilt, subs)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	42 old_nfilt = nfilt
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	43 return LayerStack(layers), in_size
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	44
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	45 def mlp(layer_sizes, dtype):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	46 layers = []
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	47 old_size = layer_sizes[0]
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	48 for size in layer_sizes[1:]:
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	49 layers.append(SimpleLayer(old_size, size, activation=nlins.tanh,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	50 dtype=dtype))
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	51 old_size = size
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	52 return LayerStack(layers)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	53
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	54 def scdae_net(in_size, num_in, filter_sizes, num_filts, subsamplings,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	55 corruptions, layer_sizes, out_size, dtype, batch_size):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	56 rl1 = ReshapeLayer((None,)+in_size)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	57 ls, outs = scdae(in_size, num_in, filter_sizes, num_filts, subsamplings,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	58 corruptions, dtype)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	59 outs = numpy.prod(outs)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	60 rl2 = ReshapeLayer((None, outs))
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	61 layer_sizes = [outs]+layer_sizes
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	62 ls2 = mlp(layer_sizes, dtype)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	63 lrl = SimpleLayer(layer_sizes[-1], out_size, activation=nlins.sigmoid)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	64 return NNet([rl1, ls, rl2, ls2, lrl], error=errors.nll)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	65
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	66 def build_funcs(batch_size, img_size, filter_sizes, num_filters, subs,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	67 noise, mlp_sizes, out_size, dtype, pretrain_lr, train_lr):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	68
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	69 n = scdae_net((1,)+img_size, batch_size, filter_sizes, num_filters, subs,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	70 noise, mlp_sizes, out_size, dtype, batch_size)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	71 x = T.fmatrix('x')
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	72 y = T.ivector('y')
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	73
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	74 def pretrainfunc(net, alpha):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	75 up = trainers.get_updates(net.params, net.cost, alpha)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	76 return theano.function([x], net.cost, updates=up)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	77
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	78 def trainfunc(net, alpha):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	79 up = trainers.get_updates(net.params, net.cost, alpha)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	80 return theano.function([x, y], net.cost, updates=up)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	81
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	82 n.build(x, y)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	83 pretrain_funcs_opt = [pretrainfunc(l, pretrain_lr) for l in n.layers[1].layers]
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	84 trainf_opt = trainfunc(n, train_lr)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	85 evalf_opt = theano.function([x, y], errors.class_error(n.output, y))
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	86
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	87 clear_imgshape(n)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	88 n.build(x, y)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	89 pretrain_funcs_reg = [pretrainfunc(l, 0.01) for l in n.layers[1].layers]
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	90 trainf_reg = trainfunc(n, 0.1)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	91 evalf_reg = theano.function([x, y], errors.class_error(n.output, y))
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	92
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	93 def select_f(f1, f2, bsize):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	94 def f(x):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	95 if x.shape[0] == bsize:
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	96 return f1(x)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	97 else:
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	98 return f2(x)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	99 return f
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	100
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	101 pretrain_funcs = [select_f(p_opt, p_reg, batch_size) for p_opt, p_reg in zip(pretrain_funcs_opt, pretrain_funcs_reg)]
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	102
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	103 def select_f2(f1, f2, bsize):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	104 def f(x, y):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	105 if x.shape[0] == bsize:
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	106 return f1(x, y)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	107 else:
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	108 return f2(x, y)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	109 return f
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	110
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	111 trainf = select_f2(trainf_opt, trainf_reg, batch_size)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	112 evalf = select_f2(evalf_opt, evalf_reg, batch_size)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	113 return pretrain_funcs, trainf, evalf
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	114
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	115 def do_pretrain(pretrain_funcs, pretrain_epochs):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	116 for f in pretrain_funcs:
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	117 for i in xrange(pretrain_epochs):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	118 f()
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	119
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	120 def massage_funcs(batch_size, dset, pretrain_funcs, trainf, evalf):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	121 def pretrain_f(f):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	122 def res():
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	123 for x, y in dset.train(batch_size):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	124 print "pretrain:", f(x)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	125 return res
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	126
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	127 pretrain_fs = map(pretrain_f, pretrain_funcs)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	128
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	129 def train_f(f, dsetf):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	130 def dset_it():
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	131 while True:
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	132 for x, y in dsetf(batch_size):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	133 yield f(x, y)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	134 it = dset_it()
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	135 return lambda: it.next()
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	136
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	137 train = train_f(trainf, dset.train)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	138
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	139 def eval_f(f, dsetf):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	140 def res():
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	141 c = 0
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	142 i = 0
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	143 for x, y in dsetf(batch_size):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	144 i += x.shape[0]
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	145 c += f(x, y)*x.shape[0]
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	146 return c/i
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	147 return res
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	148
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	149 test = eval_f(evalf, dset.test)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	150 valid = eval_f(evalf, dset.valid)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	151
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	152 return pretrain_fs, train, valid, test
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	153
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	154 def run_exp(state, channel):
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	155 from ift6266 import datasets
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	156 from sgd_opt import sgd_opt
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	157 import sys, time
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	158
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	159 channel.save()
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	160
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	161 # params: bsize, pretrain_lr, train_lr, nfilts1, nfilts2, nftils3, nfilts4
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	162 # pretrain_rounds
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	163
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	164 dset = dataset.nist_all()
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	165
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	166 nfilts = []
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	167 if state.nfilts1 != 0:
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	168 nfilts.append(state.nfilts1)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	169 if state.nfilts2 != 0:
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	170 nfilts.append(state.nfilts2)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	171 if state.nfilts3 != 0:
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	172 nfilts.append(state.nfilts3)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	173 if state.nfilts4 != 0:
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	174 nfilts.append(state.nfilts4)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	175
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	176 fsizes = [(5,5)]*len(nfilts)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	177 subs = [(2,2)]*len(nfilts)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	178 noise = [state.noise]*len(nfilts)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	179
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	180 pretrain_funcs, trainf, evalf = build_funcs(
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	181 img_size=(32, 32),
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	182 batch_size=state.bsize,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	183 filter_sizes=fsizes,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	184 num_filters=nfilts,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	185 subs=subs,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	186 noise=noise,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	187 mlp_sizes=[state.mlp_sz],
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	188 out_size=62,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	189 dtype=numpy.float32,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	190 pretrain_lr=state.pretrain_lr,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	191 train_lr=state.train_lr)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	192
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	193 pretrain_fs, train, valid, test = massage_funcs(
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	194 state.bsize, dset, pretrain_funcs, trainf, evalf)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	195
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	196 do_pretrain(pretrain_fs, state.pretrain_rounds)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	197
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	198 sgd_opt(train, valid, test, training_epochs=100000, patience=10000,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	199 patience_increase=2., improvement_threshold=0.995,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	200 validation_frequency=2500)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	201
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	202 if __name__ == '__main__':
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	203 from ift6266 import datasets
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	204 from sgd_opt import sgd_opt
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	205 import sys, time
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	206
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	207 batch_size = 100
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	208 dset = datasets.mnist(200)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	209
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	210 pretrain_funcs, trainf, evalf = build_funcs(
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	211 img_size = (28, 28),
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	212 batch_size=batch_size, filter_sizes=[(5,5), (5,5)],
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	213 num_filters=[4, 3], subs=[(2,2), (2,2)], noise=[0.2, 0.2],
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	214 mlp_sizes=[500], out_size=10, dtype=numpy.float32,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	215 pretrain_lr=0.01, train_lr=0.1)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	216
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	217 pretrain_fs, train, valid, test = massage_funcs(
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	218 batch_size, dset, pretrain_funcs, trainf, evalf)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	219
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	220 print "pretraining ...",
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	221 sys.stdout.flush()
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	222 start = time.time()
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	223 do_pretrain(pretrain_fs, 0)
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	224 end = time.time()
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	225 print "done (in", end-start, "s)"
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	226
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	227 sgd_opt(train, valid, test, training_epochs=1000, patience=1000,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	228 patience_increase=2., improvement_threshold=0.995,
727ed56fad12 Add reworked code for convolutional auto-encoder. Arnaud Bergeron <abergeron@gmail.com> parents: diff changeset	229 validation_frequency=500)

276

727ed56fad12 Add reworked code for convolutional auto-encoder.