pylearn: algorithms/daa.py annotate

annotate algorithms/daa.py @ 496:f13847478c6d

A few more ideas, in comments

author	Joseph Turian <turian@gmail.com>
date	Tue, 28 Oct 2008 12:09:49 -0400
parents	e06666ac32d5
children	c7ce66b4e8f4

rev	line source
476 8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	1
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	2 import theano
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	3 from theano import tensor as T
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	4 from theano.tensor import nnet as NN
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	5 import numpy as N
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	6
487 94a4c5b7293b DAA code more generic: Joseph Turian <turian@gmail.com> parents: 486 diff changeset	7 from pylearn import cost as cost
94a4c5b7293b DAA code more generic: Joseph Turian <turian@gmail.com> parents: 486 diff changeset	8
476 8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	9 class DenoisingAA(T.RModule):
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	10
487 94a4c5b7293b DAA code more generic: Joseph Turian <turian@gmail.com> parents: 486 diff changeset	11 def __init__(self, input = None, regularize = True, tie_weights = True,
94a4c5b7293b DAA code more generic: Joseph Turian <turian@gmail.com> parents: 486 diff changeset	12 activation_function=NN.sigmoid, reconstruction_cost_function=cost.cross_entropy):
94a4c5b7293b DAA code more generic: Joseph Turian <turian@gmail.com> parents: 486 diff changeset	13 """
94a4c5b7293b DAA code more generic: Joseph Turian <turian@gmail.com> parents: 486 diff changeset	14 @param reconstruction_cost: Should return one cost per example (row)
488 e06666ac32d5 Added another todo Joseph Turian <turian@gmail.com> parents: 487 diff changeset	15 @todo: Default noise level for all daa levels
487 94a4c5b7293b DAA code more generic: Joseph Turian <turian@gmail.com> parents: 486 diff changeset	16 """
476 8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	17 super(DenoisingAA, self).__init__()
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	18
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	19 # MODEL CONFIGURATION
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	20 self.regularize = regularize
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	21 self.tie_weights = tie_weights
487 94a4c5b7293b DAA code more generic: Joseph Turian <turian@gmail.com> parents: 486 diff changeset	22 self.activation_function = activation_function
94a4c5b7293b DAA code more generic: Joseph Turian <turian@gmail.com> parents: 486 diff changeset	23 self.reconstruction_cost_function = reconstruction_cost_function
476 8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	24
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	25 # ACQUIRE/MAKE INPUT
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	26 if not input:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	27 input = T.matrix('input')
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	28 self.input = theano.External(input)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	29
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	30 # HYPER-PARAMETERS
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	31 self.lr = theano.Member(T.scalar())
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	32
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	33 # PARAMETERS
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	34 self.w1 = theano.Member(T.matrix())
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	35 if not tie_weights:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	36 self.w2 = theano.Member(T.matrix())
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	37 else:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	38 self.w2 = self.w1.T
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	39 self.b1 = theano.Member(T.vector())
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	40 self.b2 = theano.Member(T.vector())
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	41
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	42
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	43 # REGULARIZATION COST
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	44 self.regularization = self.build_regularization()
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	45
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	46
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	47 ### NOISELESS ###
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	48
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	49 # HIDDEN LAYER
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	50 self.hidden_activation = T.dot(self.input, self.w1) + self.b1
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	51 self.hidden = self.hid_activation_function(self.hidden_activation)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	52
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	53 # RECONSTRUCTION LAYER
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	54 self.output_activation = T.dot(self.hidden, self.w2) + self.b2
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	55 self.output = self.out_activation_function(self.output_activation)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	56
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	57 # RECONSTRUCTION COST
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	58 self.reconstruction_costs = self.build_reconstruction_costs(self.output)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	59 self.reconstruction_cost = T.mean(self.reconstruction_costs)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	60
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	61 # TOTAL COST
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	62 self.cost = self.reconstruction_cost
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	63 if self.regularize:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	64 self.cost = self.cost + self.regularization
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	65
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	66
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	67 ### WITH NOISE ###
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	68 self.corrupted_input = self.build_corrupted_input()
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	69
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	70 # HIDDEN LAYER
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	71 self.nhidden_activation = T.dot(self.corrupted_input, self.w1) + self.b1
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	72 self.nhidden = self.hid_activation_function(self.nhidden_activation)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	73
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	74 # RECONSTRUCTION LAYER
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	75 self.noutput_activation = T.dot(self.nhidden, self.w2) + self.b2
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	76 self.noutput = self.out_activation_function(self.noutput_activation)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	77
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	78 # RECONSTRUCTION COST
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	79 self.nreconstruction_costs = self.build_reconstruction_costs(self.noutput)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	80 self.nreconstruction_cost = T.mean(self.nreconstruction_costs)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	81
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	82 # TOTAL COST
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	83 self.ncost = self.nreconstruction_cost
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	84 if self.regularize:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	85 self.ncost = self.ncost + self.regularization
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	86
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	87
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	88 # GRADIENTS AND UPDATES
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	89 if self.tie_weights:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	90 self.params = self.w1, self.b1, self.b2
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	91 else:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	92 self.params = self.w1, self.w2, self.b1, self.b2
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	93 gradients = T.grad(self.ncost, self.params)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	94 updates = dict((p, p - self.lr * g) for p, g in zip(self.params, gradients))
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	95
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	96 # INTERFACE METHODS
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	97 self.update = theano.Method(self.input, self.ncost, updates)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	98 self.compute_cost = theano.Method(self.input, self.cost)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	99 self.noisify = theano.Method(self.input, self.corrupted_input)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	100 self.reconstruction = theano.Method(self.input, self.output)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	101 self.representation = theano.Method(self.input, self.hidden)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	102 self.reconstruction_through_noise = theano.Method(self.input, [self.corrupted_input, self.noutput])
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	103
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	104 def _instance_initialize(self, obj, input_size = None, hidden_size = None, seed = None, **init):
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	105 if (input_size is None) ^ (hidden_size is None):
486 5ccb1662f9f6 Small change to error message Joseph Turian <turian@gmail.com> parents: 485 diff changeset	106 raise ValueError("Must specify input_size and hidden_size or neither.")
476 8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	107 super(DenoisingAA, self)._instance_initialize(obj, **init)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	108 if seed is not None:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	109 R = N.random.RandomState(seed)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	110 else:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	111 R = N.random
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	112 if input_size is not None:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	113 sz = (input_size, hidden_size)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	114 inf = 1/N.sqrt(input_size)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	115 hif = 1/N.sqrt(hidden_size)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	116 obj.w1 = R.uniform(size = sz, low = -inf, high = inf)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	117 if not self.tie_weights:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	118 obj.w2 = R.uniform(size = list(reversed(sz)), low = -inf, high = inf)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	119 obj.b1 = N.zeros(hidden_size)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	120 obj.b2 = N.zeros(input_size)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	121 if seed is not None:
482 b15dad843c8c bugfix Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: 476 diff changeset	122 obj.seed(seed)
476 8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	123 obj.__hide__ = ['params']
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	124
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	125 def build_regularization(self):
487 94a4c5b7293b DAA code more generic: Joseph Turian <turian@gmail.com> parents: 486 diff changeset	126 """
94a4c5b7293b DAA code more generic: Joseph Turian <turian@gmail.com> parents: 486 diff changeset	127 @todo: Why do we need this function?
94a4c5b7293b DAA code more generic: Joseph Turian <turian@gmail.com> parents: 486 diff changeset	128 """
476 8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	129 return T.zero() # no regularization!
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	130
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	131
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	132 class SigmoidXEDenoisingAA(DenoisingAA):
487 94a4c5b7293b DAA code more generic: Joseph Turian <turian@gmail.com> parents: 486 diff changeset	133 """
94a4c5b7293b DAA code more generic: Joseph Turian <turian@gmail.com> parents: 486 diff changeset	134 @todo: Merge this into the above.
488 e06666ac32d5 Added another todo Joseph Turian <turian@gmail.com> parents: 487 diff changeset	135 @todo: Default noise level for all daa levels
487 94a4c5b7293b DAA code more generic: Joseph Turian <turian@gmail.com> parents: 486 diff changeset	136 """
476 8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	137
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	138 def build_corrupted_input(self):
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	139 self.noise_level = theano.Member(T.scalar())
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	140 return self.random.binomial(T.shape(self.input), 1, 1 - self.noise_level) * self.input
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	141
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	142 def hid_activation_function(self, activation):
487 94a4c5b7293b DAA code more generic: Joseph Turian <turian@gmail.com> parents: 486 diff changeset	143 return self.activation_function(activation)
476 8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	144
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	145 def out_activation_function(self, activation):
487 94a4c5b7293b DAA code more generic: Joseph Turian <turian@gmail.com> parents: 486 diff changeset	146 return self.activation_function(activation)
476 8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	147
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	148 def build_reconstruction_costs(self, output):
487 94a4c5b7293b DAA code more generic: Joseph Turian <turian@gmail.com> parents: 486 diff changeset	149 return self.reconstruction_cost_function(self.input, output)
476 8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	150
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	151 def build_regularization(self):
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	152 self.l2_coef = theano.Member(T.scalar())
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	153 if self.tie_weights:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	154 return self.l2_coef * T.sum(self.w1 * self.w1)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	155 else:
485 e8c37244b54f Small bugfix in regularization Joseph Turian <turian@gmail.com> parents: 482 diff changeset	156 return self.l2_coef * (T.sum(self.w1 * self.w1) + T.sum(self.w2 * self.w2))
476 8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	157
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	158 def _instance_initialize(self, obj, input_size = None, hidden_size = None, seed = None, **init):
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	159 init.setdefault('noise_level', 0)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	160 init.setdefault('l2_coef', 0)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	161 super(SigmoidXEDenoisingAA, self)._instance_initialize(obj, input_size, hidden_size, seed, **init)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	162

Mercurial > pylearn

annotate algorithms/daa.py @ 496:f13847478c6d