pylearn: algorithms/aa.py annotate

author	Olivier Breuleux <breuleuo@iro.umontreal.ca>
date	Mon, 27 Oct 2008 17:26:00 -0400
parents
children

rev	line source
476 8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	1
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	2 import theano
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	3 from theano import tensor as T
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	4 from theano.tensor import nnet as NN
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	5 import numpy as N
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	6
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	7 class AutoEncoder(theano.FancyModule):
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	8
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	9 def __init__(self, input = None, regularize = True, tie_weights = True):
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	10 super(AutoEncoder, self).__init__()
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	11
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	12 # MODEL CONFIGURATION
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	13 self.regularize = regularize
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	14 self.tie_weights = tie_weights
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	15
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	16 # ACQUIRE/MAKE INPUT
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	17 if not input:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	18 input = T.matrix('input')
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	19 self.input = theano.External(input)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	20
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	21 # HYPER-PARAMETERS
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	22 self.lr = theano.Member(T.scalar())
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	23
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	24 # PARAMETERS
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	25 self.w1 = theano.Member(T.matrix())
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	26 if not tie_weights:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	27 self.w2 = theano.Member(T.matrix())
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	28 else:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	29 self.w2 = self.w1.T
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	30 self.b1 = theano.Member(T.vector())
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	31 self.b2 = theano.Member(T.vector())
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	32
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	33 # HIDDEN LAYER
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	34 self.hidden_activation = T.dot(input, self.w1) + self.b1
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	35 self.hidden = self.build_hidden()
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	36
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	37 # RECONSTRUCTION LAYER
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	38 self.output_activation = T.dot(self.hidden, self.w2) + self.b2
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	39 self.output = self.build_output()
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	40
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	41 # RECONSTRUCTION COST
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	42 self.reconstruction_cost = self.build_reconstruction_cost()
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	43
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	44 # REGULARIZATION COST
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	45 self.regularization = self.build_regularization()
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	46
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	47 # TOTAL COST
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	48 self.cost = self.reconstruction_cost
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	49 if self.regularize:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	50 self.cost = self.cost + self.regularization
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	51
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	52 # GRADIENTS AND UPDATES
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	53 if self.tie_weights:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	54 self.params = self.w1, self.b1, self.b2
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	55 else:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	56 self.params = self.w1, self.w2, self.b1, self.b2
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	57 gradients = T.grad(self.cost, self.params)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	58 updates = dict((p, p - self.lr * g) for p, g in zip(self.params, gradients))
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	59
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	60 # INTERFACE METHODS
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	61 self.update = theano.Method(input, self.cost, updates)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	62 self.reconstruction = theano.Method(input, self.output)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	63 self.representation = theano.Method(input, self.hidden)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	64
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	65 def _instance_initialize(self, obj, input_size = None, hidden_size = None, seed = None, **init):
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	66 if (input_size is None) ^ (hidden_size is None):
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	67 raise ValueError("Must specify hidden_size and target_size or neither.")
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	68 super(AutoEncoder, self)._instance_initialize(obj, **init)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	69 if seed is not None:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	70 R = N.random.RandomState(seed)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	71 else:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	72 R = N.random
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	73 if input_size is not None:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	74 sz = (input_size, hidden_size)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	75 range = 1/N.sqrt(input_size)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	76 obj.w1 = R.uniform(size = sz, low = -range, high = range)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	77 if not self.tie_weights:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	78 obj.w2 = R.uniform(size = list(reversed(sz)), low = -range, high = range)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	79 obj.b1 = N.zeros(hidden_size)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	80 obj.b2 = N.zeros(input_size)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	81
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	82 def build_regularization(self):
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	83 return T.zero() # no regularization!
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	84
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	85
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	86 class SigmoidXEAutoEncoder(AutoEncoder):
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	87
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	88 def build_hidden(self):
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	89 return NN.sigmoid(self.hidden_activation)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	90
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	91 def build_output(self):
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	92 return NN.sigmoid(self.output_activation)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	93
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	94 def build_reconstruction_cost(self):
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	95 self.reconstruction_cost_matrix = self.input * T.log(self.output) + (1.0 - self.input) * T.log(1.0 - self.output)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	96 self.reconstruction_costs = -T.sum(self.reconstruction_cost_matrix, axis=1)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	97 return T.sum(self.reconstruction_costs)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	98
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	99 def build_regularization(self):
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	100 self.l2_coef = theano.Member(T.scalar())
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	101 if self.tie_weights:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	102 return self.l2_coef * T.sum(self.w1 * self.w1)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	103 else:
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	104 return self.l2_coef * T.sum(self.w1 * self.w1) + T.sum(self.w2 * self.w2)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	105
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	106 def _instance_initialize(self, obj, input_size = None, hidden_size = None, **init):
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	107 init.setdefault('l2_coef', 0)
8fcd0f3d9a17 added a few algorithms Olivier Breuleux <breuleuo@iro.umontreal.ca> parents: diff changeset	108 super(SigmoidXEAutoEncoder, self)._instance_initialize(obj, input_size, hidden_size, **init)

476

8fcd0f3d9a17 added a few algorithms