Mercurial > pylearn

changeset 705:eb91fa83e2c1

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
comments DAA_inputs_groups
author Xavier Glorot <glorotxa@iro.umontreal.ca>
date Thu, 21 May 2009 14:11:54 -0400
parents 521c04f8d2b3
children 2b54c38e2c60
files pylearn/algorithms/sandbox/DAA_inputs_groups.py
diffstat 1 files changed, 50 insertions(+), 32 deletions(-) [+]
line wrap: on
line diff
--- a/pylearn/algorithms/sandbox/DAA_inputs_groups.py	Thu May 21 12:54:41 2009 -0400
+++ b/pylearn/algorithms/sandbox/DAA_inputs_groups.py	Thu May 21 14:11:54 2009 -0400
@@ -198,7 +198,7 @@
 		if not hasattr(self,'params'):
 			self.params = []
 		self.params += [self.benc]
-		self.paramsenc = self.params
+		self.paramsenc = copy.copy(self.params)
 		if not(self.input is None):
 			self.params += [self.wenc] + [self.bdec]
 			self.paramsenc += [self.wenc]
@@ -321,6 +321,7 @@
 		print '\tin_size = ', in_size
 		print '\tauxin_size = ', auxin_size
 		print '\tn_hid = ', n_hid
+		
 		# save parameters
 		self.depth = depth
 		self.input = input
@@ -345,26 +346,33 @@
 		self.unsup_lr = T.dscalar('unsup_lr')
 		self.sup_lr = T.dscalar('sup_lr')
 		
-		# methods
+		# updatemethods
 		self.localupdate = [None] * (self.depth+1) #update only on the layer parameters
 		self.globalupdate = [None] * (self.depth+1)#update wrt the layer cost backproped untill the input layer
 		if self.totalupdatebool:
 			self.totalupdate = [None] * (self.depth+1) #update wrt all the layers cost backproped untill the input layer
 		#
+		
+		#others methods
 		self.representation = [None] * (self.depth+1)
 		self.reconstruction = [None] * (self.depth)
+		self.validate = [None] * (self.depth)
+		self.noisyinputs = [None] * (self.depth)
 		self.compute_localcost = [None] * (self.depth+1)
 		self.compute_globalcost = [None] * (self.depth+1)
 		if self.totalupdatebool:
 			self.compute_totalcost = [None] * (self.depth+1)
-		self.validate = [None] * (self.depth)
-		self.noisyinputs = [None] * (self.depth)
 		#
+		
+		# some theano Variables we want to keep track on
+		if self.regularize:
+			self.regularizationenccost = [None] * (self.depth)
 		self.localcost = [None] * (self.depth+1)
 		self.globalcost = [None] * (self.depth+1)
 		if self.totalupdatebool:
 			self.totalcost = [None] * (self.depth+1)
 		
+		#params to update and inputs initialization
 		paramstot = []
 		paramsenc = []
 		self.inputs = [None] * (self.depth+1)
@@ -376,6 +384,7 @@
 		
 		offset = 0
 		for i in range(self.depth):
+			
 			if auxin_size[i] is None:
 				offset +=1
 				param = [inputprec, None, in_sizeprec, auxin_size[i], self.n_hid[i],\
@@ -383,34 +392,45 @@
 			else:
 				param = [inputprec, self.auxinput[i-offset], in_sizeprec, auxin_size[i], self.n_hid[i],\
 					False, self.tie_weights, self.hid_fn, self.reconstruction_cost_function,False]
+			
 			print '\tLayer init= ', i+1
 			self.daaig[i] = DAAig(*param)
 			
+			# method input, outputs and parameters update
 			if i:
 				self.inputs[i] = copy.copy(self.inputs[i-1])
 			if not(auxin_size[i] is None):
 				self.inputs[i] += [self.daaig[i].idx_list,self.auxinput[i-offset]]
 			
+			noisyout = []
+			if not(inputprec is None):
+				noisyout += [self.daaig[i].noisy_input]
+			if not(auxin_size[i] is None):
+				noisyout += [self.daaig[i].noisy_auxinput]
+			
 			paramstot += self.daaig[i].params
 			
+			# save the costs
+			self.localcost[i] = self.daaig[i].noise.cost
+			self.globalcost[i] = self.daaig[i].noise.cost
+			if self.totalupdatebool:
+				if i:
+					self.totalcost[i] = self.totalcost[i-1] + self.daaig[i].noise.cost
+				else:
+					self.totalcost[i] = self.daaig[i].noise.cost
+			
 			if self.regularize:
-				self.localcost[i] = self.daaig[i].noise.cost+self.daaig[i].regularization
-				self.globalcost[i] = self.daaig[i].noise.cost+self.daaig[i].regularization
-				if self.totalupdatebool:
-					self.totalcost[i] = self.daaig[i].noise.cost+self.daaig[i].regularization
-				for j in range(i):
-					self.globalcost[i] += self.daaig[j].regularizationenc
-					if self.totalupdatebool:
-						self.totalcost[i] += self.daaig[j].noise.cost+self.daaig[j].regularization
+				if i:
+					self.regularizationenccost[i] = self.regularizationenccost[i-1]+self.daaig[i-1].regularizationenc
+				else:
+					self.regularizationenccost[i] = T.zero()
 				
-			else:
-				self.localcost[i] = self.daaig[i].noise.cost
-				self.globalcost[i] = self.daaig[i].noise.cost
+				self.localcost[i] += self.daaig[i].regularization
+				self.globalcost[i] += self.regularizationenccost[i]
 				if self.totalupdatebool:
-					self.totalcost[i] = self.daaig[i].noise.cost
-					for j in range(i):
-						self.totalcost[i] += self.daaig[j].noise.cost
+					self.totalcost[i] += self.daaig[i].regularization
 			
+			#create the updates dictionnaries
 			local_grads = dict((j, j - self.unsup_lr * T.grad(self.localcost[i], j))\
 					for j in self.daaig[i].params)
 			global_grads = dict((j, j - self.unsup_lr * T.grad(self.globalcost[i], j))\
@@ -419,6 +439,7 @@
 				total_grads = dict((j, j - self.unsup_lr * T.grad(self.totalcost[i], j))\
 						for j in (paramstot))
 			
+			# method declaration
 			self.localupdate[i] = theano.Method(self.inputs[i],self.localcost[i],local_grads)
 			self.globalupdate[i] = theano.Method(self.inputs[i],self.globalcost[i],global_grads)
 			if self.totalupdatebool:
@@ -426,21 +447,18 @@
 			#
 			self.representation[i] = theano.Method(self.inputs[i],self.daaig[i].clean.hidden)
 			self.reconstruction[i] = theano.Method(self.inputs[i],self.daaig[i].clean.rec)
+			self.validate[i] =theano.Method(self.inputs[i], [self.daaig[i].clean.cost, self.daaig[i].clean.rec])
+			self.noisyinputs[i] =theano.Method(self.inputs[i], noisyout)
 			self.compute_localcost[i] = theano.Method(self.inputs[i],self.localcost[i])
 			self.compute_globalcost[i] = theano.Method(self.inputs[i],self.globalcost[i])
 			if self.totalupdatebool:
 				self.compute_totalcost[i] = theano.Method(self.inputs[i],self.totalcost[i])
-			self.validate[i] =theano.Method(self.inputs[i], [self.daaig[i].clean.cost, self.daaig[i].clean.rec])
-			noisyout = []
-			if not(inputprec is None):
-				noisyout += [self.daaig[i].noisy_input]
-			if not(auxin_size[i] is None):
-				noisyout += [self.daaig[i].noisy_auxinput]
-			self.noisyinputs[i] =theano.Method(self.inputs[i], noisyout)
 			#
 			paramsenc += self.daaig[i].paramsenc
 			inputprec = self.daaig[i].clean.hidden
 			in_sizeprec = self.n_hid[i]
+		
+		# supervised layer
 		print '\tLayer supervised init'
 		self.inputs[-1] = copy.copy(self.inputs[-2])+[self.target]
 		self.daaig[-1] = LogRegN(in_sizeprec,self.n_out,inputprec,self.target)
@@ -448,15 +466,14 @@
 		
 		if self.regularize:
 			self.localcost[-1] = self.daaig[-1].regularized_cost
-			self.globalcost[-1] = self.daaig[-1].regularized_cost
-			for j in range(self.depth):
-				self.globalcost[-1] += self.daaig[j].regularizationenc
+			self.globalcost[-1] = self.daaig[-1].regularized_cost + self.regularizationenccost[-1]
+			if self.totalupdatebool:
+				self.totalcost[-1] = self.totalcost[-2] + self.daaig[-1].regularized_cost
 		else:
 			self.localcost[-1] = self.daaig[-1].unregularized_cost
 			self.globalcost[-1] = self.daaig[-1].unregularized_cost
-		
-		if self.totalupdatebool:
-			self.totalcost[-1] = self.totalcost[-2] + self.localcost[-1]
+			if self.totalupdatebool:
+				self.totalcost[-1] = self.totalcost[-2] + self.daaig[-1].unregularized_cost
 		
 		local_grads = dict((j, j - self.sup_lr * T.grad(self.localcost[-1], j))\
 					for j in self.daaig[-1].params)
@@ -471,11 +488,12 @@
 		self.globalupdate[-1] = theano.Method(self.inputs[-1],self.globalcost[-1],global_grads)
 		if self.totalupdatebool:
 			self.totalupdate[-1] = theano.Method(self.inputs[-1],self.totalcost[-1],total_grads)
+
+		self.representation[-1] = theano.Method(self.inputs[-2],self.daaig[-1].argmax_standalone)
 		self.compute_localcost[-1] = theano.Method(self.inputs[-1],self.localcost[-1])
 		self.compute_globalcost[-1] = theano.Method(self.inputs[-1],self.globalcost[-1])
 		if self.totalupdatebool:
 			self.compute_totalcost[-1] = theano.Method(self.inputs[-1],self.totalcost[-1])
-		self.representation[-1] = theano.Method(self.inputs[-2],self.daaig[-1].argmax_standalone)
 	
 	def _instance_initialize(self,inst,unsup_lr = 0.1, sup_lr = 0.01, reg_coef = 0,
 								noise_level = 0 , noise_level_group = 0, seed = 1, Alloc = True,**init):