--- a/pylearn/sandbox/InputGroups/ScanInputsGroups.py	Tue May 12 14:35:29 2009 -0400
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,502 +0,0 @@
-import numpy
-import theano
-from theano import tensor as T
-from theano.gof import Op
-from theano.gof import Apply
-from theano import scalar as scal
-
-# These Ops allows us to deal with static groups of possibly missing inputs efficiently in the dense DAA framework
-# (for exemple with multimodal data with sometimes entire modality missing).
-# The inputs will be represented with an index list and a theano.generic variable (which will be a list of matrices
-# (numpy array), each element will correspond to an available modality and the index list will indicates the weights
-# associated to it).
-# Exemple of index list: [1, 0, -3]
-#	*the 1 says that the first element of the input list will refer to the first element of the weights_list
-#		(auxiliary target as input)
-#								if inputslist[i]>0 it refers to Weightslist[indexlist[i]-1]
-#	*the 0 means that the second element of the input list will not be encoded neither decoded (it is remplaced by zeros)
-#		this is not efficient, so in this case it is better to give: [1,-3] and [inputslist[0],inputslist[2]]
-#		but it allows us to deal with empty lists: give indexlist = [.0] and inputlist=[[.0]]
-#	*when an index is negative it means that the input will not be used for encoding but we will still reconstruct it
-#		(auxiliary target as output)
-#								if inputslist[i]<0 it refers to Weightslist[-indexlist[i]-1]
-#
-# An entire batch should have the same available inputs configuration.
-#
-# Dense DAA Exemple:----------------------------------------------------------------------------
-#
-#from theano.tensor.nnet import sigmoid
-#
-#nb_modality = 4
-#wenc = [T.dmatrix('wenc%s'%i) for i in range(nb_modality)]
-#wdec = [T.dmatrix('wdec%s'%i) for i in range(nb_modality)]
-#benc = T.dvector('benc')
-#bdec = [T.dvector('bdec%s'%i) for i in range(nb_modality)]
-#vectin = T.ivector('vectin')
-#inputpart = theano.generic('inputpart')
-#noise_bit = T.dscalar('noise_bit')
-#noise_group = T.dscalar('noise_group')
-#
-#[vectin2,inputpart2] = scannoise(vectin,inputpart,noise_bit,noise_group)
-#hid = scandotenc(vectin2, inputpart2, wenc)
-#acthid = sigmoid(hid + benc)
-#dec = sigmoid(scanbiasdec(vectin2,inputpart2,bdec) + scandotdec(vectin2, inputpart2,acthid,wdec))
-#cost = T.sum(T.sum(T.sqr( scaninput(vectin,inputpart) - rec ),1),0)
-
-# Checking inputs in make_node methods----------------------
-def Checkidx_list(idx_list):
-	idx_list = T.as_tensor_variable(idx_list)
-	nidx = idx_list.type.ndim
-	if nidx != 1: raise TypeError('not vector', idx_list)
-	return idx_list
-
-def Checkhidd(hidd):
-	hidd = T.as_tensor_variable(hidd)
-	nhidd = hidd.type.ndim
-	if nhidd not in (1,2): raise TypeError('not matrix or vector', hidd)
-	return hidd
-
-def Checkweights_list(weights_list):
-	weights_list = map(T.as_tensor_variable, weights_list)
-	for i in range(len(weights_list)):
-		nweights = weights_list[i].type.ndim
-		if nweights not in (1,2): raise TypeError('not matrix or vector', weights_list[i])
-	return weights_list
-
-def Checkbias_list(bias_list):
-	bias_list = map(T.as_tensor_variable, bias_list)
-	for i in range(len(bias_list)):
-		nbias = bias_list[i].type.ndim
-		if nbias != 1: raise TypeError('not vector', bias_list[i])
-	return bias_list
-
-# Encoding scan dot product------------------------------------
-class ScanDotEnc(Op):
-	"""This Op takes an index list (as tensor.ivector), a list of matrices representing
-	the available inputs (as theano.generic), and all the encoding weights tensor.dmatrix of the model. It will select the
-	weights corresponding to the inputs (according to index list) and compute only the necessary dot products"""
-	def __init__(self):
-		#Create Theano methods to do the dot products with blas or at least in C.
-		self.M=theano.Module()
-		inputs = T.dmatrix('input')
-		weights = T.dmatrix('weights')
-		self.M.hid = T.dmatrix('hid')
-		self.M.resultin = self.M.hid + T.dot(inputs,weights)
-		result = T.dot(inputs,weights)
-		
-		self.M.dotin = theano.Method([inputs,weights],None,{self.M.hid : self.M.resultin})
-		self.M.dot = theano.Method([inputs,weights],result)
-		self.m = self.M.make()
-	
-	def make_node(self, idx_list, inputs_list, weights_list):
-		idx_list = Checkidx_list(idx_list)
-		weights_list = Checkweights_list(weights_list)
-		return Apply(self, [idx_list] + [inputs_list] + weights_list, [T.dmatrix()])
-	
-	def perform(self, node, args, (hid,)):
-		idx_list = args[0]
-		hidcalc = False
-		
-		batchsize = (args[1][0].shape)[0]
-		n_hid = (args[2].shape)[1]
-		if len(idx_list) != len(args[1]) :
-			raise NotImplementedError('size of index different of inputs list size',idx_list)
-		if max(idx_list) >= (len(args)-2)+1 :
-			raise NotImplementedError('index superior to weight list length',idx_list)
-		for i in range(len(args[1])):
-			if (args[1][i].shape)[0] != batchsize:
-				raise NotImplementedError('different batchsize in the inputs list',args[1][i])
-		for i in range(len(args)-2):
-			if (args[2+i].shape)[1] != n_hid:
-				raise NotImplementedError('different length of hidden in the weights list',args[2+i])
-		
-		for i in range(len(idx_list)):
-			if idx_list[i]>0:
-				if hidcalc:
-					self.m.dotin(args[1][i],args[2+int(idx_list[i]-1)])
-				else:
-					self.m.hid = self.m.dot(args[1][i],args[2+int(idx_list[i]-1)])
-					hidcalc = True
-		
-		if not hidcalc:
-			hid[0] = numpy.zeros([batchsize,n_hid])
-		else:
-			hid[0] = self.m.hid
-		
-	
-	def grad(self, args, gz):
-		return [None, None] + ScanDotEncGrad()(args,gz)
-	
-	def __hash__(self):
-		return hash(ScanDotEnc)^58994
-	
-	def __str__(self):
-		return "ScanDotEnc"
-
-scandotenc=ScanDotEnc()
-
-class ScanDotEncGrad(Op):
-	"""This Op computes the gradient wrt the weights for ScanDotEnc"""
-	def __init__(self):
-		#Create Theano methods to do the dot products with blas or at least in C.
-		self.M=theano.Module()
-		input1 = T.dmatrix('input1')
-		self.M.g_out = T.dmatrix('g_out')
-		result = T.dmatrix('result')
-		input2=T.transpose(input1)
-		self.M.resultin = result + T.dot(input2,self.M.g_out)
-		self.M.result = T.dot(input2,self.M.g_out)
-		
-		self.M.dotin = theano.Method([input1,result],self.M.resultin)
-		self.M.dot = theano.Method([input1],self.M.result)
-		self.m = self.M.make()
-	
-	def make_node(self, args, g_out):
-		idx_list = Checkidx_list(args[0])
-		weights_list = Checkweights_list(args[2:])
-		return Apply(self, args + g_out, [T.dmatrix() for i in xrange(2,len(args))])
-	
-	def perform(self, node, args, z):
-		idx_list = args[0]
-		self.m.g_out = args[-1]
-		
-		batchsize = (args[1][0].shape)[0]
-		n_hid = (args[2].shape)[1]
-		if len(idx_list) != len(args[1]) :
-			raise NotImplementedError('size of index different of inputs list size',idx_list)
-		if max(idx_list) >= (len(args)-3)+1 :
-			raise NotImplementedError('index superior to weight list length',idx_list)
-		for i in range(len(args[1])):
-			if (args[1][i].shape)[0] != batchsize:
-				raise NotImplementedError('different batchsize in the inputs list',args[1][i])
-		for i in range(len(args)-3):
-			if (args[2+i].shape)[1] != n_hid:
-				raise NotImplementedError('different length of hidden in the weights list',args[2+i])
-		
-		zcalc = [False for i in range(len(args)-3)]
-		
-		for i in range(len(idx_list)):
-			if idx_list[i]>0:
-				if zcalc[int(idx_list[i]-1)]:
-					z[int(idx_list[i]-1)][0] = self.m.dotin(args[1][i],z[int(idx_list[i]-1)][0])
-				else:
-					z[int(idx_list[i]-1)][0] = self.m.dot(args[1][i])
-					zcalc[int(idx_list[i]-1)] = True
-		
-		for i in range(len(args)-3):
-			if not zcalc[i]:
-				shp = args[2+i].shape
-				z[i][0] = numpy.zeros((shp[0],shp[1]))
-		
-	def __hash__(self):
-		return hash(ScanDotEncGrad)^15684
-		
-	def __str__(self):
-		return "ScanDotEncGrad"
-
-# Decoding scan dot product------------------------------------
-class ScanDotDec(Op):
-	"""This Op takes an index list (as tensor.ivector), a list of matrices representing
-	the available inputs (as theano.generic), the hidden layer of the DAA (theano.dmatrix)
-	and all the decoding weights tensor.dmatrix of the model. It will select the
-	weights corresponding to the available inputs (according to index list) and compute
-	only the necessary dot products. The outputs will be concatenated and will represent
-	the reconstruction of the different modality in the same order than the index list"""
-	def __init__(self):
-		#Create Theano methods to do the dot products with blas or at least in C.
-		self.M=theano.Module()
-		weights = T.dmatrix('weights')
-		self.M.hid = T.dmatrix('hid')
-		oldval = T.dmatrix('oldval')
-		resultin = oldval + T.dot(self.M.hid,weights)
-		result = T.dot(self.M.hid,weights)
-		
-		self.M.dotin = theano.Method([weights,oldval],resultin)
-		self.M.dot = theano.Method([weights],result)
-		self.m = self.M.make()
-	
-	def make_node(self, idx_list, input_list, hidd, weights_list):
-		idx_list = Checkidx_list(idx_list)
-		hidd = Checkhidd(hidd)
-		weights_list = Checkweights_list(weights_list)
-		return Apply(self, [idx_list] + [input_list] +[hidd] + weights_list,[T.dmatrix()])
-	
-	def perform(self, node, args, (z,)):
-		
-		idx_list = abs(args[0])
-		self.m.hid = args[2]
-		
-		batchsize = (self.m.hid.shape)[0]
-		n_hid = self.m.hid.shape[1]
-		if max(idx_list) >= len(args)-3+1 :
-			raise NotImplementedError('index superior to weight list length',idx_list)
-		if len(idx_list) != len(args[1]) :
-			raise NotImplementedError('size of index different of inputs list size',idx_list)
-		for i in range(len(args)-3):
-			if (args[3+i].shape)[0] != n_hid:
-				raise NotImplementedError('different length of hidden in the weights list',args[3+i])
-		
-		zcalc = [False for i in idx_list]
-		z[0] = [None for i in idx_list]
-		
-		for i in range(len(idx_list)):
-			if idx_list[i]>0:
-				if zcalc[i]:
-					z[0][i] = self.m.dotin(args[3+int(idx_list[i]-1)],z[0][i])
-				else:
-					z[0][i] = self.m.dot(args[3+int(idx_list[i]-1)])
-					zcalc[i] = True
-		
-		for i in range(len(idx_list)):
-			if not zcalc[i]:
-				shp = args[1][int(idx_list[i]-1)].shape
-				z[0][i] = numpy.zeros((batchsize,shp[1]))
-		
-		z[0] = numpy.concatenate(z[0],1)
-		
-	def grad(self, args, gz):
-		return [None, None] + ScanDotDecGrad()(args,gz)
-	
-	def __hash__(self):
-		return hash(ScanDotDec)^73568
-	
-	def __str__(self):
-		return "ScanDotDec"
-
-scandotdec=ScanDotDec()
-
-class ScanDotDecGrad(Op):
-	"""This Op computes the gradient wrt the weights for ScanDotDec"""
-	def __init__(self):
-		self.M=theano.Module()
-		gout = T.dmatrix('gout')
-		self.M.hidt = T.dmatrix('hid')
-		oldval = T.dmatrix('oldval')
-		resultin1 = oldval + T.dot(self.M.hidt,gout)
-		result1 = T.dot(self.M.hidt,gout)
-		weights = T.dmatrix('weights')
-		weights2 = T.transpose(weights)
-		resultin2 = oldval + T.dot(gout,weights2)
-		result2 = T.dot(gout,weights2)
-		
-		self.M.dotin1 = theano.Method([gout,oldval],resultin1)
-		self.M.dot1 = theano.Method([gout],result1)
-		self.M.dotin2 = theano.Method([gout,weights,oldval],resultin2)
-		self.M.dot2 = theano.Method([gout,weights],result2)
-		self.m = self.M.make()
-	
-	
-	def make_node(self, args, g_out):
-		idx_list = Checkidx_list(args[0])
-		hidd = Checkhidd(args[2])
-		weights_list = Checkweights_list(args[3:])
-		return Apply(self, args + g_out, [T.dmatrix() for i in xrange(2,len(args))])
-	
-	def perform(self, node, args, z):
-		idx_list = abs(args[0])
-		self.m.hidt = args[2].T
-		
-		batchsize = (self.m.hidt.shape)[1]
-		n_hid = self.m.hidt.shape[0]
-		if max(idx_list) >= len(args)-4+1 :
-			raise NotImplementedError('index superior to weight list length',idx_list)
-		if len(idx_list) != len(args[1]) :
-			raise NotImplementedError('size of index different of inputs list size',idx_list)
-		for i in range(len(args)-4):
-			if (args[3+i].shape)[0] != n_hid:
-				raise NotImplementedError('different length of hidden in the weights list',args[3+i])
-		
-		zidx=numpy.zeros((len(idx_list)+1))
-		
-		for i in range(len(idx_list)):
-			if idx_list[i] == 0:
-				zidx[i+1] = (args[1][i].shape)[1]
-			else:
-				zidx[i+1] = (args[3+idx_list[i]-1].shape)[1]
-		
-		zidx=zidx.cumsum()
-		hidcalc = False
-		zcalc = [False for i in range((len(args)-4))]
-		
-		for i in range(len(idx_list)):
-			if idx_list[i]>0:
-				if zcalc[int(idx_list[i])-1]:
-					z[int(idx_list[i])][0] = self.m.dotin1(args[-1][:,zidx[i]:zidx[i+1]],z[int(idx_list[i])][0])
-				else:
-					z[int(idx_list[i])][0] = self.m.dot1(args[-1][:,zidx[i]:zidx[i+1]])
-					zcalc[int(idx_list[i])-1] = True
-				if hidcalc:
-					z[0][0] = self.m.dotin2(args[-1][:,zidx[i]:zidx[i+1]],args[3+int(idx_list[i]-1)],z[0][0])
-				else:
-					z[0][0] = self.m.dot2(args[-1][:,zidx[i]:zidx[i+1]],args[3+int(idx_list[i]-1)])
-					hidcalc = True
-		
-		if not hidcalc:
-			z[0][0] = numpy.zeros((self.m.hidt.shape[1],self.m.hidt.shape[0]))
-		
-		for i in range((len(args)-4)):
-			if not zcalc[i]:
-				shp = args[3+i].shape
-				z[i+1][0] = numpy.zeros((shp[0],shp[1]))
-		
-		
-	def __hash__(self):
-		return hash(ScanDotDecGrad)^87445
-	
-	def __str__(self):
-		return "ScanDotDecGrad"
-
-# DAA input noise------------------------------------
-class ScanNoise(Op):
-	"""This Op takes an index list (as tensor.ivector), a list of matrices representing
-	the available inputs (as theano.generic), a probability of individual bit masking and
-	a probability of modality masking. It will return the inputs list with randoms zeros entry
-	and the index list with some positive values changed to negative values (groups masking)"""
-	def __init__(self, seed = 1):
-		self.M=theano.Module()
-		self.M.rand = T.RandomStreams(seed)
-		self.seed = seed
-		mat = T.matrix('mat')
-		noise_level_bit = T.dscalar('noise_level_bit')
-		noise_level_group = T.dscalar('noise_level_group')
-		self.M.out1 = self.M.rand.binomial(T.shape(mat), 1, 1 - noise_level_bit) * mat
-		self.M.out2 = self.M.rand.binomial((1,1), 1, 1 - noise_level_group)
-		
-		self.M.noisify_bit = theano.Method([mat,noise_level_bit],self.M.out1)
-		self.M.noisify_group_bool = theano.Method([noise_level_group],self.M.out2)
-		self.R = self.M.make()
-		self.R.rand.initialize()
-	
-	def make_node(self, idx_list, inputs_list, noise_level_bit, noise_level_group):
-		idx_list = Checkidx_list(idx_list)
-		return Apply(self, [idx_list] + [inputs_list] + [noise_level_bit] + [noise_level_group],\
-				[T.ivector(), theano.generic()])
-	
-	def perform(self, node, (idx_list,inputs_list,noise_level_bit,noise_level_group), (y,z)):
-		
-		if len(idx_list) != len(inputs_list) :
-			raise NotImplementedError('size of index different of inputs list size',idx_list)
-		
-		y[0] = [-i if (i>0 and not(self.R.noisify_group_bool(noise_level_group))) else i for i in idx_list]
-		z[0] = [(self.R.noisify_bit(inputs_list[i],noise_level_bit) if y[0][i]>0 else numpy.zeros((inputs_list[i].shape)))\
-								for i in range(len(inputs_list))]
-	
-	def grad(self,args,gz):
-		return [None,None,None,None]
-	
-	
-	def __hash__(self):
-		return hash(ScanNoise)^hash(self.seed)^hash(self.R)^12254
-	
-	def __str__(self):
-		return "ScanNoise"
-
-scannoise=ScanNoise()
-
-# Total input matrix construction------------------------------------
-class ScanInputs(Op):
-	"""This Op takes an index list (as tensor.ivector) and a list of matrices representing
-	the available inputs (as theano.generic). It will construct the appropriate tensor.dmatrix
-	to compare to the reconstruction obtained with ScanDotDec"""
-	def make_node(self, idx_list, inputs_list):
-		idx_list = Checkidx_list(idx_list)
-		return Apply(self, [idx_list] + [inputs_list],[T.dmatrix()])
-	
-	def perform(self, node, (idx_list, inputs_list), (z,)):
-		
-		if len(idx_list) != len(inputs_list):
-			raise NotImplementedError('size of index different of inputs list size',idx_list)
-		
-		for i in range(len(idx_list)):
-			if idx_list[i] == 0:
-				inputs_list[i] = 0 * inputs_list[i]
-		
-		z[0] = numpy.concatenate(inputs_list,1)
-	
-	def grad(self,args,gz):
-		return [None,None]
-	
-	def __hash__(self):
-		return hash(ScanInputs)^75902
-	
-	def __str__(self):
-		return "ScanInputs"
-
-scaninputs=ScanInputs()
-
-# Decoding bias vector construction------------------------------------
-class ScanBiasDec(Op):
-	"""This Op takes an index list (as tensor.ivector), a list of matrices representing
-	the available inputs (as theano.generic) and the decoding bias tensor.dvector.
-	It will construct the appropriate bias tensor.dvector
-	to add to the reconstruction obtained with ScanDotDec"""
-	def make_node(self, idx_list, input_list, bias_list):
-		idx_list = Checkidx_list(idx_list)
-		bias_list = Checkbias_list(bias_list)
-		return Apply(self, [idx_list] + [input_list] + bias_list, [T.dvector()])
-	
-	def perform(self, node, args, (z,)):
-		idx_list = abs(args[0])
-		
-		if max(idx_list) >= (len(args)-2)+1 :
-			raise NotImplementedError('index superior to bias list length',idx_list)
-		if len(idx_list) != len(args[1]) :
-			raise NotImplementedError('size of index different of inputs list size',idx_list)
-		z[0] = [args[idx_list[i]+1] if idx_list[i] != 0 else numpy.zeros(args[1][i].shape[1]) \
-																		for i in range(len(idx_list))]
-		z[0] = numpy.concatenate(z[0],1)
-	
-	def __hash__(self):
-		return hash(ScanBiasDec)^60056
-	
-	def grad(self,args,gz):
-		return [None,None] + ScanBiasDecGrad()(args,gz)
-	
-	def __str__(self):
-		return "ScanBiasDec"
-
-scanbiasdec=ScanBiasDec()
-
-class ScanBiasDecGrad(Op):
-	"""This Op computes the gradient wrt the bias for ScanBiasDec"""
-	def make_node(self, args, g_out):
-		idx_list = Checkidx_list(args[0])
-		bias_list = Checkbias_list(args[2:])
-		return Apply(self, args + g_out, [T.dvector() for i in range(len(args)-2)])
-	
-	def perform(self, node, args, z):
-		idx_list = abs(args[0])
-		
-		if max(idx_list) >= (len(args)-3)+1 :
-			raise NotImplementedError('index superior to bias list length',idx_list)
-		if len(idx_list) != len(args[1]) :
-			raise NotImplementedError('size of index different of inputs list size',idx_list)
-		
-		zidx=numpy.zeros((len(idx_list)+1))
-		for i in range(len(idx_list)):
-			if idx_list[i] == 0:
-				zidx[i+1] = (args[1][i].shape)[1]
-			else:
-				zidx[i+1] = (args[2+idx_list[i]-1].size)
-		zidx=zidx.cumsum()
-		zcalc = [False for i in range((len(args)-3))]
-		
-		for i in range(len(idx_list)):
-			if idx_list[i]>0:
-				if zcalc[int(idx_list[i])-1]:
-					z[int(idx_list[i])-1][0] += args[-1][zidx[i]:zidx[i+1]]
-				else:
-					z[int(idx_list[i])-1][0] = args[-1][zidx[i]:zidx[i+1]]
-					zcalc[int(idx_list[i])-1] = True
-		
-		for i in range((len(args)-3)):
-			if not zcalc[i]:
-				shp = args[2+i].size
-				z[i][0] = numpy.zeros(shp)
-		
-	
-	def __hash__(self):
-		return hash(ScanBiasDecGrad)^41256
-	
-	def __str__(self):
-		return "ScanBiasDecGrad"

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/pylearn/sandbox/input_groups/scan_inputs_groups.py	Tue May 12 14:51:25 2009 -0400
@@ -0,0 +1,502 @@
+import numpy
+import theano
+from theano import tensor as T
+from theano.gof import Op
+from theano.gof import Apply
+from theano import scalar as scal
+
+# These Ops allows us to deal with static groups of possibly missing inputs efficiently in the dense DAA framework
+# (for exemple with multimodal data with sometimes entire modality missing).
+# The inputs will be represented with an index list and a theano.generic variable (which will be a list of matrices
+# (numpy array), each element will correspond to an available modality and the index list will indicates the weights
+# associated to it).
+# Exemple of index list: [1, 0, -3]
+#	*the 1 says that the first element of the input list will refer to the first element of the weights_list
+#		(auxiliary target as input)
+#								if inputslist[i]>0 it refers to Weightslist[indexlist[i]-1]
+#	*the 0 means that the second element of the input list will not be encoded neither decoded (it is remplaced by zeros)
+#		this is not efficient, so in this case it is better to give: [1,-3] and [inputslist[0],inputslist[2]]
+#		but it allows us to deal with empty lists: give indexlist = [.0] and inputlist=[[.0]]
+#	*when an index is negative it means that the input will not be used for encoding but we will still reconstruct it
+#		(auxiliary target as output)
+#								if inputslist[i]<0 it refers to Weightslist[-indexlist[i]-1]
+#
+# An entire batch should have the same available inputs configuration.
+#
+# Dense DAA Exemple:----------------------------------------------------------------------------
+#
+#from theano.tensor.nnet import sigmoid
+#
+#nb_modality = 4
+#wenc = [T.dmatrix('wenc%s'%i) for i in range(nb_modality)]
+#wdec = [T.dmatrix('wdec%s'%i) for i in range(nb_modality)]
+#benc = T.dvector('benc')
+#bdec = [T.dvector('bdec%s'%i) for i in range(nb_modality)]
+#vectin = T.ivector('vectin')
+#inputpart = theano.generic('inputpart')
+#noise_bit = T.dscalar('noise_bit')
+#noise_group = T.dscalar('noise_group')
+#
+#[vectin2,inputpart2] = scannoise(vectin,inputpart,noise_bit,noise_group)
+#hid = scandotenc(vectin2, inputpart2, wenc)
+#acthid = sigmoid(hid + benc)
+#dec = sigmoid(scanbiasdec(vectin2,inputpart2,bdec) + scandotdec(vectin2, inputpart2,acthid,wdec))
+#cost = T.sum(T.sum(T.sqr( scaninput(vectin,inputpart) - rec ),1),0)
+
+# Checking inputs in make_node methods----------------------
+def Checkidx_list(idx_list):
+	idx_list = T.as_tensor_variable(idx_list)
+	nidx = idx_list.type.ndim
+	if nidx != 1: raise TypeError('not vector', idx_list)
+	return idx_list
+
+def Checkhidd(hidd):
+	hidd = T.as_tensor_variable(hidd)
+	nhidd = hidd.type.ndim
+	if nhidd not in (1,2): raise TypeError('not matrix or vector', hidd)
+	return hidd
+
+def Checkweights_list(weights_list):
+	weights_list = map(T.as_tensor_variable, weights_list)
+	for i in range(len(weights_list)):
+		nweights = weights_list[i].type.ndim
+		if nweights not in (1,2): raise TypeError('not matrix or vector', weights_list[i])
+	return weights_list
+
+def Checkbias_list(bias_list):
+	bias_list = map(T.as_tensor_variable, bias_list)
+	for i in range(len(bias_list)):
+		nbias = bias_list[i].type.ndim
+		if nbias != 1: raise TypeError('not vector', bias_list[i])
+	return bias_list
+
+# Encoding scan dot product------------------------------------
+class ScanDotEnc(Op):
+	"""This Op takes an index list (as tensor.ivector), a list of matrices representing
+	the available inputs (as theano.generic), and all the encoding weights tensor.dmatrix of the model. It will select the
+	weights corresponding to the inputs (according to index list) and compute only the necessary dot products"""
+	def __init__(self):
+		#Create Theano methods to do the dot products with blas or at least in C.
+		self.M=theano.Module()
+		inputs = T.dmatrix('input')
+		weights = T.dmatrix('weights')
+		self.M.hid = T.dmatrix('hid')
+		self.M.resultin = self.M.hid + T.dot(inputs,weights)
+		result = T.dot(inputs,weights)
+		
+		self.M.dotin = theano.Method([inputs,weights],None,{self.M.hid : self.M.resultin})
+		self.M.dot = theano.Method([inputs,weights],result)
+		self.m = self.M.make()
+	
+	def make_node(self, idx_list, inputs_list, weights_list):
+		idx_list = Checkidx_list(idx_list)
+		weights_list = Checkweights_list(weights_list)
+		return Apply(self, [idx_list] + [inputs_list] + weights_list, [T.dmatrix()])
+	
+	def perform(self, node, args, (hid,)):
+		idx_list = args[0]
+		hidcalc = False
+		
+		batchsize = (args[1][0].shape)[0]
+		n_hid = (args[2].shape)[1]
+		if len(idx_list) != len(args[1]) :
+			raise NotImplementedError('size of index different of inputs list size',idx_list)
+		if max(idx_list) >= (len(args)-2)+1 :
+			raise NotImplementedError('index superior to weight list length',idx_list)
+		for i in range(len(args[1])):
+			if (args[1][i].shape)[0] != batchsize:
+				raise NotImplementedError('different batchsize in the inputs list',args[1][i])
+		for i in range(len(args)-2):
+			if (args[2+i].shape)[1] != n_hid:
+				raise NotImplementedError('different length of hidden in the weights list',args[2+i])
+		
+		for i in range(len(idx_list)):
+			if idx_list[i]>0:
+				if hidcalc:
+					self.m.dotin(args[1][i],args[2+int(idx_list[i]-1)])
+				else:
+					self.m.hid = self.m.dot(args[1][i],args[2+int(idx_list[i]-1)])
+					hidcalc = True
+		
+		if not hidcalc:
+			hid[0] = numpy.zeros([batchsize,n_hid])
+		else:
+			hid[0] = self.m.hid
+		
+	
+	def grad(self, args, gz):
+		return [None, None] + ScanDotEncGrad()(args,gz)
+	
+	def __hash__(self):
+		return hash(ScanDotEnc)^58994
+	
+	def __str__(self):
+		return "ScanDotEnc"
+
+scandotenc=ScanDotEnc()
+
+class ScanDotEncGrad(Op):
+	"""This Op computes the gradient wrt the weights for ScanDotEnc"""
+	def __init__(self):
+		#Create Theano methods to do the dot products with blas or at least in C.
+		self.M=theano.Module()
+		input1 = T.dmatrix('input1')
+		self.M.g_out = T.dmatrix('g_out')
+		result = T.dmatrix('result')
+		input2=T.transpose(input1)
+		self.M.resultin = result + T.dot(input2,self.M.g_out)
+		self.M.result = T.dot(input2,self.M.g_out)
+		
+		self.M.dotin = theano.Method([input1,result],self.M.resultin)
+		self.M.dot = theano.Method([input1],self.M.result)
+		self.m = self.M.make()
+	
+	def make_node(self, args, g_out):
+		idx_list = Checkidx_list(args[0])
+		weights_list = Checkweights_list(args[2:])
+		return Apply(self, args + g_out, [T.dmatrix() for i in xrange(2,len(args))])
+	
+	def perform(self, node, args, z):
+		idx_list = args[0]
+		self.m.g_out = args[-1]
+		
+		batchsize = (args[1][0].shape)[0]
+		n_hid = (args[2].shape)[1]
+		if len(idx_list) != len(args[1]) :
+			raise NotImplementedError('size of index different of inputs list size',idx_list)
+		if max(idx_list) >= (len(args)-3)+1 :
+			raise NotImplementedError('index superior to weight list length',idx_list)
+		for i in range(len(args[1])):
+			if (args[1][i].shape)[0] != batchsize:
+				raise NotImplementedError('different batchsize in the inputs list',args[1][i])
+		for i in range(len(args)-3):
+			if (args[2+i].shape)[1] != n_hid:
+				raise NotImplementedError('different length of hidden in the weights list',args[2+i])
+		
+		zcalc = [False for i in range(len(args)-3)]
+		
+		for i in range(len(idx_list)):
+			if idx_list[i]>0:
+				if zcalc[int(idx_list[i]-1)]:
+					z[int(idx_list[i]-1)][0] = self.m.dotin(args[1][i],z[int(idx_list[i]-1)][0])
+				else:
+					z[int(idx_list[i]-1)][0] = self.m.dot(args[1][i])
+					zcalc[int(idx_list[i]-1)] = True
+		
+		for i in range(len(args)-3):
+			if not zcalc[i]:
+				shp = args[2+i].shape
+				z[i][0] = numpy.zeros((shp[0],shp[1]))
+		
+	def __hash__(self):
+		return hash(ScanDotEncGrad)^15684
+		
+	def __str__(self):
+		return "ScanDotEncGrad"
+
+# Decoding scan dot product------------------------------------
+class ScanDotDec(Op):
+	"""This Op takes an index list (as tensor.ivector), a list of matrices representing
+	the available inputs (as theano.generic), the hidden layer of the DAA (theano.dmatrix)
+	and all the decoding weights tensor.dmatrix of the model. It will select the
+	weights corresponding to the available inputs (according to index list) and compute
+	only the necessary dot products. The outputs will be concatenated and will represent
+	the reconstruction of the different modality in the same order than the index list"""
+	def __init__(self):
+		#Create Theano methods to do the dot products with blas or at least in C.
+		self.M=theano.Module()
+		weights = T.dmatrix('weights')
+		self.M.hid = T.dmatrix('hid')
+		oldval = T.dmatrix('oldval')
+		resultin = oldval + T.dot(self.M.hid,weights)
+		result = T.dot(self.M.hid,weights)
+		
+		self.M.dotin = theano.Method([weights,oldval],resultin)
+		self.M.dot = theano.Method([weights],result)
+		self.m = self.M.make()
+	
+	def make_node(self, idx_list, input_list, hidd, weights_list):
+		idx_list = Checkidx_list(idx_list)
+		hidd = Checkhidd(hidd)
+		weights_list = Checkweights_list(weights_list)
+		return Apply(self, [idx_list] + [input_list] +[hidd] + weights_list,[T.dmatrix()])
+	
+	def perform(self, node, args, (z,)):
+		
+		idx_list = abs(args[0])
+		self.m.hid = args[2]
+		
+		batchsize = (self.m.hid.shape)[0]
+		n_hid = self.m.hid.shape[1]
+		if max(idx_list) >= len(args)-3+1 :
+			raise NotImplementedError('index superior to weight list length',idx_list)
+		if len(idx_list) != len(args[1]) :
+			raise NotImplementedError('size of index different of inputs list size',idx_list)
+		for i in range(len(args)-3):
+			if (args[3+i].shape)[0] != n_hid:
+				raise NotImplementedError('different length of hidden in the weights list',args[3+i])
+		
+		zcalc = [False for i in idx_list]
+		z[0] = [None for i in idx_list]
+		
+		for i in range(len(idx_list)):
+			if idx_list[i]>0:
+				if zcalc[i]:
+					z[0][i] = self.m.dotin(args[3+int(idx_list[i]-1)],z[0][i])
+				else:
+					z[0][i] = self.m.dot(args[3+int(idx_list[i]-1)])
+					zcalc[i] = True
+		
+		for i in range(len(idx_list)):
+			if not zcalc[i]:
+				shp = args[1][int(idx_list[i]-1)].shape
+				z[0][i] = numpy.zeros((batchsize,shp[1]))
+		
+		z[0] = numpy.concatenate(z[0],1)
+		
+	def grad(self, args, gz):
+		return [None, None] + ScanDotDecGrad()(args,gz)
+	
+	def __hash__(self):
+		return hash(ScanDotDec)^73568
+	
+	def __str__(self):
+		return "ScanDotDec"
+
+scandotdec=ScanDotDec()
+
+class ScanDotDecGrad(Op):
+	"""This Op computes the gradient wrt the weights for ScanDotDec"""
+	def __init__(self):
+		self.M=theano.Module()
+		gout = T.dmatrix('gout')
+		self.M.hidt = T.dmatrix('hid')
+		oldval = T.dmatrix('oldval')
+		resultin1 = oldval + T.dot(self.M.hidt,gout)
+		result1 = T.dot(self.M.hidt,gout)
+		weights = T.dmatrix('weights')
+		weights2 = T.transpose(weights)
+		resultin2 = oldval + T.dot(gout,weights2)
+		result2 = T.dot(gout,weights2)
+		
+		self.M.dotin1 = theano.Method([gout,oldval],resultin1)
+		self.M.dot1 = theano.Method([gout],result1)
+		self.M.dotin2 = theano.Method([gout,weights,oldval],resultin2)
+		self.M.dot2 = theano.Method([gout,weights],result2)
+		self.m = self.M.make()
+	
+	
+	def make_node(self, args, g_out):
+		idx_list = Checkidx_list(args[0])
+		hidd = Checkhidd(args[2])
+		weights_list = Checkweights_list(args[3:])
+		return Apply(self, args + g_out, [T.dmatrix() for i in xrange(2,len(args))])
+	
+	def perform(self, node, args, z):
+		idx_list = abs(args[0])
+		self.m.hidt = args[2].T
+		
+		batchsize = (self.m.hidt.shape)[1]
+		n_hid = self.m.hidt.shape[0]
+		if max(idx_list) >= len(args)-4+1 :
+			raise NotImplementedError('index superior to weight list length',idx_list)
+		if len(idx_list) != len(args[1]) :
+			raise NotImplementedError('size of index different of inputs list size',idx_list)
+		for i in range(len(args)-4):
+			if (args[3+i].shape)[0] != n_hid:
+				raise NotImplementedError('different length of hidden in the weights list',args[3+i])
+		
+		zidx=numpy.zeros((len(idx_list)+1))
+		
+		for i in range(len(idx_list)):
+			if idx_list[i] == 0:
+				zidx[i+1] = (args[1][i].shape)[1]
+			else:
+				zidx[i+1] = (args[3+idx_list[i]-1].shape)[1]
+		
+		zidx=zidx.cumsum()
+		hidcalc = False
+		zcalc = [False for i in range((len(args)-4))]
+		
+		for i in range(len(idx_list)):
+			if idx_list[i]>0:
+				if zcalc[int(idx_list[i])-1]:
+					z[int(idx_list[i])][0] = self.m.dotin1(args[-1][:,zidx[i]:zidx[i+1]],z[int(idx_list[i])][0])
+				else:
+					z[int(idx_list[i])][0] = self.m.dot1(args[-1][:,zidx[i]:zidx[i+1]])
+					zcalc[int(idx_list[i])-1] = True
+				if hidcalc:
+					z[0][0] = self.m.dotin2(args[-1][:,zidx[i]:zidx[i+1]],args[3+int(idx_list[i]-1)],z[0][0])
+				else:
+					z[0][0] = self.m.dot2(args[-1][:,zidx[i]:zidx[i+1]],args[3+int(idx_list[i]-1)])
+					hidcalc = True
+		
+		if not hidcalc:
+			z[0][0] = numpy.zeros((self.m.hidt.shape[1],self.m.hidt.shape[0]))
+		
+		for i in range((len(args)-4)):
+			if not zcalc[i]:
+				shp = args[3+i].shape
+				z[i+1][0] = numpy.zeros((shp[0],shp[1]))
+		
+		
+	def __hash__(self):
+		return hash(ScanDotDecGrad)^87445
+	
+	def __str__(self):
+		return "ScanDotDecGrad"
+
+# DAA input noise------------------------------------
+class ScanNoise(Op):
+	"""This Op takes an index list (as tensor.ivector), a list of matrices representing
+	the available inputs (as theano.generic), a probability of individual bit masking and
+	a probability of modality masking. It will return the inputs list with randoms zeros entry
+	and the index list with some positive values changed to negative values (groups masking)"""
+	def __init__(self, seed = 1):
+		self.M=theano.Module()
+		self.M.rand = T.RandomStreams(seed)
+		self.seed = seed
+		mat = T.matrix('mat')
+		noise_level_bit = T.dscalar('noise_level_bit')
+		noise_level_group = T.dscalar('noise_level_group')
+		self.M.out1 = self.M.rand.binomial(T.shape(mat), 1, 1 - noise_level_bit) * mat
+		self.M.out2 = self.M.rand.binomial((1,1), 1, 1 - noise_level_group)
+		
+		self.M.noisify_bit = theano.Method([mat,noise_level_bit],self.M.out1)
+		self.M.noisify_group_bool = theano.Method([noise_level_group],self.M.out2)
+		self.R = self.M.make()
+		self.R.rand.initialize()
+	
+	def make_node(self, idx_list, inputs_list, noise_level_bit, noise_level_group):
+		idx_list = Checkidx_list(idx_list)
+		return Apply(self, [idx_list] + [inputs_list] + [noise_level_bit] + [noise_level_group],\
+				[T.ivector(), theano.generic()])
+	
+	def perform(self, node, (idx_list,inputs_list,noise_level_bit,noise_level_group), (y,z)):
+		
+		if len(idx_list) != len(inputs_list) :
+			raise NotImplementedError('size of index different of inputs list size',idx_list)
+		
+		y[0] = [-i if (i>0 and not(self.R.noisify_group_bool(noise_level_group))) else i for i in idx_list]
+		z[0] = [(self.R.noisify_bit(inputs_list[i],noise_level_bit) if y[0][i]>0 else numpy.zeros((inputs_list[i].shape)))\
+								for i in range(len(inputs_list))]
+	
+	def grad(self,args,gz):
+		return [None,None,None,None]
+	
+	
+	def __hash__(self):
+		return hash(ScanNoise)^hash(self.seed)^hash(self.R)^12254
+	
+	def __str__(self):
+		return "ScanNoise"
+
+scannoise=ScanNoise()
+
+# Total input matrix construction------------------------------------
+class ScanInputs(Op):
+	"""This Op takes an index list (as tensor.ivector) and a list of matrices representing
+	the available inputs (as theano.generic). It will construct the appropriate tensor.dmatrix
+	to compare to the reconstruction obtained with ScanDotDec"""
+	def make_node(self, idx_list, inputs_list):
+		idx_list = Checkidx_list(idx_list)
+		return Apply(self, [idx_list] + [inputs_list],[T.dmatrix()])
+	
+	def perform(self, node, (idx_list, inputs_list), (z,)):
+		
+		if len(idx_list) != len(inputs_list):
+			raise NotImplementedError('size of index different of inputs list size',idx_list)
+		
+		for i in range(len(idx_list)):
+			if idx_list[i] == 0:
+				inputs_list[i] = 0 * inputs_list[i]
+		
+		z[0] = numpy.concatenate(inputs_list,1)
+	
+	def grad(self,args,gz):
+		return [None,None]
+	
+	def __hash__(self):
+		return hash(ScanInputs)^75902
+	
+	def __str__(self):
+		return "ScanInputs"
+
+scaninputs=ScanInputs()
+
+# Decoding bias vector construction------------------------------------
+class ScanBiasDec(Op):
+	"""This Op takes an index list (as tensor.ivector), a list of matrices representing
+	the available inputs (as theano.generic) and the decoding bias tensor.dvector.
+	It will construct the appropriate bias tensor.dvector
+	to add to the reconstruction obtained with ScanDotDec"""
+	def make_node(self, idx_list, input_list, bias_list):
+		idx_list = Checkidx_list(idx_list)
+		bias_list = Checkbias_list(bias_list)
+		return Apply(self, [idx_list] + [input_list] + bias_list, [T.dvector()])
+	
+	def perform(self, node, args, (z,)):
+		idx_list = abs(args[0])
+		
+		if max(idx_list) >= (len(args)-2)+1 :
+			raise NotImplementedError('index superior to bias list length',idx_list)
+		if len(idx_list) != len(args[1]) :
+			raise NotImplementedError('size of index different of inputs list size',idx_list)
+		z[0] = [args[idx_list[i]+1] if idx_list[i] != 0 else numpy.zeros(args[1][i].shape[1]) \
+																		for i in range(len(idx_list))]
+		z[0] = numpy.concatenate(z[0],1)
+	
+	def __hash__(self):
+		return hash(ScanBiasDec)^60056
+	
+	def grad(self,args,gz):
+		return [None,None] + ScanBiasDecGrad()(args,gz)
+	
+	def __str__(self):
+		return "ScanBiasDec"
+
+scanbiasdec=ScanBiasDec()
+
+class ScanBiasDecGrad(Op):
+	"""This Op computes the gradient wrt the bias for ScanBiasDec"""
+	def make_node(self, args, g_out):
+		idx_list = Checkidx_list(args[0])
+		bias_list = Checkbias_list(args[2:])
+		return Apply(self, args + g_out, [T.dvector() for i in range(len(args)-2)])
+	
+	def perform(self, node, args, z):
+		idx_list = abs(args[0])
+		
+		if max(idx_list) >= (len(args)-3)+1 :
+			raise NotImplementedError('index superior to bias list length',idx_list)
+		if len(idx_list) != len(args[1]) :
+			raise NotImplementedError('size of index different of inputs list size',idx_list)
+		
+		zidx=numpy.zeros((len(idx_list)+1))
+		for i in range(len(idx_list)):
+			if idx_list[i] == 0:
+				zidx[i+1] = (args[1][i].shape)[1]
+			else:
+				zidx[i+1] = (args[2+idx_list[i]-1].size)
+		zidx=zidx.cumsum()
+		zcalc = [False for i in range((len(args)-3))]
+		
+		for i in range(len(idx_list)):
+			if idx_list[i]>0:
+				if zcalc[int(idx_list[i])-1]:
+					z[int(idx_list[i])-1][0] += args[-1][zidx[i]:zidx[i+1]]
+				else:
+					z[int(idx_list[i])-1][0] = args[-1][zidx[i]:zidx[i+1]]
+					zcalc[int(idx_list[i])-1] = True
+		
+		for i in range((len(args)-3)):
+			if not zcalc[i]:
+				shp = args[2+i].size
+				z[i][0] = numpy.zeros(shp)
+		
+	
+	def __hash__(self):
+		return hash(ScanBiasDecGrad)^41256
+	
+	def __str__(self):
+		return "ScanBiasDecGrad"