Mercurial > pylearn

comparison pylearn/sandbox/scan_inputs_groups.py @ 714:8d5d42274bd1

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
improved readability DAA_inputs_groups and scan_inputs_groups
author Xavier Glorot <glorotxa@iro.umontreal.ca>
date Fri, 22 May 2009 15:14:34 -0400
parents 0eae6d5315b5
children 573e3370d0fa
comparison
equal deleted inserted replaced
713:a268c5ea0db4 714:8d5d42274bd1
9 # (for exemple with multimodal data with sometimes entire modality missing). 9 # (for exemple with multimodal data with sometimes entire modality missing).
10 # The inputs will be represented with an index list and a theano.generic variable (which will be a list of matrices 10 # The inputs will be represented with an index list and a theano.generic variable (which will be a list of matrices
11 # (numpy array), each element will correspond to an available modality and the index list will indicate the weights 11 # (numpy array), each element will correspond to an available modality and the index list will indicate the weights
12 # associated to it). 12 # associated to it).
13 # Exemple of index list: [1, 0, -3] 13 # Exemple of index list: [1, 0, -3]
14 # *the 1 says that the first element of the input list will refer to the first element of the weights_list 14 # *the 1 says that the first element of the input list will refer to the first element of the weights_list
15 # (auxiliary target as input) 15 # (auxiliary target as input)
16 # if inputslist[i]>0 it refers to Weightslist[indexlist[i]-1] 16 # if inputslist[i]>0 it refers to Weightslist[indexlist[i]-1]
17 # *the 0 means that the second element of the input list will not be encoded neither decoded (it is remplaced by zeros) 17 # *the 0 means that the second element of the input list will not be encoded neither decoded (it is remplaced by zeros)
18 # this is not efficient, so in this case it is better to give: [1,-3] and [inputslist[0],inputslist[2]] 18 # this is not efficient, so in this case it is better to give: [1,-3] and [inputslist[0],inputslist[2]]
19 # but it allows us to deal with empty lists: give indexlist = numpy.asarray([.0]) 19 # but it allows us to deal with empty lists: give indexlist = numpy.asarray([.0])
20 # and inputlist=numpy.zeros((batchsize,1)) 20 # and inputlist=numpy.zeros((batchsize,1))
21 # *when an index is negative it means that the input will not be used for encoding but we will still reconstruct it 21 # *when an index is negative it means that the input will not be used for encoding but we will still reconstruct it
22 # (auxiliary target as output) 22 # (auxiliary target as output)
23 # if inputslist[i]<0 it refers to Weightslist[-indexlist[i]-1] 23 # if inputslist[i]<0 it refers to Weightslist[-indexlist[i]-1]
24 # 24 #
25 # An entire batch should have the same available inputs configuration. 25 # An entire batch should have the same available inputs configuration.
26 # 26 #
27 # Dense DAA Exemple:---------------------------------------------------------------------------- 27 # Dense DAA Exemple:----------------------------------------------------------------------------
28 # 28 #
44 #dec = sigmoid(scanbiasdec(vectin2,inputpart2,bdec) + scandotdec(vectin2, inputpart2,acthid,wdec)) 44 #dec = sigmoid(scanbiasdec(vectin2,inputpart2,bdec) + scandotdec(vectin2, inputpart2,acthid,wdec))
45 #cost = T.sum(T.sum(T.sqr( scaninput(vectin,inputpart) - rec ),1),0) 45 #cost = T.sum(T.sum(T.sqr( scaninput(vectin,inputpart) - rec ),1),0)
46 46
47 # Checking inputs in make_node methods---------------------- 47 # Checking inputs in make_node methods----------------------
48 def Checkidx_list(idx_list): 48 def Checkidx_list(idx_list):
49 idx_list = T.as_tensor_variable(idx_list) 49 idx_list = T.as_tensor_variable(idx_list)
50 nidx = idx_list.type.ndim 50 nidx = idx_list.type.ndim
51 if nidx != 1: raise TypeError('not vector', idx_list) 51 if nidx != 1: raise TypeError('not vector', idx_list)
52 return idx_list 52 return idx_list
53 53
54 def Checkhidd(hidd): 54 def Checkhidd(hidd):
55 hidd = T.as_tensor_variable(hidd) 55 hidd = T.as_tensor_variable(hidd)
56 nhidd = hidd.type.ndim 56 nhidd = hidd.type.ndim
57 if nhidd not in (1,2): raise TypeError('not matrix or vector', hidd) 57 if nhidd not in (1,2): raise TypeError('not matrix or vector', hidd)
58 return hidd 58 return hidd
59 59
60 def Checkweights_list(weights_list): 60 def Checkweights_list(weights_list):
61 weights_list = map(T.as_tensor_variable, weights_list) 61 weights_list = map(T.as_tensor_variable, weights_list)
62 for i in range(len(weights_list)): 62 for i in range(len(weights_list)):
63 nweights = weights_list[i].type.ndim 63 nweights = weights_list[i].type.ndim
64 if nweights not in (1,2): raise TypeError('not matrix or vector', weights_list[i]) 64 if nweights not in (1,2): raise TypeError('not matrix or vector', weights_list[i])
65 return weights_list 65 return weights_list
66 66
67 def Checkbias_list(bias_list): 67 def Checkbias_list(bias_list):
68 bias_list = map(T.as_tensor_variable, bias_list) 68 bias_list = map(T.as_tensor_variable, bias_list)
69 for i in range(len(bias_list)): 69 for i in range(len(bias_list)):
70 nbias = bias_list[i].type.ndim 70 nbias = bias_list[i].type.ndim
71 if nbias != 1: raise TypeError('not vector', bias_list[i]) 71 if nbias != 1: raise TypeError('not vector', bias_list[i])
72 return bias_list 72 return bias_list
73 73
74 # Encoding scan dot product------------------------------------ 74 # Encoding scan dot product------------------------------------
75 class ScanDotEnc(Op): 75 class ScanDotEnc(Op):
76 """This Op takes an index list (as tensor.ivector), a list of matrices representing 76 """This Op takes an index list (as tensor.ivector), a list of matrices representing
77 the available inputs (as theano.generic), and all the encoding weights tensor.dmatrix of the model. It will select the 77 the available inputs (as theano.generic), and all the encoding weights tensor.dmatrix of the model. It will select the
78 weights corresponding to the inputs (according to index list) and compute only the necessary dot products""" 78 weights corresponding to the inputs (according to index list) and compute only the necessary dot products"""
79 def __init__(self): 79 def __init__(self):
80 #Create Theano methods to do the dot products with blas or at least in C. 80 #Create Theano methods to do the dot products with blas or at least in C.
81 self.M=theano.Module() 81 self.M=theano.Module()
82 inputs = T.dmatrix('input') 82 inputs = T.dmatrix('input')
83 weights = T.dmatrix('weights') 83 weights = T.dmatrix('weights')
84 self.M.hid = T.dmatrix('hid') 84 self.M.hid = T.dmatrix('hid')
85 self.M.resultin = self.M.hid + T.dot(inputs,weights) 85 self.M.resultin = self.M.hid + T.dot(inputs,weights)
86 result = T.dot(inputs,weights) 86 result = T.dot(inputs,weights)
87 87
88 self.M.dotin = theano.Method([inputs,weights],None,{self.M.hid : self.M.resultin}) 88 self.M.dotin = theano.Method([inputs,weights],None,{self.M.hid : self.M.resultin})
89 self.M.dot = theano.Method([inputs,weights],result) 89 self.M.dot = theano.Method([inputs,weights],result)
90 self.m = self.M.make() 90 self.m = self.M.make()
91 91
92 def make_node(self, idx_list, inputs_list, weights_list): 92 def make_node(self, idx_list, inputs_list, weights_list):
93 idx_list = Checkidx_list(idx_list) 93 idx_list = Checkidx_list(idx_list)
94 weights_list = Checkweights_list(weights_list) 94 weights_list = Checkweights_list(weights_list)
95 return Apply(self, [idx_list] + [inputs_list] + weights_list, [T.dmatrix()]) 95 return Apply(self, [idx_list] + [inputs_list] + weights_list, [T.dmatrix()])
96 96
97 def perform(self, node, args, (hid,)): 97 def perform(self, node, args, (hid,)):
98 idx_list = args[0] 98 idx_list = args[0]
99 hidcalc = False 99 hidcalc = False
100 100
101 batchsize = (args[1][0].shape)[0] 101 batchsize = (args[1][0].shape)[0]
102 n_hid = (args[2].shape)[1] 102 n_hid = (args[2].shape)[1]
103 if len(idx_list) != len(args[1]) : 103 if len(idx_list) != len(args[1]) :
104 raise NotImplementedError('size of index different of inputs list size',idx_list) 104 raise NotImplementedError('size of index different of inputs list size',idx_list)
105 if max(idx_list) >= (len(args)-2)+1 : 105 if max(idx_list) >= (len(args)-2)+1 :
106 raise NotImplementedError('index superior to weight list length',idx_list) 106 raise NotImplementedError('index superior to weight list length',idx_list)
107 for i in range(len(args[1])): 107 for i in range(len(args[1])):
108 if (args[1][i].shape)[0] != batchsize: 108 if (args[1][i].shape)[0] != batchsize:
109 raise NotImplementedError('different batchsize in the inputs list',args[1][i].shape) 109 raise NotImplementedError('different batchsize in the inputs list',args[1][i].shape)
110 for i in range(len(args)-2): 110 for i in range(len(args)-2):
111 if (args[2+i].shape)[1] != n_hid: 111 if (args[2+i].shape)[1] != n_hid:
112 raise NotImplementedError('different length of hidden in the weights list',args[2+i].shape) 112 raise NotImplementedError('different length of hidden in the weights list',args[2+i].shape)
113 113
114 for i in range(len(idx_list)): 114 for i in range(len(idx_list)):
115 if idx_list[i]>0: 115 if idx_list[i]>0:
116 if hidcalc: 116 if hidcalc:
117 self.m.dotin(args[1][i],args[2+int(idx_list[i]-1)]) 117 self.m.dotin(args[1][i],args[2+int(idx_list[i]-1)])
118 else: 118 else:
119 self.m.hid = self.m.dot(args[1][i],args[2+int(idx_list[i]-1)]) 119 self.m.hid = self.m.dot(args[1][i],args[2+int(idx_list[i]-1)])
120 hidcalc = True 120 hidcalc = True
121 121
122 if not hidcalc: 122 if not hidcalc:
123 hid[0] = numpy.zeros([batchsize,n_hid]) 123 hid[0] = numpy.zeros([batchsize,n_hid])
124 else: 124 else:
125 hid[0] = self.m.hid 125 hid[0] = self.m.hid
126 126
127 127
128 def grad(self, args, gz): 128 def grad(self, args, gz):
129 gradi = ScanDotEncGrad()(args,gz) 129 gradi = ScanDotEncGrad()(args,gz)
130 if type(gradi) != list: 130 if type(gradi) != list:
131 return [None, None] + [gradi] 131 return [None, None] + [gradi]
132 else: 132 else:
133 return [None, None] + gradi 133 return [None, None] + gradi
134 134
135 def __hash__(self): 135 def __hash__(self):
136 return hash(ScanDotEnc)^58994 136 return hash(ScanDotEnc)^58994
137 137
138 def __str__(self): 138 def __str__(self):
139 return "ScanDotEnc" 139 return "ScanDotEnc"
140 140
141 scandotenc=ScanDotEnc() 141 scandotenc=ScanDotEnc()
142 142
143 class ScanDotEncGrad(Op): 143 class ScanDotEncGrad(Op):
144 """This Op computes the gradient wrt the weights for ScanDotEnc""" 144 """This Op computes the gradient wrt the weights for ScanDotEnc"""
145 def __init__(self): 145 def __init__(self):
146 #Create Theano methods to do the dot products with blas or at least in C. 146 #Create Theano methods to do the dot products with blas or at least in C.
147 self.M=theano.Module() 147 self.M=theano.Module()
148 input1 = T.dmatrix('input1') 148 input1 = T.dmatrix('input1')
149 self.M.g_out = T.dmatrix('g_out') 149 self.M.g_out = T.dmatrix('g_out')
150 result = T.dmatrix('result') 150 result = T.dmatrix('result')
151 input2=T.transpose(input1) 151 input2=T.transpose(input1)
152 self.M.resultin = result + T.dot(input2,self.M.g_out) 152 self.M.resultin = result + T.dot(input2,self.M.g_out)
153 self.M.result = T.dot(input2,self.M.g_out) 153 self.M.result = T.dot(input2,self.M.g_out)
154 154
155 self.M.dotin = theano.Method([input1,result],self.M.resultin) 155 self.M.dotin = theano.Method([input1,result],self.M.resultin)
156 self.M.dot = theano.Method([input1],self.M.result) 156 self.M.dot = theano.Method([input1],self.M.result)
157 self.m = self.M.make() 157 self.m = self.M.make()
158 158
159 def make_node(self, args, g_out): 159 def make_node(self, args, g_out):
160 idx_list = Checkidx_list(args[0]) 160 idx_list = Checkidx_list(args[0])
161 weights_list = Checkweights_list(args[2:]) 161 weights_list = Checkweights_list(args[2:])
162 return Apply(self, args + g_out, [T.dmatrix() for i in xrange(2,len(args))]) 162 return Apply(self, args + g_out, [T.dmatrix() for i in xrange(2,len(args))])
163 163
164 def perform(self, node, args, z): 164 def perform(self, node, args, z):
165 idx_list = args[0] 165 idx_list = args[0]
166 self.m.g_out = args[-1] 166 self.m.g_out = args[-1]
167 167
168 batchsize = (args[1][0].shape)[0] 168 batchsize = (args[1][0].shape)[0]
169 n_hid = (args[2].shape)[1] 169 n_hid = (args[2].shape)[1]
170 if len(idx_list) != len(args[1]) : 170 if len(idx_list) != len(args[1]) :
171 raise NotImplementedError('size of index different of inputs list size',idx_list) 171 raise NotImplementedError('size of index different of inputs list size',idx_list)
172 if max(idx_list) >= (len(args)-3)+1 : 172 if max(idx_list) >= (len(args)-3)+1 :
173 raise NotImplementedError('index superior to weight list length',idx_list) 173 raise NotImplementedError('index superior to weight list length',idx_list)
174 for i in range(len(args[1])): 174 for i in range(len(args[1])):
175 if (args[1][i].shape)[0] != batchsize: 175 if (args[1][i].shape)[0] != batchsize:
176 raise NotImplementedError('different batchsize in the inputs list',args[1][i].shape) 176 raise NotImplementedError('different batchsize in the inputs list',args[1][i].shape)
177 for i in range(len(args)-3): 177 for i in range(len(args)-3):
178 if (args[2+i].shape)[1] != n_hid: 178 if (args[2+i].shape)[1] != n_hid:
179 raise NotImplementedError('different length of hidden in the weights list',args[2+i].shape) 179 raise NotImplementedError('different length of hidden in the weights list',args[2+i].shape)
180 180
181 zcalc = [False for i in range(len(args)-3)] 181 zcalc = [False for i in range(len(args)-3)]
182 182
183 for i in range(len(idx_list)): 183 for i in range(len(idx_list)):
184 if idx_list[i]>0: 184 if idx_list[i]>0:
185 if zcalc[int(idx_list[i]-1)]: 185 if zcalc[int(idx_list[i]-1)]:
186 z[int(idx_list[i]-1)][0] = self.m.dotin(args[1][i],z[int(idx_list[i]-1)][0]) 186 z[int(idx_list[i]-1)][0] = self.m.dotin(args[1][i],z[int(idx_list[i]-1)][0])
187 else: 187 else:
188 z[int(idx_list[i]-1)][0] = self.m.dot(args[1][i]) 188 z[int(idx_list[i]-1)][0] = self.m.dot(args[1][i])
189 zcalc[int(idx_list[i]-1)] = True 189 zcalc[int(idx_list[i]-1)] = True
190 190
191 for i in range(len(args)-3): 191 for i in range(len(args)-3):
192 if not zcalc[i]: 192 if not zcalc[i]:
193 shp = args[2+i].shape 193 shp = args[2+i].shape
194 z[i][0] = numpy.zeros(shp) 194 z[i][0] = numpy.zeros(shp)
195 195
196 def __hash__(self): 196 def __hash__(self):
197 return hash(ScanDotEncGrad)^15684 197 return hash(ScanDotEncGrad)^15684
198 198
199 def __str__(self): 199 def __str__(self):
200 return "ScanDotEncGrad" 200 return "ScanDotEncGrad"
201 201
202 # Decoding scan dot product------------------------------------ 202 # Decoding scan dot product------------------------------------
203 class ScanDotDec(Op): 203 class ScanDotDec(Op):
204 """This Op takes an index list (as tensor.ivector), a list of matrices representing 204 """This Op takes an index list (as tensor.ivector), a list of matrices representing
205 the available inputs (as theano.generic), the hidden layer of the DAA (theano.dmatrix) 205 the available inputs (as theano.generic), the hidden layer of the DAA (theano.dmatrix)
206 and all the decoding weights tensor.dmatrix of the model. It will select the 206 and all the decoding weights tensor.dmatrix of the model. It will select the
207 weights corresponding to the available inputs (according to index list) and compute 207 weights corresponding to the available inputs (according to index list) and compute
208 only the necessary dot products. The outputs will be concatenated and will represent 208 only the necessary dot products. The outputs will be concatenated and will represent
209 the reconstruction of the different modality in the same order than the index list""" 209 the reconstruction of the different modality in the same order than the index list"""
210 def __init__(self): 210 def __init__(self):
211 #Create Theano methods to do the dot products with blas or at least in C. 211 #Create Theano methods to do the dot products with blas or at least in C.
212 self.M=theano.Module() 212 self.M=theano.Module()
213 weights = T.dmatrix('weights') 213 weights = T.dmatrix('weights')
214 self.M.hid = T.dmatrix('hid') 214 self.M.hid = T.dmatrix('hid')
215 oldval = T.dmatrix('oldval') 215 oldval = T.dmatrix('oldval')
216 resultin = oldval + T.dot(self.M.hid,weights) 216 resultin = oldval + T.dot(self.M.hid,weights)
217 result = T.dot(self.M.hid,weights) 217 result = T.dot(self.M.hid,weights)
218 218
219 self.M.dotin = theano.Method([weights,oldval],resultin) 219 self.M.dotin = theano.Method([weights,oldval],resultin)
220 self.M.dot = theano.Method([weights],result) 220 self.M.dot = theano.Method([weights],result)
221 self.m = self.M.make() 221 self.m = self.M.make()
222 222
223 def make_node(self, idx_list, input_list, hidd, weights_list): 223 def make_node(self, idx_list, input_list, hidd, weights_list):
224 idx_list = Checkidx_list(idx_list) 224 idx_list = Checkidx_list(idx_list)
225 hidd = Checkhidd(hidd) 225 hidd = Checkhidd(hidd)
226 weights_list = Checkweights_list(weights_list) 226 weights_list = Checkweights_list(weights_list)
227 return Apply(self, [idx_list] + [input_list] +[hidd] + weights_list,[T.dmatrix()]) 227 return Apply(self, [idx_list] + [input_list] +[hidd] + weights_list,[T.dmatrix()])
228 228
229 def perform(self, node, args, (z,)): 229 def perform(self, node, args, (z,)):
230 230
231 idx_list = abs(args[0]) 231 idx_list = abs(args[0])
232 self.m.hid = args[2] 232 self.m.hid = args[2]
233 233
234 batchsize = (self.m.hid.shape)[0] 234 batchsize = (self.m.hid.shape)[0]
235 n_hid = self.m.hid.shape[1] 235 n_hid = self.m.hid.shape[1]
236 if max(idx_list) >= len(args)-3+1 : 236 if max(idx_list) >= len(args)-3+1 :
237 raise NotImplementedError('index superior to weight list length',idx_list) 237 raise NotImplementedError('index superior to weight list length',idx_list)
238 if len(idx_list) != len(args[1]) : 238 if len(idx_list) != len(args[1]) :
239 raise NotImplementedError('size of index different of inputs list size',idx_list) 239 raise NotImplementedError('size of index different of inputs list size',idx_list)
240 for i in range(len(args)-3): 240 for i in range(len(args)-3):
241 if (args[3+i].shape)[0] != n_hid: 241 if (args[3+i].shape)[0] != n_hid:
242 raise NotImplementedError('different length of hidden in the weights list',args[3+i].shape) 242 raise NotImplementedError('different length of hidden in the weights list',args[3+i].shape)
243 243
244 zcalc = [False for i in idx_list] 244 zcalc = [False for i in idx_list]
245 z[0] = [None for i in idx_list] 245 z[0] = [None for i in idx_list]
246 246
247 for i in range(len(idx_list)): 247 for i in range(len(idx_list)):
248 if idx_list[i]>0: 248 if idx_list[i]>0:
249 if zcalc[i]: 249 if zcalc[i]:
250 z[0][i] = self.m.dotin(args[3+int(idx_list[i]-1)],z[0][i]) 250 z[0][i] = self.m.dotin(args[3+int(idx_list[i]-1)],z[0][i])
251 else: 251 else:
252 z[0][i] = self.m.dot(args[3+int(idx_list[i]-1)]) 252 z[0][i] = self.m.dot(args[3+int(idx_list[i]-1)])
253 zcalc[i] = True 253 zcalc[i] = True
254 254
255 for i in range(len(idx_list)): 255 for i in range(len(idx_list)):
256 if not zcalc[i]: 256 if not zcalc[i]:
257 shp = args[1][int(idx_list[i]-1)].shape 257 shp = args[1][int(idx_list[i]-1)].shape
258 z[0][i] = numpy.zeros((batchsize,shp[1])) 258 z[0][i] = numpy.zeros((batchsize,shp[1]))
259 259
260 z[0] = numpy.concatenate(z[0],1) 260 z[0] = numpy.concatenate(z[0],1)
261 261
262 def grad(self, args, gz): 262 def grad(self, args, gz):
263 gradi = ScanDotDecGrad()(args,gz) 263 gradi = ScanDotDecGrad()(args,gz)
264 if type(gradi) != list: 264 if type(gradi) != list:
265 return [None, None] + [gradi] 265 return [None, None] + [gradi]
266 else: 266 else:
267 return [None, None] + gradi 267 return [None, None] + gradi
268 268
269 def __hash__(self): 269 def __hash__(self):
270 return hash(ScanDotDec)^73568 270 return hash(ScanDotDec)^73568
271 271
272 def __str__(self): 272 def __str__(self):
273 return "ScanDotDec" 273 return "ScanDotDec"
274 274
275 scandotdec=ScanDotDec() 275 scandotdec=ScanDotDec()
276 276
277 class ScanDotDecGrad(Op): 277 class ScanDotDecGrad(Op):
278 """This Op computes the gradient wrt the weights for ScanDotDec""" 278 """This Op computes the gradient wrt the weights for ScanDotDec"""
279 def __init__(self): 279 def __init__(self):
280 self.M=theano.Module() 280 self.M=theano.Module()
281 gout = T.dmatrix('gout') 281 gout = T.dmatrix('gout')
282 self.M.hidt = T.dmatrix('hid') 282 self.M.hidt = T.dmatrix('hid')
283 oldval = T.dmatrix('oldval') 283 oldval = T.dmatrix('oldval')
284 resultin1 = oldval + T.dot(self.M.hidt,gout) 284 resultin1 = oldval + T.dot(self.M.hidt,gout)
285 result1 = T.dot(self.M.hidt,gout) 285 result1 = T.dot(self.M.hidt,gout)
286 weights = T.dmatrix('weights') 286 weights = T.dmatrix('weights')
287 weights2 = T.transpose(weights) 287 weights2 = T.transpose(weights)
288 resultin2 = oldval + T.dot(gout,weights2) 288 resultin2 = oldval + T.dot(gout,weights2)
289 result2 = T.dot(gout,weights2) 289 result2 = T.dot(gout,weights2)
290 290
291 self.M.dotin1 = theano.Method([gout,oldval],resultin1) 291 self.M.dotin1 = theano.Method([gout,oldval],resultin1)
292 self.M.dot1 = theano.Method([gout],result1) 292 self.M.dot1 = theano.Method([gout],result1)
293 self.M.dotin2 = theano.Method([gout,weights,oldval],resultin2) 293 self.M.dotin2 = theano.Method([gout,weights,oldval],resultin2)
294 self.M.dot2 = theano.Method([gout,weights],result2) 294 self.M.dot2 = theano.Method([gout,weights],result2)
295 self.m = self.M.make() 295 self.m = self.M.make()
296 296
297 297
298 def make_node(self, args, g_out): 298 def make_node(self, args, g_out):
299 idx_list = Checkidx_list(args[0]) 299 idx_list = Checkidx_list(args[0])
300 hidd = Checkhidd(args[2]) 300 hidd = Checkhidd(args[2])
301 weights_list = Checkweights_list(args[3:]) 301 weights_list = Checkweights_list(args[3:])
302 return Apply(self, args + g_out, [T.dmatrix() for i in xrange(2,len(args))]) 302 return Apply(self, args + g_out, [T.dmatrix() for i in xrange(2,len(args))])
303 303
304 def perform(self, node, args, z): 304 def perform(self, node, args, z):
305 idx_list = abs(args[0]) 305 idx_list = abs(args[0])
306 self.m.hidt = args[2].T 306 self.m.hidt = args[2].T
307 307
308 batchsize = (self.m.hidt.shape)[1] 308 batchsize = (self.m.hidt.shape)[1]
309 n_hid = self.m.hidt.shape[0] 309 n_hid = self.m.hidt.shape[0]
310 if max(idx_list) >= len(args)-4+1 : 310 if max(idx_list) >= len(args)-4+1 :
311 raise NotImplementedError('index superior to weight list length',idx_list) 311 raise NotImplementedError('index superior to weight list length',idx_list)
312 if len(idx_list) != len(args[1]) : 312 if len(idx_list) != len(args[1]) :
313 raise NotImplementedError('size of index different of inputs list size',idx_list) 313 raise NotImplementedError('size of index different of inputs list size',idx_list)
314 for i in range(len(args)-4): 314 for i in range(len(args)-4):
315 if (args[3+i].shape)[0] != n_hid: 315 if (args[3+i].shape)[0] != n_hid:
316 raise NotImplementedError('different length of hidden in the weights list',args[3+i].shape) 316 raise NotImplementedError('different length of hidden in the weights list',args[3+i].shape)
317 317
318 zidx=numpy.zeros((len(idx_list)+1)) 318 zidx=numpy.zeros((len(idx_list)+1))
319 319
320 for i in range(len(idx_list)): 320 for i in range(len(idx_list)):
321 if idx_list[i] == 0: 321 if idx_list[i] == 0:
322 zidx[i+1] = (args[1][i].shape)[1] 322 zidx[i+1] = (args[1][i].shape)[1]
323 else: 323 else:
324 zidx[i+1] = (args[3+idx_list[i]-1].shape)[1] 324 zidx[i+1] = (args[3+idx_list[i]-1].shape)[1]
325 325
326 zidx=zidx.cumsum() 326 zidx=zidx.cumsum()
327 hidcalc = False 327 hidcalc = False
328 zcalc = [False for i in range((len(args)-4))] 328 zcalc = [False for i in range((len(args)-4))]
329 329
330 for i in range(len(idx_list)): 330 for i in range(len(idx_list)):
331 if idx_list[i]>0: 331 if idx_list[i]>0:
332 if zcalc[int(idx_list[i])-1]: 332 if zcalc[int(idx_list[i])-1]:
333 z[int(idx_list[i])][0] = self.m.dotin1(args[-1][:,zidx[i]:zidx[i+1]],z[int(idx_list[i])][0]) 333 z[int(idx_list[i])][0] = self.m.dotin1(args[-1][:,zidx[i]:zidx[i+1]],z[int(idx_list[i])][0])
334 else: 334 else:
335 z[int(idx_list[i])][0] = self.m.dot1(args[-1][:,zidx[i]:zidx[i+1]]) 335 z[int(idx_list[i])][0] = self.m.dot1(args[-1][:,zidx[i]:zidx[i+1]])
336 zcalc[int(idx_list[i])-1] = True 336 zcalc[int(idx_list[i])-1] = True
337 if hidcalc: 337 if hidcalc:
338 z[0][0] = self.m.dotin2(args[-1][:,zidx[i]:zidx[i+1]],args[3+int(idx_list[i]-1)],z[0][0]) 338 z[0][0] = self.m.dotin2(args[-1][:,zidx[i]:zidx[i+1]],args[3+int(idx_list[i]-1)],z[0][0])
339 else: 339 else:
340 z[0][0] = self.m.dot2(args[-1][:,zidx[i]:zidx[i+1]],args[3+int(idx_list[i]-1)]) 340 z[0][0] = self.m.dot2(args[-1][:,zidx[i]:zidx[i+1]],args[3+int(idx_list[i]-1)])
341 hidcalc = True 341 hidcalc = True
342 342
343 if not hidcalc: 343 if not hidcalc:
344 z[0][0] = numpy.zeros((self.m.hidt.shape[1],self.m.hidt.shape[0])) 344 z[0][0] = numpy.zeros((self.m.hidt.shape[1],self.m.hidt.shape[0]))
345 345
346 for i in range((len(args)-4)): 346 for i in range((len(args)-4)):
347 if not zcalc[i]: 347 if not zcalc[i]:
348 shp = args[3+i].shape 348 shp = args[3+i].shape
349 z[i+1][0] = numpy.zeros(shp) 349 z[i+1][0] = numpy.zeros(shp)
350 350
351 351
352 def __hash__(self): 352 def __hash__(self):
353 return hash(ScanDotDecGrad)^87445 353 return hash(ScanDotDecGrad)^87445
354 354
355 def __str__(self): 355 def __str__(self):
356 return "ScanDotDecGrad" 356 return "ScanDotDecGrad"
357 357
358 # DAA input noise------------------------------------ 358 # DAA input noise------------------------------------
359 class ScanNoise(Op): 359 class ScanNoise(Op):
360 """This Op takes an index list (as tensor.ivector), a list of matrices representing 360 """This Op takes an index list (as tensor.ivector), a list of matrices representing
361 the available inputs (as theano.generic), a probability of individual bit masking and 361 the available inputs (as theano.generic), a probability of individual bit masking and
362 a probability of modality masking. It will return the inputs list with randoms zeros entry 362 a probability of modality masking. It will return the inputs list with randoms zeros entry
363 and the index list with some positive values changed to negative values (groups masking)""" 363 and the index list with some positive values changed to negative values (groups masking)"""
364 def __init__(self, seed = 1): 364 def __init__(self, seed = 1):
365 self.M=theano.Module() 365 self.M=theano.Module()
366 self.M.rand = T.RandomStreams(seed) 366 self.M.rand = T.RandomStreams(seed)
367 self.seed = seed 367 self.seed = seed
368 mat = T.matrix('mat') 368 mat = T.matrix('mat')
369 noise_level_bit = T.dscalar('noise_level_bit') 369 noise_level_bit = T.dscalar('noise_level_bit')
370 noise_level_group = T.dscalar('noise_level_group') 370 noise_level_group = T.dscalar('noise_level_group')
371 self.M.out1 = self.M.rand.binomial(T.shape(mat), 1, 1 - noise_level_bit) * mat 371 self.M.out1 = self.M.rand.binomial(T.shape(mat), 1, 1 - noise_level_bit) * mat
372 self.M.out2 = self.M.rand.binomial((1,1), 1, 1 - noise_level_group) 372 self.M.out2 = self.M.rand.binomial((1,1), 1, 1 - noise_level_group)
373 373
374 self.M.noisify_bit = theano.Method([mat,noise_level_bit],self.M.out1) 374 self.M.noisify_bit = theano.Method([mat,noise_level_bit],self.M.out1)
375 self.M.noisify_group_bool = theano.Method([noise_level_group],self.M.out2) 375 self.M.noisify_group_bool = theano.Method([noise_level_group],self.M.out2)
376 self.R = self.M.make() 376 self.R = self.M.make()
377 self.R.rand.initialize() 377 self.R.rand.initialize()
378 378
379 def make_node(self, idx_list, inputs_list, noise_level_bit, noise_level_group): 379 def make_node(self, idx_list, inputs_list, noise_level_bit, noise_level_group):
380 idx_list = Checkidx_list(idx_list) 380 idx_list = Checkidx_list(idx_list)
381 return Apply(self, [idx_list] + [inputs_list] + [noise_level_bit] + [noise_level_group],\ 381 return Apply(self, [idx_list] + [inputs_list] + [noise_level_bit] + [noise_level_group],\
382 [T.ivector(), theano.generic()]) 382 [T.ivector(), theano.generic()])
383 383
384 def perform(self, node, (idx_list,inputs_list,noise_level_bit,noise_level_group), (y,z)): 384 def perform(self, node, (idx_list,inputs_list,noise_level_bit,noise_level_group), (y,z)):
385 385
386 if len(idx_list) != len(inputs_list) : 386 if len(idx_list) != len(inputs_list) :
387 raise NotImplementedError('size of index different of inputs list size',idx_list) 387 raise NotImplementedError('size of index different of inputs list size',idx_list)
388 388
389 y[0] = numpy.asarray([-i if (i>0 and not(self.R.noisify_group_bool(noise_level_group))) else i for i in idx_list]) 389 y[0] = numpy.asarray([-i if (i>0 and not(self.R.noisify_group_bool(noise_level_group))) else i for i in idx_list])
390 z[0] = [(self.R.noisify_bit(inputs_list[i],noise_level_bit) if y[0][i]>0 else numpy.zeros((inputs_list[i].shape)))\ 390 z[0] = [(self.R.noisify_bit(inputs_list[i],noise_level_bit) if y[0][i]>0 else numpy.zeros((inputs_list[i].shape)))\
391 for i in range(len(inputs_list))] 391 for i in range(len(inputs_list))]
392 392
393 def grad(self,args,gz): 393 def grad(self,args,gz):
394 return [None,None,None,None] 394 return [None,None,None,None]
395 395
396 396
397 def __hash__(self): 397 def __hash__(self):
398 return hash(ScanNoise)^hash(self.seed)^hash(self.R.rand)^12254 398 return hash(ScanNoise)^hash(self.seed)^hash(self.R.rand)^12254
399 399
400 def __str__(self): 400 def __str__(self):
401 return "ScanNoise" 401 return "ScanNoise"
402 402
403 scannoise=ScanNoise() 403 scannoise=ScanNoise()
404 404
405 # Total input matrix construction------------------------------------ 405 # Total input matrix construction------------------------------------
406 class ScanInputs(Op): 406 class ScanInputs(Op):
407 """This Op takes an index list (as tensor.ivector) and a list of matrices representing 407 """This Op takes an index list (as tensor.ivector) and a list of matrices representing
408 the available inputs (as theano.generic). It will construct the appropriate tensor.dmatrix 408 the available inputs (as theano.generic). It will construct the appropriate tensor.dmatrix
409 to compare to the reconstruction obtained with ScanDotDec""" 409 to compare to the reconstruction obtained with ScanDotDec"""
410 def make_node(self, idx_list, inputs_list): 410 def make_node(self, idx_list, inputs_list):
411 idx_list = Checkidx_list(idx_list) 411 idx_list = Checkidx_list(idx_list)
412 return Apply(self, [idx_list] + [inputs_list],[T.dmatrix()]) 412 return Apply(self, [idx_list] + [inputs_list],[T.dmatrix()])
413 413
414 def perform(self, node, (idx_list, inputs_list), (z,)): 414 def perform(self, node, (idx_list, inputs_list), (z,)):
415 415
416 if len(idx_list) != len(inputs_list): 416 if len(idx_list) != len(inputs_list):
417 raise NotImplementedError('size of index different of inputs list size',idx_list) 417 raise NotImplementedError('size of index different of inputs list size',idx_list)
418 418
419 for i in range(len(idx_list)): 419 for i in range(len(idx_list)):
420 if idx_list[i] == 0: 420 if idx_list[i] == 0:
421 inputs_list[i] = 0 * inputs_list[i] 421 inputs_list[i] = 0 * inputs_list[i]
422 422
423 z[0] = numpy.concatenate(inputs_list,1) 423 z[0] = numpy.concatenate(inputs_list,1)
424 424
425 def grad(self,args,gz): 425 def grad(self,args,gz):
426 return [None,None] 426 return [None,None]
427 427
428 def __hash__(self): 428 def __hash__(self):
429 return hash(ScanInputs)^75902 429 return hash(ScanInputs)^75902
430 430
431 def __str__(self): 431 def __str__(self):
432 return "ScanInputs" 432 return "ScanInputs"
433 433
434 scaninputs=ScanInputs() 434 scaninputs=ScanInputs()
435 435
436 # Decoding bias vector construction------------------------------------ 436 # Decoding bias vector construction------------------------------------
437 class ScanBiasDec(Op): 437 class ScanBiasDec(Op):
438 """This Op takes an index list (as tensor.ivector), a list of matrices representing 438 """This Op takes an index list (as tensor.ivector), a list of matrices representing
439 the available inputs (as theano.generic) and the decoding bias tensor.dvector. 439 the available inputs (as theano.generic) and the decoding bias tensor.dvector.
440 It will construct the appropriate bias tensor.dvector 440 It will construct the appropriate bias tensor.dvector
441 to add to the reconstruction obtained with ScanDotDec""" 441 to add to the reconstruction obtained with ScanDotDec"""
442 def make_node(self, idx_list, input_list, bias_list): 442 def make_node(self, idx_list, input_list, bias_list):
443 idx_list = Checkidx_list(idx_list) 443 idx_list = Checkidx_list(idx_list)
444 bias_list = Checkbias_list(bias_list) 444 bias_list = Checkbias_list(bias_list)
445 return Apply(self, [idx_list] + [input_list] + bias_list, [T.dvector()]) 445 return Apply(self, [idx_list] + [input_list] + bias_list, [T.dvector()])
446 446
447 def perform(self, node, args, (z,)): 447 def perform(self, node, args, (z,)):
448 idx_list = abs(args[0]) 448 idx_list = abs(args[0])
449 449
450 if max(idx_list) >= (len(args)-2)+1 : 450 if max(idx_list) >= (len(args)-2)+1 :
451 raise NotImplementedError('index superior to bias list length',idx_list) 451 raise NotImplementedError('index superior to bias list length',idx_list)
452 if len(idx_list) != len(args[1]) : 452 if len(idx_list) != len(args[1]) :
453 raise NotImplementedError('size of index different of inputs list size',idx_list) 453 raise NotImplementedError('size of index different of inputs list size',idx_list)
454 z[0] = [args[idx_list[i]+1] if idx_list[i] != 0 else numpy.zeros(args[1][i].shape[1]) \ 454 z[0] = [args[idx_list[i]+1] if idx_list[i] != 0 else numpy.zeros(args[1][i].shape[1]) \
455 for i in range(len(idx_list))] 455 for i in range(len(idx_list))]
456 z[0] = numpy.concatenate(z[0],1) 456 z[0] = numpy.concatenate(z[0],1)
457 457
458 def __hash__(self): 458 def __hash__(self):
459 return hash(ScanBiasDec)^60056 459 return hash(ScanBiasDec)^60056
460 460
461 def grad(self,args,gz): 461 def grad(self,args,gz):
462 gradi = ScanBiasDecGrad()(args,gz) 462 gradi = ScanBiasDecGrad()(args,gz)
463 if type(gradi) != list: 463 if type(gradi) != list:
464 return [None, None] + [gradi] 464 return [None, None] + [gradi]
465 else: 465 else:
466 return [None, None] + gradi 466 return [None, None] + gradi
467 467
468 def __str__(self): 468 def __str__(self):
469 return "ScanBiasDec" 469 return "ScanBiasDec"
470 470
471 scanbiasdec=ScanBiasDec() 471 scanbiasdec=ScanBiasDec()
472 472
473 class ScanBiasDecGrad(Op): 473 class ScanBiasDecGrad(Op):
474 """This Op computes the gradient wrt the bias for ScanBiasDec""" 474 """This Op computes the gradient wrt the bias for ScanBiasDec"""
475 def make_node(self, args, g_out): 475 def make_node(self, args, g_out):
476 idx_list = Checkidx_list(args[0]) 476 idx_list = Checkidx_list(args[0])
477 bias_list = Checkbias_list(args[2:]) 477 bias_list = Checkbias_list(args[2:])
478 return Apply(self, args + g_out, [T.dvector() for i in range(len(args)-2)]) 478 return Apply(self, args + g_out, [T.dvector() for i in range(len(args)-2)])
479 479
480 def perform(self, node, args, z): 480 def perform(self, node, args, z):
481 idx_list = abs(args[0]) 481 idx_list = abs(args[0])
482 482
483 if max(idx_list) >= (len(args)-3)+1 : 483 if max(idx_list) >= (len(args)-3)+1 :
484 raise NotImplementedError('index superior to bias list length',idx_list) 484 raise NotImplementedError('index superior to bias list length',idx_list)
485 if len(idx_list) != len(args[1]) : 485 if len(idx_list) != len(args[1]) :
486 raise NotImplementedError('size of index different of inputs list size',idx_list) 486 raise NotImplementedError('size of index different of inputs list size',idx_list)
487 487
488 zidx=numpy.zeros((len(idx_list)+1)) 488 zidx=numpy.zeros((len(idx_list)+1))
489 for i in range(len(idx_list)): 489 for i in range(len(idx_list)):
490 if idx_list[i] == 0: 490 if idx_list[i] == 0:
491 zidx[i+1] = (args[1][i].shape)[1] 491 zidx[i+1] = (args[1][i].shape)[1]
492 else: 492 else:
493 zidx[i+1] = (args[2+idx_list[i]-1].size) 493 zidx[i+1] = (args[2+idx_list[i]-1].size)
494 zidx=zidx.cumsum() 494 zidx=zidx.cumsum()
495 zcalc = [False for i in range((len(args)-3))] 495 zcalc = [False for i in range((len(args)-3))]
496 496
497 for i in range(len(idx_list)): 497 for i in range(len(idx_list)):
498 if idx_list[i]>0: 498 if idx_list[i]>0:
499 if zcalc[int(idx_list[i])-1]: 499 if zcalc[int(idx_list[i])-1]:
500 z[int(idx_list[i])-1][0] += args[-1][zidx[i]:zidx[i+1]] 500 z[int(idx_list[i])-1][0] += args[-1][zidx[i]:zidx[i+1]]
501 else: 501 else:
502 z[int(idx_list[i])-1][0] = args[-1][zidx[i]:zidx[i+1]] 502 z[int(idx_list[i])-1][0] = args[-1][zidx[i]:zidx[i+1]]
503 zcalc[int(idx_list[i])-1] = True 503 zcalc[int(idx_list[i])-1] = True
504 504
505 for i in range((len(args)-3)): 505 for i in range((len(args)-3)):
506 if not zcalc[i]: 506 if not zcalc[i]:
507 shp = args[2+i].size 507 shp = args[2+i].size
508 z[i][0] = numpy.zeros(shp) 508 z[i][0] = numpy.zeros(shp)
509 509
510 510
511 def __hash__(self): 511 def __hash__(self):
512 return hash(ScanBiasDecGrad)^41256 512 return hash(ScanBiasDecGrad)^41256
513 513
514 def __str__(self): 514 def __str__(self):
515 return "ScanBiasDecGrad" 515 return "ScanBiasDecGrad"
516 516
517 # Mask construction------------------------------------ 517 # Mask construction------------------------------------
518 class ScanMask(Op): 518 class ScanMask(Op):
519 """This Op takes an index list (as tensor.ivector) and a list of weigths. 519 """This Op takes an index list (as tensor.ivector) and a list of weigths.
520 It will construct a list of T.iscalar representing the Mask 520 It will construct a list of T.iscalar representing the Mask
521 to do the correct regularisation on the weigths""" 521 to do the correct regularisation on the weigths"""
522 def __init__(self,encbool=True): 522 def __init__(self,encbool=True):
523 self.encbool = encbool 523 self.encbool = encbool
524 524
525 def make_node(self, idx_list, weights_list): 525 def make_node(self, idx_list, weights_list):
526 idx_list = Checkidx_list(idx_list) 526 idx_list = Checkidx_list(idx_list)
527 weights_list = Checkweights_list(weights_list) 527 weights_list = Checkweights_list(weights_list)
528 return Apply(self, [idx_list] + weights_list, [T.iscalar() for i in range(len(weights_list))]) 528 return Apply(self, [idx_list] + weights_list, [T.iscalar() for i in range(len(weights_list))])
529 529
530 def perform(self, node, args, z): 530 def perform(self, node, args, z):
531 if self.encbool: 531 if self.encbool:
532 idx_list = args[0] 532 idx_list = args[0]
533 dim = 1 533 dim = 1
534 else: 534 else:
535 idx_list = abs(args[0]) 535 idx_list = abs(args[0])
536 dim = 0 536 dim = 0
537 n_hid = args[1].shape[dim] 537 n_hid = args[1].shape[dim]
538 538
539 if max(idx_list) >= (len(args)-1)+1 : 539 if max(idx_list) >= (len(args)-1)+1 :
540 raise NotImplementedError('index superior to weights list length',idx_listdec) 540 raise NotImplementedError('index superior to weights list length',idx_listdec)
541 for i in range(len(args)-1): 541 for i in range(len(args)-1):
542 if args[1+i].shape[dim] != n_hid: 542 if args[1+i].shape[dim] != n_hid:
543 raise NotImplementedError('different length of hidden in the encoding weights list',args[1+i].shape) 543 raise NotImplementedError('different length of hidden in the encoding weights list',args[1+i].shape)
544 544
545 for i in range(len(args[1:])): 545 for i in range(len(args[1:])):
546 z[i][0] = numpy.asarray((idx_list == i+1).sum(),dtype='int32') 546 z[i][0] = numpy.asarray((idx_list == i+1).sum(),dtype='int32')
547 547
548 def __hash__(self): 548 def __hash__(self):
549 return hash(ScanMask)^hash(self.encbool)^11447 549 return hash(ScanMask)^hash(self.encbool)^11447
550 550
551 def grad(self,args,gz): 551 def grad(self,args,gz):
552 return [None] * len(args) 552 return [None] * len(args)
553 553
554 def __str__(self): 554 def __str__(self):
555 if self.encbool: 555 if self.encbool:
556 string = "Enc" 556 string = "Enc"
557 else: 557 else:
558 string = "Dec" 558 string = "Dec"
559 return "ScanMask" + string 559 return "ScanMask" + string
560 560
561 scanmaskenc=ScanMask(True) 561 scanmaskenc=ScanMask(True)
562 scanmaskdec=ScanMask(False) 562 scanmaskdec=ScanMask(False)