Mercurial > pylearn

comparison nnet_ops.py @ 443:060c12314734

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Hopefully last bugfix in Softmax
author Pascal Lamblin <lamblinp@iro.umontreal.ca>
date Fri, 22 Aug 2008 17:33:06 -0400
parents b3315b252824
children 23960ee12b52
comparison
equal deleted inserted replaced
442:b3315b252824 443:060c12314734
103 sm = numpy.zeros_like(x) 103 sm = numpy.zeros_like(x)
104 for i in xrange(sm.shape[0]): 104 for i in xrange(sm.shape[0]):
105 row = x[i] + b 105 row = x[i] + b
106 sm[i] = numpy.exp(row - numpy.max(row)) 106 sm[i] = numpy.exp(row - numpy.max(row))
107 sm[i] *= 1.0 / numpy.sum(sm[i]) 107 sm[i] *= 1.0 / numpy.sum(sm[i])
108 output_storage[0][0] = nll 108 output_storage[0][0] = sm
109 109
110 def grad(self, (x, b), (g_sm,)): 110 def grad(self, (x, b), (g_sm,)):
111 sm = softmax_with_bias(x, b) 111 sm = softmax_with_bias(x, b)
112 dx = SoftmaxWithBiasDx()(g_sm, sm) 112 dx = SoftmaxWithBiasDx()(g_sm, sm)
113 db = tensor.sum(dx, axis = 0) 113 db = tensor.sum(dx, axis = 0)
255 dy, sm = input_storage 255 dy, sm = input_storage
256 dx = numpy.zeros_like(sm) 256 dx = numpy.zeros_like(sm)
257 #dx[i,j] = - (\sum_k dy[i,k] sm[i,k]) sm[i,j] + dy[i,j] sm[i,j] 257 #dx[i,j] = - (\sum_k dy[i,k] sm[i,k]) sm[i,j] + dy[i,j] sm[i,j]
258 for i in xrange(sm.shape[0]): 258 for i in xrange(sm.shape[0]):
259 dy_times_sm_i = dy[i] * sm[i] 259 dy_times_sm_i = dy[i] * sm[i]
260 dx[i] = dy_times_sm - sum(dy_times_sm_i) * y[i] 260 dx[i] = dy_times_sm_i - sum(dy_times_sm_i) * sm[i]
261 output_storage[0][0] = dx 261 output_storage[0][0] = dx
262 262
263 def grad(self, *args): 263 def grad(self, *args):
264 raise NotImplementedError() 264 raise NotImplementedError()
265 265
315 { 315 {
316 dx_i[j * Sdx] -= sum_dy_times_sm * sm_i[j * Ssm]; 316 dx_i[j * Sdx] -= sum_dy_times_sm * sm_i[j * Ssm];
317 } 317 }
318 } 318 }
319 ''' % dict(locals(), **sub) 319 ''' % dict(locals(), **sub)
320
321 def softmax(x, **kwargs):
322 b = tensor.zeros_like(x[0,:])
323 return softmax_with_bias(x, b, **kwargs)
320 324
321 325
322 class CrossentropySoftmax1HotWithBias(theano.Op): 326 class CrossentropySoftmax1HotWithBias(theano.Op):
323 """A special compound L{Op} for the output of neural-net classifiers. 327 """A special compound L{Op} for the output of neural-net classifiers.
324 328
574 raise TypeError("Expected a matrix as input") 578 raise TypeError("Expected a matrix as input")
575 x = tensor.as_tensor(mat) 579 x = tensor.as_tensor(mat)
576 y = tensor.as_tensor(self.val) 580 y = tensor.as_tensor(self.val)
577 if x.type.dtype != y.type.dtype: 581 if x.type.dtype != y.type.dtype:
578 TypeError("the value to prepend don't have the same type as the matrix") 582 TypeError("the value to prepend don't have the same type as the matrix")
579 583
580 node = theano.Apply(op=self, inputs=[mat], outputs=[tensor.matrix()]) 584 node = theano.Apply(op=self, inputs=[mat], outputs=[tensor.matrix()])
581 return node 585 return node
582 586
583 def perform(self, node, (mat, ), (output, )): 587 def perform(self, node, (mat, ), (output, )):
584 new_shape=(mat.shape[0],mat.shape[1]+1) 588 new_shape=(mat.shape[0],mat.shape[1]+1)
597 out[:,1:]=mat 601 out[:,1:]=mat
598 602
599 def grad(self, (mat,), (goutput,)): 603 def grad(self, (mat,), (goutput,)):
600 return goutput[:,1:] 604 return goutput[:,1:]
601 605
602 class Prepend_scalar_to_each_row(theano.Op): 606 class Prepend_scalar_to_each_row(theano.Op):
603 def make_node(self, val, mat): 607 def make_node(self, val, mat):
604 #check type of input 608 #check type of input
605 if isinstance(val, float): 609 if isinstance(val, float):
606 val = scalar.constant(val) 610 val = scalar.constant(val)
607 if not isinstance(mat,theano.Result) or not mat.type==tensor.matrix().type: 611 if not isinstance(mat,theano.Result) or not mat.type==tensor.matrix().type:
608 raise TypeError("Expected a matrix as input") 612 raise TypeError("Expected a matrix as input")
609 x = tensor.as_tensor(mat) 613 x = tensor.as_tensor(mat)
610 y = tensor.as_tensor(val) 614 y = tensor.as_tensor(val)
611 if x.type.dtype != y.type.dtype: 615 if x.type.dtype != y.type.dtype:
612 TypeError("the value to prepend don't have the same type as the matrix") 616 TypeError("the value to prepend don't have the same type as the matrix")
613 617
614 node = theano.Apply(op=self, inputs=[val,mat], outputs=[tensor.matrix()]) 618 node = theano.Apply(op=self, inputs=[val,mat], outputs=[tensor.matrix()])
615 return node 619 return node
616 620
617 def perform(self, node, (val,mat), (output, )): 621 def perform(self, node, (val,mat), (output, )):
618 new_shape=(mat.shape[0],mat.shape[1]+1) 622 new_shape=(mat.shape[0],mat.shape[1]+1)
657 output[0]=ret 661 output[0]=ret
658 662
659 def grad(self, (theta, A, B), (gtheta,)): 663 def grad(self, (theta, A, B), (gtheta,)):
660 raise NotImplementedError() 664 raise NotImplementedError()
661 665
662 666