Mercurial > pylearn
comparison nnet_ops.py @ 443:060c12314734
Hopefully last bugfix in Softmax
author | Pascal Lamblin <lamblinp@iro.umontreal.ca> |
---|---|
date | Fri, 22 Aug 2008 17:33:06 -0400 |
parents | b3315b252824 |
children | 23960ee12b52 |
comparison
equal
deleted
inserted
replaced
442:b3315b252824 | 443:060c12314734 |
---|---|
103 sm = numpy.zeros_like(x) | 103 sm = numpy.zeros_like(x) |
104 for i in xrange(sm.shape[0]): | 104 for i in xrange(sm.shape[0]): |
105 row = x[i] + b | 105 row = x[i] + b |
106 sm[i] = numpy.exp(row - numpy.max(row)) | 106 sm[i] = numpy.exp(row - numpy.max(row)) |
107 sm[i] *= 1.0 / numpy.sum(sm[i]) | 107 sm[i] *= 1.0 / numpy.sum(sm[i]) |
108 output_storage[0][0] = nll | 108 output_storage[0][0] = sm |
109 | 109 |
110 def grad(self, (x, b), (g_sm,)): | 110 def grad(self, (x, b), (g_sm,)): |
111 sm = softmax_with_bias(x, b) | 111 sm = softmax_with_bias(x, b) |
112 dx = SoftmaxWithBiasDx()(g_sm, sm) | 112 dx = SoftmaxWithBiasDx()(g_sm, sm) |
113 db = tensor.sum(dx, axis = 0) | 113 db = tensor.sum(dx, axis = 0) |
255 dy, sm = input_storage | 255 dy, sm = input_storage |
256 dx = numpy.zeros_like(sm) | 256 dx = numpy.zeros_like(sm) |
257 #dx[i,j] = - (\sum_k dy[i,k] sm[i,k]) sm[i,j] + dy[i,j] sm[i,j] | 257 #dx[i,j] = - (\sum_k dy[i,k] sm[i,k]) sm[i,j] + dy[i,j] sm[i,j] |
258 for i in xrange(sm.shape[0]): | 258 for i in xrange(sm.shape[0]): |
259 dy_times_sm_i = dy[i] * sm[i] | 259 dy_times_sm_i = dy[i] * sm[i] |
260 dx[i] = dy_times_sm - sum(dy_times_sm_i) * y[i] | 260 dx[i] = dy_times_sm_i - sum(dy_times_sm_i) * sm[i] |
261 output_storage[0][0] = dx | 261 output_storage[0][0] = dx |
262 | 262 |
263 def grad(self, *args): | 263 def grad(self, *args): |
264 raise NotImplementedError() | 264 raise NotImplementedError() |
265 | 265 |
315 { | 315 { |
316 dx_i[j * Sdx] -= sum_dy_times_sm * sm_i[j * Ssm]; | 316 dx_i[j * Sdx] -= sum_dy_times_sm * sm_i[j * Ssm]; |
317 } | 317 } |
318 } | 318 } |
319 ''' % dict(locals(), **sub) | 319 ''' % dict(locals(), **sub) |
320 | |
321 def softmax(x, **kwargs): | |
322 b = tensor.zeros_like(x[0,:]) | |
323 return softmax_with_bias(x, b, **kwargs) | |
320 | 324 |
321 | 325 |
322 class CrossentropySoftmax1HotWithBias(theano.Op): | 326 class CrossentropySoftmax1HotWithBias(theano.Op): |
323 """A special compound L{Op} for the output of neural-net classifiers. | 327 """A special compound L{Op} for the output of neural-net classifiers. |
324 | 328 |
574 raise TypeError("Expected a matrix as input") | 578 raise TypeError("Expected a matrix as input") |
575 x = tensor.as_tensor(mat) | 579 x = tensor.as_tensor(mat) |
576 y = tensor.as_tensor(self.val) | 580 y = tensor.as_tensor(self.val) |
577 if x.type.dtype != y.type.dtype: | 581 if x.type.dtype != y.type.dtype: |
578 TypeError("the value to prepend don't have the same type as the matrix") | 582 TypeError("the value to prepend don't have the same type as the matrix") |
579 | 583 |
580 node = theano.Apply(op=self, inputs=[mat], outputs=[tensor.matrix()]) | 584 node = theano.Apply(op=self, inputs=[mat], outputs=[tensor.matrix()]) |
581 return node | 585 return node |
582 | 586 |
583 def perform(self, node, (mat, ), (output, )): | 587 def perform(self, node, (mat, ), (output, )): |
584 new_shape=(mat.shape[0],mat.shape[1]+1) | 588 new_shape=(mat.shape[0],mat.shape[1]+1) |
597 out[:,1:]=mat | 601 out[:,1:]=mat |
598 | 602 |
599 def grad(self, (mat,), (goutput,)): | 603 def grad(self, (mat,), (goutput,)): |
600 return goutput[:,1:] | 604 return goutput[:,1:] |
601 | 605 |
602 class Prepend_scalar_to_each_row(theano.Op): | 606 class Prepend_scalar_to_each_row(theano.Op): |
603 def make_node(self, val, mat): | 607 def make_node(self, val, mat): |
604 #check type of input | 608 #check type of input |
605 if isinstance(val, float): | 609 if isinstance(val, float): |
606 val = scalar.constant(val) | 610 val = scalar.constant(val) |
607 if not isinstance(mat,theano.Result) or not mat.type==tensor.matrix().type: | 611 if not isinstance(mat,theano.Result) or not mat.type==tensor.matrix().type: |
608 raise TypeError("Expected a matrix as input") | 612 raise TypeError("Expected a matrix as input") |
609 x = tensor.as_tensor(mat) | 613 x = tensor.as_tensor(mat) |
610 y = tensor.as_tensor(val) | 614 y = tensor.as_tensor(val) |
611 if x.type.dtype != y.type.dtype: | 615 if x.type.dtype != y.type.dtype: |
612 TypeError("the value to prepend don't have the same type as the matrix") | 616 TypeError("the value to prepend don't have the same type as the matrix") |
613 | 617 |
614 node = theano.Apply(op=self, inputs=[val,mat], outputs=[tensor.matrix()]) | 618 node = theano.Apply(op=self, inputs=[val,mat], outputs=[tensor.matrix()]) |
615 return node | 619 return node |
616 | 620 |
617 def perform(self, node, (val,mat), (output, )): | 621 def perform(self, node, (val,mat), (output, )): |
618 new_shape=(mat.shape[0],mat.shape[1]+1) | 622 new_shape=(mat.shape[0],mat.shape[1]+1) |
657 output[0]=ret | 661 output[0]=ret |
658 | 662 |
659 def grad(self, (theta, A, B), (gtheta,)): | 663 def grad(self, (theta, A, B), (gtheta,)): |
660 raise NotImplementedError() | 664 raise NotImplementedError() |
661 | 665 |
662 | 666 |