Mercurial > pylearn
annotate nnet_ops.py @ 111:88257dfedf8c
Added another work in progress, for mlp's
| author | bengioy@bengiomac.local |
|---|---|
| date | Wed, 07 May 2008 09:16:04 -0400 |
| parents | 76e5c0f37165 |
| children | 3ef569b92fba |
| rev | line source |
|---|---|
| 24 | 1 import theano |
| 2 from theano import tensor, gof, scalar | |
| 3 import numpy | |
| 4 | |
|
69
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
5 ############ |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
6 # |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
7 # SCALAR OPS |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
8 # |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
9 |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
10 class ScalarSigmoid(scalar.FloatUnaryScalarOp): |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
11 @staticmethod |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
12 def st_impl(x): |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
13 if x < -30.0: |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
14 return 0.0 |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
15 if x > 30.0: |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
16 return 1.0 |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
17 return 1.0 / (1.0 + numpy.exp(-x)) |
| 24 | 18 def impl(self, x): |
|
69
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
19 return ScalarSigmoid.st_impl(x) |
| 24 | 20 def grad(self, (x,), (gz,)): |
|
69
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
21 y = scalar_sigmoid(x) |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
22 return [gz * y * (1.0 - y)] |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
23 def c_foreach(self, (x,), (z,), sub): |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
24 if 'float' in self.inputs[0].dtype: |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
25 return """%(z)s = |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
26 %(x)s < -30.0 |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
27 ? 0.0 |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
28 : %(x)s > 30.0 |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
29 ? 1.0 |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
30 : 1.0 /(1.0+exp(-%(x)s));""" % locals() |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
31 raise NotImplementedError('only floatingpoint is implemented') |
| 24 | 32 scalar_sigmoid = gof.op.constructor(ScalarSigmoid) |
|
69
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
33 Sigmoid, sigmoid, SigmoidInplace, sigmoid_inplace =\ |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
34 tensor.broadcast(ScalarSigmoid, 'Sigmoid') |
| 24 | 35 |
|
69
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
36 class ScalarSoftplus(scalar.FloatUnaryScalarOp): |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
37 @staticmethod |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
38 def static_impl(x): |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
39 if x < -30.0: |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
40 return 0.0 |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
41 if x > 30.0: |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
42 return x |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
43 return numpy.log1p(numpy.exp(x)) |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
44 def impl(self, x): |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
45 return ScalarSoftplus.static_impl(x) |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
46 def grad(self, (x,), (gz,)): |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
47 return [gz * scalar_sigmoid(x)] |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
48 def c_foreach(self, (x,), (z,), sub): |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
49 if 'float' in self.inputs[0].dtype: |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
50 return """%(z)s = |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
51 %(x)s < -30.0 |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
52 ? 0.0 |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
53 : %(x)s > 30.0 |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
54 ? %(x)s |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
55 : log1p(exp(%(x)s));""" % locals() |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
56 raise NotImplementedError('only floating point x is implemented') |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
57 scalar_softplus = gof.op.constructor(ScalarSoftplus) |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
58 Softplus, softplus, SoftplusInplace, softplus_inplace =\ |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
59 tensor.broadcast(ScalarSoftplus, 'Softplus') |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
60 |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
61 |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
62 ############ |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
63 # |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
64 # TENSOR OPS |
|
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
65 # |
| 24 | 66 |
| 67 | |
|
70
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
68 class CrossentropySoftmax1HotWithBias(gof.op.Op): |
|
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
69 """A special compound L{Op} for the output of neural-net classifiers. |
|
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
70 |
|
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
71 @type x: is a matrix of floats (32 or 64) |
|
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
72 @type b: is a [row] vector of floats (32 or 64), length is number of cols in x |
|
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
73 @type y_idx: a [column] vector of int (32 or 64), length is number of rows in x |
|
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
74 |
|
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
75 @precondition: every entry in y_idx is a valid (non-negative) column index into x |
| 24 | 76 |
|
70
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
77 This L{Op} has two outputs: |
|
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
78 - KL(softmax(x+b), y) |
|
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
79 - softmax(x+b) |
| 24 | 80 |
|
70
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
81 |
| 24 | 82 softmax(x[i]) is the i'th distribution over len(x[i]) options |
|
70
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
83 |
|
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
84 y_idx[i] is an integer index, encoding a 1-hot distribution. |
|
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
85 |
|
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
86 In practice, when we're trying to do classification, we have one row in x |
|
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
87 and y_idx per example, and y[i] is the index of the (correct) class of the |
|
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
88 i'th example. |
| 24 | 89 |
| 90 """ | |
|
70
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
91 nin=3 |
| 24 | 92 nout=2 |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
93 def __init__(self, x, b, y_idx, **kwargs): |
| 24 | 94 x = tensor._as_tensor(x) |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
95 b = tensor._as_tensor(b) |
| 24 | 96 y_idx = tensor._as_tensor(y_idx) |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
97 if len(x.broadcastable) != 2 \ |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
98 or x.dtype not in ['float32', 'float64']: |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
99 raise ValueError('x must be 2-d tensor of floats') |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
100 if len(b.broadcastable) != 1 \ |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
101 or x.dtype not in ['float32', 'float64']: |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
102 raise ValueError('x must be 1-d tensor of floats') |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
103 if len(y_idx.broadcastable) != 1 \ |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
104 or y_idx.dtype not in ['int32', 'int64']: |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
105 raise ValueError('x must be 1-d tensor of ints') |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
106 |
| 24 | 107 # TODO: Is this correct? It used to be y, not y_idx |
| 108 nll = tensor.Tensor(x.dtype, y_idx.broadcastable) | |
| 109 # nll = Tensor(x.dtype, y.broadcastable) | |
| 110 sm = tensor.Tensor(x.dtype, x.broadcastable) | |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
111 self.inputs = [x, b, y_idx] |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
112 self.outputs = [nll, sm] |
| 24 | 113 def perform(self): |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
114 x, b, y_idx = [i.data for i in self.inputs] |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
115 if b.shape[0] != x.shape[1]: |
|
70
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
116 raise ValueError('b must have same number of columns as x') |
|
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
117 if y_idx.shape[0] != x.shape[0]: |
|
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
118 raise ValueError('y_idx must have same number of rows as x') |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
119 |
| 24 | 120 sm = numpy.zeros_like(x) # softmax |
| 121 nll = numpy.zeros(x.shape[0]) #nll(y | softmax(x)) | |
| 122 for i in xrange(sm.shape[0]): | |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
123 row = x[i] + b |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
124 sm[i] = numpy.exp(row - numpy.max(row)) #softmax |
| 24 | 125 sm[i] *= 1.0 / numpy.sum(sm[i]) #vector scale |
| 126 nll[i] = -numpy.log( sm[i, y_idx[i]]) #cross-entropy | |
| 127 self.outputs[0].data = nll | |
| 128 self.outputs[1].data = sm | |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
129 def grad(self, (x, b, y_idx), (g_nll, g_sm)): |
| 24 | 130 if g_sm is not None: |
| 131 raise NotImplementedError() | |
|
70
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
132 nll, sm = crossentropy_softmax_1hot_with_bias(x, b, y_idx) |
|
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
133 dx = CrossentropySoftmax1HotWithBiasDx(g_nll, sm, y_idx).outputs[0] |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
134 db = tensor.Sum(dx, axis = [0]).outputs[0] |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
135 return dx, db, None |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
136 |
|
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
137 def c_headers(self): return ['<iostream>'] |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
138 def c_code(self, (x, b, y_idx), (nll, sm), sub): |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
139 # this implementation was lifted from |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
140 # /u/bergstrj/cvs/bergstrj/src/feb07/nn.cxx |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
141 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
142 #TODO: put this into a templated function, in the support code |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
143 #TODO: declare the max of each row as an Op output |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
144 |
| 32 | 145 #TODO: set error messages for failures in this code |
| 146 | |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
147 return """ |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
148 npy_intp* Nx = %(x)s->dimensions; |
| 34 | 149 |
|
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
150 if (%(x)s->nd != 2) |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
151 { |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
152 PyErr_SetString(PyExc_ValueError, "a not 2d tensor"); |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
153 %(fail)s; |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
154 } |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
155 if (%(b)s->nd != 1) |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
156 { |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
157 PyErr_SetString(PyExc_ValueError, "b not 1d tensor"); |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
158 %(fail)s; |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
159 } |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
160 if (%(y_idx)s->nd != 1) |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
161 { |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
162 PyErr_SetString(PyExc_ValueError, "y_idx not 1d tensor"); |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
163 %(fail)s; |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
164 } |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
165 if (%(x)s->descr->type_num != PyArray_DOUBLE) |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
166 { |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
167 PyErr_SetString(PyExc_TypeError, "a not float64"); |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
168 %(fail)s; |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
169 } |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
170 if (%(b)s->descr->type_num != PyArray_DOUBLE) |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
171 { |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
172 PyErr_SetString(PyExc_TypeError, "b not float64"); |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
173 %(fail)s; |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
174 } |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
175 if (%(y_idx)s->descr->type_num != PyArray_INT64) |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
176 { |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
177 PyErr_SetString(PyExc_TypeError, "y_idx not int64"); |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
178 %(fail)s; |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
179 } |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
180 if ((%(x)s->dimensions[1] != %(b)s->dimensions[0]) |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
181 || (%(x)s->dimensions[0] != %(y_idx)s->dimensions[0])) |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
182 { |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
183 PyErr_SetString(PyExc_ValueError, "dimension mismatch in arguments"); |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
184 %(fail)s; |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
185 } |
| 34 | 186 |
|
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
187 if ((NULL == %(nll)s) //initial condition |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
188 || (%(nll)s->dimensions[0] != %(y_idx)s->dimensions[0])) |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
189 { |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
190 if (NULL != %(nll)s) Py_XDECREF(%(nll)s); |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
191 %(nll)s = (PyArrayObject*)PyArray_SimpleNew(1, PyArray_DIMS(%(y_idx)s), type_num_%(x)s); |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
192 if(!%(nll)s) |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
193 { |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
194 PyErr_SetString(PyExc_MemoryError, "failed to alloc nll output"); |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
195 %(fail)s; |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
196 } |
| 34 | 197 } |
|
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
198 if ((NULL == %(sm)s) |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
199 || (%(sm)s->dimensions[0] != %(x)s->dimensions[0]) |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
200 || (%(sm)s->dimensions[1] != %(x)s->dimensions[1])) |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
201 { |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
202 if (NULL != %(sm)s) Py_XDECREF(%(sm)s); |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
203 %(sm)s = (PyArrayObject*)PyArray_SimpleNew(2, PyArray_DIMS(%(x)s), type_num_%(x)s); |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
204 if(!%(sm)s) { |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
205 // The normal cleanup code will take care of %(nll)s |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
206 // Py_XDECREF(%(nll)s); %(nll)s=NULL; |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
207 PyErr_SetString(PyExc_MemoryError, "failed to alloc sm output"); |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
208 %(fail)s |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
209 } |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
210 } |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
211 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
212 for (size_t i = 0; i < Nx[0]; ++i) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
213 { |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
214 size_t j; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
215 double sum = 0.0; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
216 bool discount_max = false; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
217 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
218 const double* __restrict__ x_i = (double*)(%(x)s->data + %(x)s->strides[0] * i); |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
219 const double* __restrict__ b_i = (double*)(%(b)s->data); |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
220 const long int y_i = ((long int*)(%(y_idx)s->data + %(y_idx)s->strides[0] * i))[0]; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
221 double* __restrict__ sm_i = (double*)(%(sm)s->data + %(sm)s->strides[0] * i); |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
222 double* __restrict__ nll_i = (double*)(%(nll)s->data + %(nll)s->strides[0] * i); |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
223 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
224 npy_intp Sx = %(x)s->strides[1]/sizeof(double); |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
225 npy_intp Sb = %(b)s->strides[0]/sizeof(double); |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
226 npy_intp Ssm = %(sm)s->strides[1]/sizeof(double); |
| 24 | 227 |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
228 size_t row_max_j=0; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
229 double row_max = x_i[0] + b_i[0]; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
230 //try to compute sum and sm the easy way |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
231 for (j = 0; j < Nx[1]; ++j) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
232 { |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
233 double row_ij = x_i[j * Sx] + b_i[j * Sb]; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
234 row_max_j = (row_ij > row_max) ? j : row_max_j; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
235 row_max = (row_ij > row_max) ? row_ij : row_max; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
236 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
237 double sm_ij = exp(row_ij); |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
238 sum += sm_ij; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
239 sm_i[j * Ssm] = sm_ij; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
240 } |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
241 if ((0.0 == sum) || (isinf(sum))) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
242 { |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
243 //our cheap trick didn't work... try again and do it better. |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
244 discount_max = true; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
245 sum = 0.0; //reset sum and recompute.... |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
246 for (j = 0; j < Nx[1]; ++j) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
247 { |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
248 double row_ij = x_i[j * Sx] + b_i[j * Sb]; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
249 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
250 double sm_ij = exp(row_ij - row_max); |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
251 sum += sm_ij; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
252 sm_i[j * Ssm] = sm_ij; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
253 } |
| 32 | 254 if ( (0.0 == sum) || (isinf(sum))) |
| 255 { | |
| 256 //that was our best... | |
| 257 %(fail)s; | |
| 258 } | |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
259 //if we still can't sum it up, we're screwed. |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
260 //So far, this assertion has never failed... |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
261 } |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
262 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
263 //cblas_dscal(x.N, 1.0 / sum, &mat_at(s,i,0), s.n); |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
264 double sum_inv = 1.0 / sum; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
265 for (j = 0; j < Nx[1]; ++j) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
266 { |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
267 sm_i[j * Ssm] *= sum_inv; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
268 } |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
269 |
| 32 | 270 if (y_i >= Nx[1]) |
| 271 { | |
| 272 %(fail)s; | |
| 273 } | |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
274 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
275 nll_i[0] = - x_i[y_i*Sx] |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
276 - b_i[y_i*Sb] |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
277 + (discount_max ? row_max : 0.0) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
278 + log(sum); |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
279 //mat_at(y,i,0) = -log( mat_at(s,i,t[i])); //less accurate? |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
280 //mat_at(y,i,0) = - mat_at(x,i,t[i]) - mat_at(b,0,t[i]) + (discount_max ? maxi : 0.0) + log(sum); |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
281 } |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
282 """ % dict(locals(), **sub) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
283 |
|
70
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
284 crossentropy_softmax_1hot_with_bias = \ |
|
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
285 gof.op.constructor(CrossentropySoftmax1HotWithBias) |
| 24 | 286 |
|
70
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
287 class CrossentropySoftmax1HotWithBiasDx (gof.op.Op): |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
288 nin=3 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
289 nout=1 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
290 """Gradient wrt x of the CrossentropySoftmax1Hot Op""" |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
291 def __init__(self, dy, sm, y_idx,**kwargs): |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
292 dy = tensor._as_tensor(dy) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
293 sm = tensor._as_tensor(sm) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
294 y_idx = tensor._as_tensor(y_idx) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
295 self.inputs = [dy, sm, y_idx] |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
296 self.outputs = [tensor.Tensor(sm.dtype, sm.broadcastable)] |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
297 def perform(self): |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
298 dy,sm,y_idx = [i.data for i in self.inputs] |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
299 dx = numpy.zeros_like(sm) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
300 for i in xrange(sm.shape[0]): |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
301 dx[i] = dy[i] * sm[i] #vector scale |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
302 dx[i, y_idx[i]] -= dy[i] #scalar decrement |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
303 self.outputs[0].data = dx |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
304 def grad(self, *args): |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
305 raise NotImplementedError() |
| 32 | 306 def c_code(self, (dnll, sm, y_idx), (dx,), sub): |
| 307 return """ | |
| 308 | |
|
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
309 if ((%(dnll)s->descr->type_num != PyArray_DOUBLE) |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
310 || (%(sm)s->descr->type_num != PyArray_DOUBLE) |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
311 || (%(y_idx)s->descr->type_num != PyArray_INT64)) |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
312 { |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
313 PyErr_SetString(PyExc_TypeError, "types should be float64, float64, int64"); |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
314 %(fail)s; |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
315 } |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
316 if ((%(dnll)s->nd != 1) |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
317 || (%(sm)s->nd != 2) |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
318 || (%(y_idx)s->nd != 1)) |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
319 { |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
320 PyErr_SetString(PyExc_ValueError, "rank error"); |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
321 %(fail)s; |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
322 } |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
323 if ((%(dnll)s->dimensions[0] != %(sm)s->dimensions[0]) |
| 68 | 324 || (%(dnll)s->dimensions[0] != %(y_idx)s->dimensions[0])) |
|
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
325 { |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
326 PyErr_SetString(PyExc_ValueError, "dimension mismatch"); |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
327 %(fail)s; |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
328 } |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
329 if ((NULL == %(dx)s) |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
330 || (%(dx)s->dimensions[0] != %(sm)s->dimensions[0]) |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
331 || (%(dx)s->dimensions[1] != %(sm)s->dimensions[1])) |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
332 { |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
333 if (NULL != %(dx)s) Py_XDECREF(%(dx)s); |
| 68 | 334 %(dx)s = (PyArrayObject*)PyArray_SimpleNew(2, PyArray_DIMS(%(sm)s), type_num_%(sm)s); |
|
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
335 if(!%(dx)s) { |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
336 PyErr_SetString(PyExc_MemoryError, "failed to alloc dx output"); |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
337 %(fail)s |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
338 } |
|
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
339 } |
| 24 | 340 |
|
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
341 for (size_t i = 0; i < %(dx)s->dimensions[0]; ++i) |
| 32 | 342 { |
| 343 const double dnll_i = ((double*)(%(dnll)s->data + %(dnll)s->strides[0] * i))[0]; | |
| 344 | |
| 345 const long int y_i = ((long int*)(%(y_idx)s->data + %(y_idx)s->strides[0] * i))[0]; | |
| 346 | |
| 347 const double* __restrict__ sm_i = (double*)(%(sm)s->data + %(sm)s->strides[0] * i); | |
| 348 npy_intp Ssm = %(sm)s->strides[1]/sizeof(double); | |
| 349 | |
| 350 double* __restrict__ dx_i = (double*)(%(dx)s->data + %(dx)s->strides[0] * i); | |
| 351 npy_intp Sdx = %(dx)s->strides[1]/sizeof(double); | |
| 352 | |
|
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
353 for (size_t j = 0; j < %(dx)s->dimensions[1]; ++j) |
| 32 | 354 { |
| 355 dx_i[j * Sdx] = dnll_i * sm_i[j * Ssm]; | |
| 356 } | |
|
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
357 if (y_i >= %(dx)s->dimensions[1]) |
| 32 | 358 { |
| 359 %(fail)s; | |
| 360 } | |
| 361 dx_i[y_i * Sdx] -= dnll_i; | |
| 362 } | |
| 363 """ % dict(locals(), **sub) | |
|
69
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
364 |
|
70
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
365 def crossentropy_softmax_1hot(x, y_idx, **kwargs): |
|
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
366 b = tensor.zeros_like(x[0,:]) |
|
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
367 return crossentropy_softmax_1hot_with_bias(x, b, y_idx, **kwargs) |
|
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
368 |