Mercurial > pylearn
annotate nnet_ops.py @ 61:a8b70a9117ad
bugfix: in MinibatchDataSet renamed the class variable fields to _fields as parent class have a function called field.
bugfix: next_example is a class variable...
| author | Frederic Bastien <bastienf@iro.umontreal.ca> |
|---|---|
| date | Fri, 02 May 2008 09:55:38 -0400 |
| parents | 1b152f46ad0c |
| children | 810a8e3c85e1 |
| rev | line source |
|---|---|
| 24 | 1 import theano |
| 2 from theano import tensor, gof, scalar | |
| 3 import numpy | |
| 4 | |
| 5 class ScalarSigmoid(scalar.UnaryScalarOp): | |
| 6 def impl(self, x): | |
| 7 return 1.0 / (1 + numpy.exp(-x)) | |
| 8 def grad(self, (x,), (gz,)): | |
| 9 return gz * scalar_sigmoid(x) * (1.0 - scalar_sigmoid(x)), | |
| 10 def c_foreach(self, (x,), (z,)): | |
| 11 return "%(z)s = 1.0 / (1 + exp(-%(x)s));" % locals() | |
| 12 scalar_sigmoid = gof.op.constructor(ScalarSigmoid) | |
| 13 Sigmoid, sigmoid, SigmoidInplace, sigmoid_inplace \ | |
| 14 = theano.tensor.broadcast(ScalarSigmoid, 'Sigmoid') | |
| 15 | |
| 16 | |
| 17 | |
| 18 class CrossentropySoftmax1Hot(gof.op.Op): | |
| 19 """A special compound Op for the output of neural-net classifiers. | |
| 20 | |
| 21 This Op has two outputs: | |
| 22 - KL(softmax(x), y) | |
| 23 - softmax(x) | |
| 24 | |
| 25 x[i] is assumed to be a dense vector | |
| 26 softmax(x[i]) is the i'th distribution over len(x[i]) options | |
| 27 y[i] is an integer index, encoding a 1-hot distribution | |
| 28 | |
| 29 """ | |
| 30 nin=2 | |
| 31 nout=2 | |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
32 def __init__(self, x, b, y_idx, **kwargs): |
| 24 | 33 x = tensor._as_tensor(x) |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
34 b = tensor._as_tensor(b) |
| 24 | 35 y_idx = tensor._as_tensor(y_idx) |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
36 if len(x.broadcastable) != 2 \ |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
37 or x.dtype not in ['float32', 'float64']: |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
38 raise ValueError('x must be 2-d tensor of floats') |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
39 if len(b.broadcastable) != 1 \ |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
40 or x.dtype not in ['float32', 'float64']: |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
41 raise ValueError('x must be 1-d tensor of floats') |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
42 if len(y_idx.broadcastable) != 1 \ |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
43 or y_idx.dtype not in ['int32', 'int64']: |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
44 raise ValueError('x must be 1-d tensor of ints') |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
45 |
| 24 | 46 # TODO: Is this correct? It used to be y, not y_idx |
| 47 nll = tensor.Tensor(x.dtype, y_idx.broadcastable) | |
| 48 # nll = Tensor(x.dtype, y.broadcastable) | |
| 49 sm = tensor.Tensor(x.dtype, x.broadcastable) | |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
50 self.inputs = [x, b, y_idx] |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
51 self.outputs = [nll, sm] |
| 24 | 52 def perform(self): |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
53 x, b, y_idx = [i.data for i in self.inputs] |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
54 if b.shape[0] != x.shape[1]: |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
55 raise ValueError('b must have same shape as x[0]') |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
56 |
| 24 | 57 sm = numpy.zeros_like(x) # softmax |
| 58 nll = numpy.zeros(x.shape[0]) #nll(y | softmax(x)) | |
| 59 for i in xrange(sm.shape[0]): | |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
60 row = x[i] + b |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
61 sm[i] = numpy.exp(row - numpy.max(row)) #softmax |
| 24 | 62 sm[i] *= 1.0 / numpy.sum(sm[i]) #vector scale |
| 63 nll[i] = -numpy.log( sm[i, y_idx[i]]) #cross-entropy | |
| 64 self.outputs[0].data = nll | |
| 65 self.outputs[1].data = sm | |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
66 def grad(self, (x, b, y_idx), (g_nll, g_sm)): |
| 24 | 67 if g_sm is not None: |
| 68 raise NotImplementedError() | |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
69 nll, sm = crossentropy_softmax_1hot(x, b, y_idx) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
70 dx = CrossentropySoftmax1HotDx(g_nll, sm, y_idx).outputs[0] |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
71 db = tensor.Sum(dx, axis = [0]).outputs[0] |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
72 return dx, db, None |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
73 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
74 def c_validate_cleanup(self, (x, b, y_idx), (nll, sm), sub): |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
75 """Not sure...""" |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
76 return "" |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
77 def c_support_code(self): |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
78 return """ |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
79 """ |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
80 def c_code(self, (x, b, y_idx), (nll, sm), sub): |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
81 # this implementation was lifted from |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
82 # /u/bergstrj/cvs/bergstrj/src/feb07/nn.cxx |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
83 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
84 #TODO: put this into a templated function, in the support code |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
85 #TODO: declare the max of each row as an Op output |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
86 |
| 32 | 87 #TODO: set error messages for failures in this code |
| 88 | |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
89 return """ |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
90 npy_intp* Nx = %(x)s->dimensions; |
| 34 | 91 |
| 92 if (%(x)s->nd != 2) { %(fail)s } | |
| 93 if (%(b)s->nd != 1) { %(fail)s } | |
| 94 if (%(y_idx)s->nd != 1) { %(fail)s } | |
| 95 if (%(x)s->descr->type_num != PyArray_DOUBLE) { %(fail)s} | |
| 96 if (%(b)s->descr->type_num != PyArray_DOUBLE) { %(fail)s} | |
| 97 if (%(y_idx)s->descr->type_num != PyArray_INT64) { %(fail)s} | |
| 98 | |
| 99 %(nll)s = (PyArrayObject*)PyArray_SimpleNew(1, PyArray_DIMS(%(y_idx)s), type_num_%(x)s); | |
| 100 if(!%(nll)s){%(fail)s} | |
| 101 | |
| 102 %(sm)s = (PyArrayObject*)PyArray_SimpleNew(2, PyArray_DIMS(%(x)s), type_num_%(x)s); | |
| 103 if(!%(sm)s) { | |
| 104 // The normal cleanup code will take care of %(nll)s | |
| 105 // Py_XDECREF(%(nll)s); %(nll)s=NULL; | |
| 106 %(fail)s | |
| 107 } | |
| 32 | 108 if (%(x)s->dimensions[1] != %(b)s->dimensions[0]) {%(fail)s} |
| 109 if (%(sm)s->dimensions[0] != %(x)s->dimensions[0]) {%(fail)s} | |
| 110 if (%(sm)s->dimensions[1] != %(x)s->dimensions[1]) {%(fail)s} | |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
111 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
112 for (size_t i = 0; i < Nx[0]; ++i) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
113 { |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
114 size_t j; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
115 double sum = 0.0; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
116 bool discount_max = false; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
117 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
118 const double* __restrict__ x_i = (double*)(%(x)s->data + %(x)s->strides[0] * i); |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
119 const double* __restrict__ b_i = (double*)(%(b)s->data); |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
120 const long int y_i = ((long int*)(%(y_idx)s->data + %(y_idx)s->strides[0] * i))[0]; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
121 double* __restrict__ sm_i = (double*)(%(sm)s->data + %(sm)s->strides[0] * i); |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
122 double* __restrict__ nll_i = (double*)(%(nll)s->data + %(nll)s->strides[0] * i); |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
123 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
124 npy_intp Sx = %(x)s->strides[1]/sizeof(double); |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
125 npy_intp Sb = %(b)s->strides[0]/sizeof(double); |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
126 npy_intp Ssm = %(sm)s->strides[1]/sizeof(double); |
| 24 | 127 |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
128 size_t row_max_j=0; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
129 double row_max = x_i[0] + b_i[0]; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
130 //try to compute sum and sm the easy way |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
131 for (j = 0; j < Nx[1]; ++j) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
132 { |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
133 double row_ij = x_i[j * Sx] + b_i[j * Sb]; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
134 row_max_j = (row_ij > row_max) ? j : row_max_j; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
135 row_max = (row_ij > row_max) ? row_ij : row_max; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
136 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
137 double sm_ij = exp(row_ij); |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
138 sum += sm_ij; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
139 sm_i[j * Ssm] = sm_ij; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
140 } |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
141 if ((0.0 == sum) || (isinf(sum))) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
142 { |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
143 //our cheap trick didn't work... try again and do it better. |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
144 discount_max = true; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
145 sum = 0.0; //reset sum and recompute.... |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
146 for (j = 0; j < Nx[1]; ++j) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
147 { |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
148 double row_ij = x_i[j * Sx] + b_i[j * Sb]; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
149 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
150 double sm_ij = exp(row_ij - row_max); |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
151 sum += sm_ij; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
152 sm_i[j * Ssm] = sm_ij; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
153 } |
| 32 | 154 if ( (0.0 == sum) || (isinf(sum))) |
| 155 { | |
| 156 //that was our best... | |
| 157 %(fail)s; | |
| 158 } | |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
159 //if we still can't sum it up, we're screwed. |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
160 //So far, this assertion has never failed... |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
161 } |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
162 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
163 //cblas_dscal(x.N, 1.0 / sum, &mat_at(s,i,0), s.n); |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
164 double sum_inv = 1.0 / sum; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
165 for (j = 0; j < Nx[1]; ++j) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
166 { |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
167 sm_i[j * Ssm] *= sum_inv; |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
168 } |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
169 |
| 32 | 170 if (y_i >= Nx[1]) |
| 171 { | |
| 172 %(fail)s; | |
| 173 } | |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
174 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
175 nll_i[0] = - x_i[y_i*Sx] |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
176 - b_i[y_i*Sb] |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
177 + (discount_max ? row_max : 0.0) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
178 + log(sum); |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
179 //mat_at(y,i,0) = -log( mat_at(s,i,t[i])); //less accurate? |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
180 //mat_at(y,i,0) = - mat_at(x,i,t[i]) - mat_at(b,0,t[i]) + (discount_max ? maxi : 0.0) + log(sum); |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
181 } |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
182 """ % dict(locals(), **sub) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
183 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
184 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
185 |
|
25
b63e8c0bf21b
added __init__.py, fixed crossentropy_softmax_1hot function name
bergstrj@iro.umontreal.ca
parents:
24
diff
changeset
|
186 crossentropy_softmax_1hot = gof.op.constructor(CrossentropySoftmax1Hot) |
| 24 | 187 |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
188 class CrossentropySoftmax1HotDx (gof.op.Op): |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
189 nin=3 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
190 nout=1 |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
191 """Gradient wrt x of the CrossentropySoftmax1Hot Op""" |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
192 def __init__(self, dy, sm, y_idx,**kwargs): |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
193 dy = tensor._as_tensor(dy) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
194 sm = tensor._as_tensor(sm) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
195 y_idx = tensor._as_tensor(y_idx) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
196 self.inputs = [dy, sm, y_idx] |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
197 self.outputs = [tensor.Tensor(sm.dtype, sm.broadcastable)] |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
198 def perform(self): |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
199 dy,sm,y_idx = [i.data for i in self.inputs] |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
200 dx = numpy.zeros_like(sm) |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
201 for i in xrange(sm.shape[0]): |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
202 dx[i] = dy[i] * sm[i] #vector scale |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
203 dx[i, y_idx[i]] -= dy[i] #scalar decrement |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
204 self.outputs[0].data = dx |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
205 def grad(self, *args): |
|
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
206 raise NotImplementedError() |
| 32 | 207 def c_validate_update(self, (dnll, sm, y_idx), (dx,), sub): |
| 208 """Allocate output storage""" | |
| 209 return """ | |
| 210 if (%(dnll)s->nd != 1) { %(fail)s } | |
| 211 if (%(sm)s->nd != 2) { %(fail)s } | |
| 212 if (%(y_idx)s->nd != 1) { %(fail)s } | |
| 213 if (%(dnll)s->descr->type_num != PyArray_DOUBLE) { %(fail)s} | |
| 214 if (%(sm)s->descr->type_num != PyArray_DOUBLE) { %(fail)s} | |
| 215 if (%(y_idx)s->descr->type_num != PyArray_INT64) { %(fail)s} | |
|
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
216 |
| 32 | 217 %(dx)s = (PyArrayObject*)PyArray_SimpleNew(2, PyArray_DIMS(%(sm)s), type_num_%(sm)s); |
| 218 if(!%(dx)s){%(fail)s} | |
| 219 | |
| 220 """ % dict(locals(), **sub) | |
| 221 def c_validate_cleanup(self, inputs, outputs, sub): | |
| 222 """Not sure...""" | |
| 223 return "" | |
| 224 def c_support_code(self): | |
| 225 return """ | |
| 226 """ | |
| 227 def c_code(self, (dnll, sm, y_idx), (dx,), sub): | |
| 228 return """ | |
| 229 npy_intp* shape = %(dx)s->dimensions; | |
| 230 if (%(dnll)s->dimensions[0] != %(sm)s->dimensions[0]) {%(fail)s} | |
| 231 if (%(dnll)s->dimensions[0] != %(y_idx)s->dimensions[0]) {%(fail)s} | |
| 232 if (%(dnll)s->dimensions[0] != %(dx)s->dimensions[0]) {%(fail)s} | |
| 233 | |
| 234 if (%(sm)s->dimensions[1] != %(dx)s->dimensions[1]) {%(fail)s} | |
| 24 | 235 |
| 32 | 236 for (size_t i = 0; i < shape[0]; ++i) |
| 237 { | |
| 238 const double dnll_i = ((double*)(%(dnll)s->data + %(dnll)s->strides[0] * i))[0]; | |
| 239 | |
| 240 const long int y_i = ((long int*)(%(y_idx)s->data + %(y_idx)s->strides[0] * i))[0]; | |
| 241 | |
| 242 const double* __restrict__ sm_i = (double*)(%(sm)s->data + %(sm)s->strides[0] * i); | |
| 243 npy_intp Ssm = %(sm)s->strides[1]/sizeof(double); | |
| 244 | |
| 245 double* __restrict__ dx_i = (double*)(%(dx)s->data + %(dx)s->strides[0] * i); | |
| 246 npy_intp Sdx = %(dx)s->strides[1]/sizeof(double); | |
| 247 | |
| 248 for (size_t j = 0; j < shape[1]; ++j) | |
| 249 { | |
| 250 dx_i[j * Sdx] = dnll_i * sm_i[j * Ssm]; | |
| 251 } | |
| 252 if (y_i >= shape[1]) | |
| 253 { | |
| 254 %(fail)s; | |
| 255 } | |
| 256 dx_i[y_i * Sdx] -= dnll_i; | |
| 257 } | |
| 258 """ % dict(locals(), **sub) |