Mercurial > pylearn

annotate nnet_ops.py @ 30:bf0145fa73e8

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
added c implementation for CrossentropySoftmax1Hot
author bergstrj@iro.umontreal.ca
date Fri, 11 Apr 2008 21:41:09 -0400
parents b63e8c0bf21b
children 039c0f249859
rev   line source
24
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
1 import theano
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
2 from theano import tensor, gof, scalar
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
3 import numpy
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
4
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
5 class ScalarSigmoid(scalar.UnaryScalarOp):
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
6 def impl(self, x):
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
7 return 1.0 / (1 + numpy.exp(-x))
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
8 def grad(self, (x,), (gz,)):
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
9 return gz * scalar_sigmoid(x) * (1.0 - scalar_sigmoid(x)),
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
10 def c_foreach(self, (x,), (z,)):
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
11 return "%(z)s = 1.0 / (1 + exp(-%(x)s));" % locals()
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
12 scalar_sigmoid = gof.op.constructor(ScalarSigmoid)
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
13 Sigmoid, sigmoid, SigmoidInplace, sigmoid_inplace \
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
14 = theano.tensor.broadcast(ScalarSigmoid, 'Sigmoid')
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
15
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
16
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
17
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
18 class CrossentropySoftmax1Hot(gof.op.Op):
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
19 """A special compound Op for the output of neural-net classifiers.
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
20
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
21 This Op has two outputs:
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
22 - KL(softmax(x), y)
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
23 - softmax(x)
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
24
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
25 x[i] is assumed to be a dense vector
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
26 softmax(x[i]) is the i'th distribution over len(x[i]) options
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
27 y[i] is an integer index, encoding a 1-hot distribution
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
28
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
29 """
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
30 nin=2
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
31 nout=2
30
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
32 def __init__(self, x, b, y_idx, **kwargs):
24
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
33 x = tensor._as_tensor(x)
30
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
34 b = tensor._as_tensor(b)
24
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
35 y_idx = tensor._as_tensor(y_idx)
30
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
36 if len(x.broadcastable) != 2 \
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
37 or x.dtype not in ['float32', 'float64']:
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
38 raise ValueError('x must be 2-d tensor of floats')
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
39 if len(b.broadcastable) != 1 \
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
40 or x.dtype not in ['float32', 'float64']:
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
41 raise ValueError('x must be 1-d tensor of floats')
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
42 if len(y_idx.broadcastable) != 1 \
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
43 or y_idx.dtype not in ['int32', 'int64']:
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
44 raise ValueError('x must be 1-d tensor of ints')
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
45
24
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
46 # TODO: Is this correct? It used to be y, not y_idx
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
47 nll = tensor.Tensor(x.dtype, y_idx.broadcastable)
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
48 # nll = Tensor(x.dtype, y.broadcastable)
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
49 sm = tensor.Tensor(x.dtype, x.broadcastable)
30
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
50 self.inputs = [x, b, y_idx]
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
51 self.outputs = [nll, sm]
24
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
52 def perform(self):
30
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
53 x, b, y_idx = [i.data for i in self.inputs]
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
54 if b.shape[0] != x.shape[1]:
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
55 raise ValueError('b must have same shape as x[0]')
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
56
24
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
57 sm = numpy.zeros_like(x) # softmax
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
58 nll = numpy.zeros(x.shape[0]) #nll(y | softmax(x))
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
59 for i in xrange(sm.shape[0]):
30
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
60 row = x[i] + b
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
61 sm[i] = numpy.exp(row - numpy.max(row)) #softmax
24
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
62 sm[i] *= 1.0 / numpy.sum(sm[i]) #vector scale
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
63 nll[i] = -numpy.log( sm[i, y_idx[i]]) #cross-entropy
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
64 self.outputs[0].data = nll
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
65 self.outputs[1].data = sm
30
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
66 def grad(self, (x, b, y_idx), (g_nll, g_sm)):
24
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
67 if g_sm is not None:
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
68 raise NotImplementedError()
30
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
69 nll, sm = crossentropy_softmax_1hot(x, b, y_idx)
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
70 dx = CrossentropySoftmax1HotDx(g_nll, sm, y_idx).outputs[0]
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
71 db = tensor.Sum(dx, axis = [0]).outputs[0]
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
72 return dx, db, None
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
73
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
74 def c_validate_update(self, (x, b, y_idx), (nll, sm), sub):
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
75 """Allocate output storage"""
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
76 return """
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
77 if (%(x)s->nd != 2) { %(fail)s }
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
78 if (%(b)s->nd != 1) { %(fail)s }
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
79 if (%(y_idx)s->nd != 1) { %(fail)s }
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
80 if (%(x)s->descr->type_num != PyArray_DOUBLE) { %(fail)s}
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
81 if (%(b)s->descr->type_num != PyArray_DOUBLE) { %(fail)s}
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
82 if (%(y_idx)s->descr->type_num != PyArray_INT64) { %(fail)s}
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
83
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
84 %(nll)s = (PyArrayObject*)PyArray_SimpleNew(1, PyArray_DIMS(%(y_idx)s), type_num_%(x)s);
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
85 if(!%(nll)s){%(fail)s}
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
86
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
87 %(sm)s = (PyArrayObject*)PyArray_SimpleNew(2, PyArray_DIMS(%(x)s), type_num_%(x)s);
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
88 if(!%(sm)s){Py_XDECREF(%(nll)s); %(fail)s}
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
89
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
90 """ % dict(locals(), **sub)
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
91 def c_validate_cleanup(self, (x, b, y_idx), (nll, sm), sub):
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
92 """Not sure..."""
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
93 return ""
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
94 def c_support_code(self):
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
95 return """
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
96 """
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
97 def c_code(self, (x, b, y_idx), (nll, sm), sub):
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
98 # this implementation was lifted from
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
99 # /u/bergstrj/cvs/bergstrj/src/feb07/nn.cxx
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
100
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
101 #TODO: put this into a templated function, in the support code
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
102 #TODO: declare the max of each row as an Op output
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
103
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
104 return """
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
105 npy_intp* Nx = %(x)s->dimensions;
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
106 assert(%(x)s->dimensions[1] == %(b)s->dimensions[0]);
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
107 assert(%(sm)s->dimensions[0] == %(x)s->dimensions[0]);
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
108 assert(%(sm)s->dimensions[1] == %(x)s->dimensions[1]);
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
109
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
110 for (size_t i = 0; i < Nx[0]; ++i)
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
111 {
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
112 size_t j;
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
113 double sum = 0.0;
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
114 bool discount_max = false;
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
115
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
116 const double* __restrict__ x_i = (double*)(%(x)s->data + %(x)s->strides[0] * i);
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
117 const double* __restrict__ b_i = (double*)(%(b)s->data);
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
118 const long int y_i = ((long int*)(%(y_idx)s->data + %(y_idx)s->strides[0] * i))[0];
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
119 double* __restrict__ sm_i = (double*)(%(sm)s->data + %(sm)s->strides[0] * i);
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
120 double* __restrict__ nll_i = (double*)(%(nll)s->data + %(nll)s->strides[0] * i);
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
121
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
122 npy_intp Sx = %(x)s->strides[1]/sizeof(double);
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
123 npy_intp Sb = %(b)s->strides[0]/sizeof(double);
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
124 npy_intp Ssm = %(sm)s->strides[1]/sizeof(double);
24
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
125
30
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
126 size_t row_max_j=0;
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
127 double row_max = x_i[0] + b_i[0];
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
128 //try to compute sum and sm the easy way
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
129 for (j = 0; j < Nx[1]; ++j)
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
130 {
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
131 double row_ij = x_i[j * Sx] + b_i[j * Sb];
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
132 row_max_j = (row_ij > row_max) ? j : row_max_j;
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
133 row_max = (row_ij > row_max) ? row_ij : row_max;
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
134
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
135 double sm_ij = exp(row_ij);
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
136 sum += sm_ij;
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
137 sm_i[j * Ssm] = sm_ij;
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
138 }
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
139 if ((0.0 == sum) || (isinf(sum)))
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
140 {
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
141 //our cheap trick didn't work... try again and do it better.
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
142 discount_max = true;
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
143 sum = 0.0; //reset sum and recompute....
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
144 for (j = 0; j < Nx[1]; ++j)
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
145 {
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
146 double row_ij = x_i[j * Sx] + b_i[j * Sb];
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
147
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
148 double sm_ij = exp(row_ij - row_max);
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
149 sum += sm_ij;
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
150 sm_i[j * Ssm] = sm_ij;
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
151 }
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
152 assert( (0.0 != sum) && (!isinf(sum))); //that was our best...
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
153 //if we still can't sum it up, we're screwed.
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
154 //So far, this assertion has never failed...
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
155 }
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
156
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
157 //cblas_dscal(x.N, 1.0 / sum, &mat_at(s,i,0), s.n);
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
158 double sum_inv = 1.0 / sum;
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
159 for (j = 0; j < Nx[1]; ++j)
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
160 {
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
161 sm_i[j * Ssm] *= sum_inv;
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
162 }
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
163
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
164 assert(y_i < Nx[1]);
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
165
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
166 nll_i[0] = - x_i[y_i*Sx]
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
167 - b_i[y_i*Sb]
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
168 + (discount_max ? row_max : 0.0)
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
169 + log(sum);
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
170 //mat_at(y,i,0) = -log( mat_at(s,i,t[i])); //less accurate?
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
171 //mat_at(y,i,0) = - mat_at(x,i,t[i]) - mat_at(b,0,t[i]) + (discount_max ? maxi : 0.0) + log(sum);
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
172 }
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
173 """ % dict(locals(), **sub)
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
174
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
175
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
176
25
b63e8c0bf21b added __init__.py, fixed crossentropy_softmax_1hot function name
bergstrj@iro.umontreal.ca
parents: 24
diff changeset
177 crossentropy_softmax_1hot = gof.op.constructor(CrossentropySoftmax1Hot)
24
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
178
30
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
179 class CrossentropySoftmax1HotDx (gof.op.Op):
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
180 nin=3
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
181 nout=1
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
182 """Gradient wrt x of the CrossentropySoftmax1Hot Op"""
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
183 def __init__(self, dy, sm, y_idx,**kwargs):
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
184 dy = tensor._as_tensor(dy)
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
185 sm = tensor._as_tensor(sm)
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
186 y_idx = tensor._as_tensor(y_idx)
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
187 self.inputs = [dy, sm, y_idx]
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
188 self.outputs = [tensor.Tensor(sm.dtype, sm.broadcastable)]
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
189 def perform(self):
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
190 dy,sm,y_idx = [i.data for i in self.inputs]
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
191 dx = numpy.zeros_like(sm)
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
192 for i in xrange(sm.shape[0]):
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
193 dx[i] = dy[i] * sm[i] #vector scale
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
194 dx[i, y_idx[i]] -= dy[i] #scalar decrement
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
195 self.outputs[0].data = dx
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
196 def grad(self, *args):
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
197 raise NotImplementedError()
bf0145fa73e8 added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents: 25
diff changeset
198
24
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
199 #TODO: write a version of CrossentropySoftmax1Hot that accepts a bias for x, if
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
200 # this op needs to be faster.
2e8be9f5412b added nnet_ops
bergstrj@iro.umontreal.ca
parents:
diff changeset
201