Mercurial > pylearn
annotate nnet_ops.py @ 420:040cb796f4e0
Removed feature of passing the file as a pathname from filetensor.{read, write}
Rationale:
1. The common pattern of multiple successive reads to load a list of arrays
doesn't work for the string argument.
2. It isn't clear without checking the docs whether the write() will open a file
in append mode, which could lead to bugs.
3. It's a fairly useless feature anyway, because the truly lazy programmer can
call write(open(filename, 'w'), mat) to reproduce the old behaviour if he
wants it, and without the open-mode ambiguity.
author | James Bergstra <bergstrj@iro.umontreal.ca> |
---|---|
date | Tue, 15 Jul 2008 12:57:21 -0400 |
parents | 43d9aa93934e |
children | 18dbc1c11647 |
rev | line source |
---|---|
419
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
1 ## This file contain ops that are not currently integrated in the core of threano. |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
2 ## Not all of those ops have been thoroughly tested. |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
3 |
24 | 4 import theano |
117
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
5 from theano import tensor, scalar |
24 | 6 import numpy |
7 | |
69
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
8 ############ |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
9 # |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
10 # SCALAR OPS |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
11 # |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
12 |
117
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
13 class ScalarSigmoid(scalar.UnaryScalarOp): |
69
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
14 @staticmethod |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
15 def st_impl(x): |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
16 if x < -30.0: |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
17 return 0.0 |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
18 if x > 30.0: |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
19 return 1.0 |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
20 return 1.0 / (1.0 + numpy.exp(-x)) |
24 | 21 def impl(self, x): |
69
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
22 return ScalarSigmoid.st_impl(x) |
24 | 23 def grad(self, (x,), (gz,)): |
69
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
24 y = scalar_sigmoid(x) |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
25 return [gz * y * (1.0 - y)] |
181
1b06bc2c3ca9
fixed c_code for the ops in nnet_ops.py
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
121
diff
changeset
|
26 def c_code(self, node, name, (x,), (z,), sub): |
1b06bc2c3ca9
fixed c_code for the ops in nnet_ops.py
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
121
diff
changeset
|
27 if node.inputs[0].type in [scalar.float32, scalar.float64]: |
69
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
28 return """%(z)s = |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
29 %(x)s < -30.0 |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
30 ? 0.0 |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
31 : %(x)s > 30.0 |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
32 ? 1.0 |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
33 : 1.0 /(1.0+exp(-%(x)s));""" % locals() |
181
1b06bc2c3ca9
fixed c_code for the ops in nnet_ops.py
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
121
diff
changeset
|
34 raise NotImplementedError('only floatingpoint is implemented') |
117
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
35 scalar_sigmoid = ScalarSigmoid(scalar.upgrade_to_float, name='scalar_sigmoid') |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
36 sigmoid = tensor.Elemwise(scalar_sigmoid, name='sigmoid') |
24 | 37 |
117
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
38 class ScalarSoftplus(scalar.UnaryScalarOp): |
69
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
39 @staticmethod |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
40 def static_impl(x): |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
41 if x < -30.0: |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
42 return 0.0 |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
43 if x > 30.0: |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
44 return x |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
45 return numpy.log1p(numpy.exp(x)) |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
46 def impl(self, x): |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
47 return ScalarSoftplus.static_impl(x) |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
48 def grad(self, (x,), (gz,)): |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
49 return [gz * scalar_sigmoid(x)] |
222
f6a7eb1b7970
redo what James had done, so invert node and name
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
218
diff
changeset
|
50 def c_code(self, node, name, (x,), (z,), sub): |
181
1b06bc2c3ca9
fixed c_code for the ops in nnet_ops.py
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
121
diff
changeset
|
51 if node.inputs[0].type in [scalar.float32, scalar.float64]: |
69
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
52 return """%(z)s = |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
53 %(x)s < -30.0 |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
54 ? 0.0 |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
55 : %(x)s > 30.0 |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
56 ? %(x)s |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
57 : log1p(exp(%(x)s));""" % locals() |
181
1b06bc2c3ca9
fixed c_code for the ops in nnet_ops.py
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
121
diff
changeset
|
58 raise NotImplementedError('only floating point x is implemented') |
117
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
59 scalar_softplus = ScalarSoftplus(scalar.upgrade_to_float, name='scalar_softplus') |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
60 softplus = tensor.Elemwise(scalar_softplus, name='softplus') |
69
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
61 |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
62 |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
63 ############ |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
64 # |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
65 # TENSOR OPS |
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
66 # |
24 | 67 |
117
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
68 class CrossentropySoftmax1HotWithBias(theano.Op): |
70
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
69 """A special compound L{Op} for the output of neural-net classifiers. |
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
70 |
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
71 @type x: is a matrix of floats (32 or 64) |
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
72 @type b: is a [row] vector of floats (32 or 64), length is number of cols in x |
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
73 @type y_idx: a [column] vector of int (32 or 64), length is number of rows in x |
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
74 |
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
75 @precondition: every entry in y_idx is a valid (non-negative) column index into x |
24 | 76 |
70
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
77 This L{Op} has two outputs: |
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
78 - KL(softmax(x+b), y) |
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
79 - softmax(x+b) |
24 | 80 |
70
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
81 |
24 | 82 softmax(x[i]) is the i'th distribution over len(x[i]) options |
70
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
83 |
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
84 y_idx[i] is an integer index, encoding a 1-hot distribution. |
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
85 |
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
86 In practice, when we're trying to do classification, we have one row in x |
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
87 and y_idx per example, and y[i] is the index of the (correct) class of the |
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
88 i'th example. |
24 | 89 |
90 """ | |
70
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
91 nin=3 |
24 | 92 nout=2 |
117
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
93 def __init__(self, **kwargs): |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
94 theano.Op.__init__(self, **kwargs) |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
95 |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
96 def make_node(self, x, b, y_idx): |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
97 x = tensor.as_tensor(x) |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
98 b = tensor.as_tensor(b) |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
99 y_idx = tensor.as_tensor(y_idx) |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
100 if x.type.ndim != 2 \ |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
101 or x.type.dtype not in ['float32', 'float64']: |
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
102 raise ValueError('x must be 2-d tensor of floats') |
117
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
103 if b.type.ndim != 1 \ |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
104 or x.type.dtype not in ['float32', 'float64']: |
121 | 105 raise ValueError('b must be 1-d tensor of floats') |
117
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
106 if y_idx.type.ndim != 1 \ |
185
3d953844abd3
support for more int types in crossentropysoftmax1hot
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
184
diff
changeset
|
107 or y_idx.type.dtype not in ['int8', 'int16', 'int32', 'int64']: |
121 | 108 raise ValueError('y_idx must be 1-d tensor of ints') |
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
109 |
24 | 110 # TODO: Is this correct? It used to be y, not y_idx |
117
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
111 nll = tensor.Tensor(x.type.dtype, |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
112 y_idx.type.broadcastable).make_result() |
24 | 113 # nll = Tensor(x.dtype, y.broadcastable) |
117
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
114 sm = x.type.make_result() |
185
3d953844abd3
support for more int types in crossentropysoftmax1hot
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
184
diff
changeset
|
115 return theano.Apply(self, [x, b, y_idx], [nll, sm]) |
117
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
116 def perform(self, node, input_storage, output_storage): |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
117 x, b, y_idx = input_storage |
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
118 if b.shape[0] != x.shape[1]: |
70
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
119 raise ValueError('b must have same number of columns as x') |
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
120 if y_idx.shape[0] != x.shape[0]: |
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
121 raise ValueError('y_idx must have same number of rows as x') |
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
122 |
24 | 123 sm = numpy.zeros_like(x) # softmax |
124 nll = numpy.zeros(x.shape[0]) #nll(y | softmax(x)) | |
125 for i in xrange(sm.shape[0]): | |
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
126 row = x[i] + b |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
127 sm[i] = numpy.exp(row - numpy.max(row)) #softmax |
24 | 128 sm[i] *= 1.0 / numpy.sum(sm[i]) #vector scale |
129 nll[i] = -numpy.log( sm[i, y_idx[i]]) #cross-entropy | |
117
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
130 output_storage[0][0] = nll |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
131 output_storage[1][0] = sm |
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
132 def grad(self, (x, b, y_idx), (g_nll, g_sm)): |
24 | 133 if g_sm is not None: |
134 raise NotImplementedError() | |
70
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
135 nll, sm = crossentropy_softmax_1hot_with_bias(x, b, y_idx) |
117
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
136 dx = CrossentropySoftmax1HotWithBiasDx()(g_nll, sm, y_idx) |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
137 db = tensor.sum(dx, axis = [0]) |
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
138 return dx, db, None |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
139 |
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
140 def c_headers(self): return ['<iostream>'] |
181
1b06bc2c3ca9
fixed c_code for the ops in nnet_ops.py
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
121
diff
changeset
|
141 def c_code(self, node, name, (x, b, y_idx), (nll, sm), sub): |
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
142 # this implementation was lifted from |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
143 # /u/bergstrj/cvs/bergstrj/src/feb07/nn.cxx |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
144 |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
145 #TODO: put this into a templated function, in the support code |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
146 #TODO: declare the max of each row as an Op output |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
147 |
32 | 148 #TODO: set error messages for failures in this code |
149 | |
184
9a2aecc57a79
added TODO to nnet_ops
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
181
diff
changeset
|
150 #TODO: use this to accept float32 and int32: node.inputs[0].type.dtype_specs()[1] |
185
3d953844abd3
support for more int types in crossentropysoftmax1hot
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
184
diff
changeset
|
151 y_idx_type = node.inputs[2].type.dtype_specs()[1] |
184
9a2aecc57a79
added TODO to nnet_ops
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
181
diff
changeset
|
152 |
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
153 return """ |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
154 npy_intp* Nx = %(x)s->dimensions; |
34 | 155 |
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
156 if (%(x)s->nd != 2) |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
157 { |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
158 PyErr_SetString(PyExc_ValueError, "a not 2d tensor"); |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
159 %(fail)s; |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
160 } |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
161 if (%(b)s->nd != 1) |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
162 { |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
163 PyErr_SetString(PyExc_ValueError, "b not 1d tensor"); |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
164 %(fail)s; |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
165 } |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
166 if (%(y_idx)s->nd != 1) |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
167 { |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
168 PyErr_SetString(PyExc_ValueError, "y_idx not 1d tensor"); |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
169 %(fail)s; |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
170 } |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
171 if (%(x)s->descr->type_num != PyArray_DOUBLE) |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
172 { |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
173 PyErr_SetString(PyExc_TypeError, "a not float64"); |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
174 %(fail)s; |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
175 } |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
176 if (%(b)s->descr->type_num != PyArray_DOUBLE) |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
177 { |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
178 PyErr_SetString(PyExc_TypeError, "b not float64"); |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
179 %(fail)s; |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
180 } |
185
3d953844abd3
support for more int types in crossentropysoftmax1hot
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
184
diff
changeset
|
181 if ((%(y_idx)s->descr->type_num != PyArray_INT64) |
3d953844abd3
support for more int types in crossentropysoftmax1hot
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
184
diff
changeset
|
182 && (%(y_idx)s->descr->type_num != PyArray_INT32) |
3d953844abd3
support for more int types in crossentropysoftmax1hot
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
184
diff
changeset
|
183 && (%(y_idx)s->descr->type_num != PyArray_INT16) |
3d953844abd3
support for more int types in crossentropysoftmax1hot
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
184
diff
changeset
|
184 && (%(y_idx)s->descr->type_num != PyArray_INT8)) |
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
185 { |
185
3d953844abd3
support for more int types in crossentropysoftmax1hot
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
184
diff
changeset
|
186 PyErr_SetString(PyExc_TypeError, "y_idx not int8, int16, int32, or int64"); |
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
187 %(fail)s; |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
188 } |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
189 if ((%(x)s->dimensions[1] != %(b)s->dimensions[0]) |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
190 || (%(x)s->dimensions[0] != %(y_idx)s->dimensions[0])) |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
191 { |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
192 PyErr_SetString(PyExc_ValueError, "dimension mismatch in arguments"); |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
193 %(fail)s; |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
194 } |
34 | 195 |
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
196 if ((NULL == %(nll)s) //initial condition |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
197 || (%(nll)s->dimensions[0] != %(y_idx)s->dimensions[0])) |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
198 { |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
199 if (NULL != %(nll)s) Py_XDECREF(%(nll)s); |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
200 %(nll)s = (PyArrayObject*)PyArray_SimpleNew(1, PyArray_DIMS(%(y_idx)s), type_num_%(x)s); |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
201 if(!%(nll)s) |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
202 { |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
203 PyErr_SetString(PyExc_MemoryError, "failed to alloc nll output"); |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
204 %(fail)s; |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
205 } |
34 | 206 } |
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
207 if ((NULL == %(sm)s) |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
208 || (%(sm)s->dimensions[0] != %(x)s->dimensions[0]) |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
209 || (%(sm)s->dimensions[1] != %(x)s->dimensions[1])) |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
210 { |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
211 if (NULL != %(sm)s) Py_XDECREF(%(sm)s); |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
212 %(sm)s = (PyArrayObject*)PyArray_SimpleNew(2, PyArray_DIMS(%(x)s), type_num_%(x)s); |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
213 if(!%(sm)s) { |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
214 // The normal cleanup code will take care of %(nll)s |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
215 // Py_XDECREF(%(nll)s); %(nll)s=NULL; |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
216 PyErr_SetString(PyExc_MemoryError, "failed to alloc sm output"); |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
217 %(fail)s |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
218 } |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
219 } |
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
220 |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
221 for (size_t i = 0; i < Nx[0]; ++i) |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
222 { |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
223 size_t j; |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
224 double sum = 0.0; |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
225 bool discount_max = false; |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
226 |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
227 const double* __restrict__ x_i = (double*)(%(x)s->data + %(x)s->strides[0] * i); |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
228 const double* __restrict__ b_i = (double*)(%(b)s->data); |
185
3d953844abd3
support for more int types in crossentropysoftmax1hot
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
184
diff
changeset
|
229 const %(y_idx_type)s y_i = ((%(y_idx_type)s*)(%(y_idx)s->data + %(y_idx)s->strides[0] * i))[0]; |
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
230 double* __restrict__ sm_i = (double*)(%(sm)s->data + %(sm)s->strides[0] * i); |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
231 double* __restrict__ nll_i = (double*)(%(nll)s->data + %(nll)s->strides[0] * i); |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
232 |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
233 npy_intp Sx = %(x)s->strides[1]/sizeof(double); |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
234 npy_intp Sb = %(b)s->strides[0]/sizeof(double); |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
235 npy_intp Ssm = %(sm)s->strides[1]/sizeof(double); |
24 | 236 |
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
237 size_t row_max_j=0; |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
238 double row_max = x_i[0] + b_i[0]; |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
239 //try to compute sum and sm the easy way |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
240 for (j = 0; j < Nx[1]; ++j) |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
241 { |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
242 double row_ij = x_i[j * Sx] + b_i[j * Sb]; |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
243 row_max_j = (row_ij > row_max) ? j : row_max_j; |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
244 row_max = (row_ij > row_max) ? row_ij : row_max; |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
245 |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
246 double sm_ij = exp(row_ij); |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
247 sum += sm_ij; |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
248 sm_i[j * Ssm] = sm_ij; |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
249 } |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
250 if ((0.0 == sum) || (isinf(sum))) |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
251 { |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
252 //our cheap trick didn't work... try again and do it better. |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
253 discount_max = true; |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
254 sum = 0.0; //reset sum and recompute.... |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
255 for (j = 0; j < Nx[1]; ++j) |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
256 { |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
257 double row_ij = x_i[j * Sx] + b_i[j * Sb]; |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
258 |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
259 double sm_ij = exp(row_ij - row_max); |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
260 sum += sm_ij; |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
261 sm_i[j * Ssm] = sm_ij; |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
262 } |
32 | 263 if ( (0.0 == sum) || (isinf(sum))) |
264 { | |
265 //that was our best... | |
266 %(fail)s; | |
267 } | |
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
268 //if we still can't sum it up, we're screwed. |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
269 //So far, this assertion has never failed... |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
270 } |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
271 |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
272 //cblas_dscal(x.N, 1.0 / sum, &mat_at(s,i,0), s.n); |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
273 double sum_inv = 1.0 / sum; |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
274 for (j = 0; j < Nx[1]; ++j) |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
275 { |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
276 sm_i[j * Ssm] *= sum_inv; |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
277 } |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
278 |
32 | 279 if (y_i >= Nx[1]) |
280 { | |
281 %(fail)s; | |
282 } | |
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
283 |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
284 nll_i[0] = - x_i[y_i*Sx] |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
285 - b_i[y_i*Sb] |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
286 + (discount_max ? row_max : 0.0) |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
287 + log(sum); |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
288 //mat_at(y,i,0) = -log( mat_at(s,i,t[i])); //less accurate? |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
289 //mat_at(y,i,0) = - mat_at(x,i,t[i]) - mat_at(b,0,t[i]) + (discount_max ? maxi : 0.0) + log(sum); |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
290 } |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
291 """ % dict(locals(), **sub) |
117
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
292 crossentropy_softmax_1hot_with_bias = CrossentropySoftmax1HotWithBias() |
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
293 |
117
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
294 class CrossentropySoftmax1HotWithBiasDx (theano.Op): |
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
295 nin=3 |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
296 nout=1 |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
297 """Gradient wrt x of the CrossentropySoftmax1Hot Op""" |
117
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
298 def __init__(self, **kwargs): |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
299 theano.Op.__init__(self,**kwargs) |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
300 def make_node(self, dy, sm, y_idx,**kwargs): |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
301 dy = tensor.as_tensor(dy) |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
302 sm = tensor.as_tensor(sm) |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
303 y_idx = tensor.as_tensor(y_idx) |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
304 return theano.Apply(self, [dy, sm, y_idx],[sm.type.make_result()]) |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
305 def perform(self, node, input_storage, output_storage): |
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
306 dy,sm,y_idx = input_storage |
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
307 dx = numpy.zeros_like(sm) |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
308 for i in xrange(sm.shape[0]): |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
309 dx[i] = dy[i] * sm[i] #vector scale |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
310 dx[i, y_idx[i]] -= dy[i] #scalar decrement |
117
3ef569b92fba
ported nnet_ops to new theano
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
70
diff
changeset
|
311 output_storage[0][0] = dx |
30
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
312 def grad(self, *args): |
bf0145fa73e8
added c implementation for CrossentropySoftmax1Hot
bergstrj@iro.umontreal.ca
parents:
25
diff
changeset
|
313 raise NotImplementedError() |
181
1b06bc2c3ca9
fixed c_code for the ops in nnet_ops.py
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
121
diff
changeset
|
314 def c_code(self, node, name, (dnll, sm, y_idx), (dx,), sub): |
185
3d953844abd3
support for more int types in crossentropysoftmax1hot
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
184
diff
changeset
|
315 y_idx_type = node.inputs[2].type.dtype_specs()[1] |
32 | 316 return """ |
317 | |
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
318 if ((%(dnll)s->descr->type_num != PyArray_DOUBLE) |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
319 || (%(sm)s->descr->type_num != PyArray_DOUBLE) |
185
3d953844abd3
support for more int types in crossentropysoftmax1hot
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
184
diff
changeset
|
320 ) |
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
321 { |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
322 PyErr_SetString(PyExc_TypeError, "types should be float64, float64, int64"); |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
323 %(fail)s; |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
324 } |
185
3d953844abd3
support for more int types in crossentropysoftmax1hot
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
184
diff
changeset
|
325 if ((%(y_idx)s->descr->type_num != PyArray_INT64) |
3d953844abd3
support for more int types in crossentropysoftmax1hot
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
184
diff
changeset
|
326 && (%(y_idx)s->descr->type_num != PyArray_INT32) |
3d953844abd3
support for more int types in crossentropysoftmax1hot
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
184
diff
changeset
|
327 && (%(y_idx)s->descr->type_num != PyArray_INT16) |
3d953844abd3
support for more int types in crossentropysoftmax1hot
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
184
diff
changeset
|
328 && (%(y_idx)s->descr->type_num != PyArray_INT8)) |
3d953844abd3
support for more int types in crossentropysoftmax1hot
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
184
diff
changeset
|
329 { |
3d953844abd3
support for more int types in crossentropysoftmax1hot
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
184
diff
changeset
|
330 PyErr_SetString(PyExc_TypeError, "y_idx not int8, int16, int32, or int64"); |
3d953844abd3
support for more int types in crossentropysoftmax1hot
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
184
diff
changeset
|
331 %(fail)s; |
3d953844abd3
support for more int types in crossentropysoftmax1hot
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
184
diff
changeset
|
332 } |
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
333 if ((%(dnll)s->nd != 1) |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
334 || (%(sm)s->nd != 2) |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
335 || (%(y_idx)s->nd != 1)) |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
336 { |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
337 PyErr_SetString(PyExc_ValueError, "rank error"); |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
338 %(fail)s; |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
339 } |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
340 if ((%(dnll)s->dimensions[0] != %(sm)s->dimensions[0]) |
68 | 341 || (%(dnll)s->dimensions[0] != %(y_idx)s->dimensions[0])) |
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
342 { |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
343 PyErr_SetString(PyExc_ValueError, "dimension mismatch"); |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
344 %(fail)s; |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
345 } |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
346 if ((NULL == %(dx)s) |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
347 || (%(dx)s->dimensions[0] != %(sm)s->dimensions[0]) |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
348 || (%(dx)s->dimensions[1] != %(sm)s->dimensions[1])) |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
349 { |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
350 if (NULL != %(dx)s) Py_XDECREF(%(dx)s); |
68 | 351 %(dx)s = (PyArrayObject*)PyArray_SimpleNew(2, PyArray_DIMS(%(sm)s), type_num_%(sm)s); |
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
352 if(!%(dx)s) { |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
353 PyErr_SetString(PyExc_MemoryError, "failed to alloc dx output"); |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
354 %(fail)s |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
355 } |
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
356 } |
24 | 357 |
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
358 for (size_t i = 0; i < %(dx)s->dimensions[0]; ++i) |
32 | 359 { |
360 const double dnll_i = ((double*)(%(dnll)s->data + %(dnll)s->strides[0] * i))[0]; | |
361 | |
185
3d953844abd3
support for more int types in crossentropysoftmax1hot
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
184
diff
changeset
|
362 const %(y_idx_type)s y_i = ((%(y_idx_type)s*)(%(y_idx)s->data + %(y_idx)s->strides[0] * i))[0]; |
32 | 363 |
364 const double* __restrict__ sm_i = (double*)(%(sm)s->data + %(sm)s->strides[0] * i); | |
365 npy_intp Ssm = %(sm)s->strides[1]/sizeof(double); | |
366 | |
367 double* __restrict__ dx_i = (double*)(%(dx)s->data + %(dx)s->strides[0] * i); | |
368 npy_intp Sdx = %(dx)s->strides[1]/sizeof(double); | |
369 | |
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
370 for (size_t j = 0; j < %(dx)s->dimensions[1]; ++j) |
32 | 371 { |
372 dx_i[j * Sdx] = dnll_i * sm_i[j * Ssm]; | |
373 } | |
67
810a8e3c85e1
fixed horrible memory leak from crossentropy...
bergstra@is23.m
parents:
34
diff
changeset
|
374 if (y_i >= %(dx)s->dimensions[1]) |
32 | 375 { |
376 %(fail)s; | |
377 } | |
378 dx_i[y_i * Sdx] -= dnll_i; | |
379 } | |
380 """ % dict(locals(), **sub) | |
69
8c2607f387e6
added softplus, elaborated sigmoid
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
68
diff
changeset
|
381 |
70
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
382 def crossentropy_softmax_1hot(x, y_idx, **kwargs): |
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
383 b = tensor.zeros_like(x[0,:]) |
76e5c0f37165
better docs & precondition testing for cross_entropy_softmax_1hot & friends
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
69
diff
changeset
|
384 return crossentropy_softmax_1hot_with_bias(x, b, y_idx, **kwargs) |
382
b4efd192d880
Moved xent loss to nnet_ups
Joseph Turian <turian@gmail.com>
parents:
381
diff
changeset
|
385 |
b4efd192d880
Moved xent loss to nnet_ups
Joseph Turian <turian@gmail.com>
parents:
381
diff
changeset
|
386 def binary_crossentropy(output, target): |
b4efd192d880
Moved xent loss to nnet_ups
Joseph Turian <turian@gmail.com>
parents:
381
diff
changeset
|
387 """ |
b4efd192d880
Moved xent loss to nnet_ups
Joseph Turian <turian@gmail.com>
parents:
381
diff
changeset
|
388 Compute the crossentropy of binary output wrt binary target. |
b4efd192d880
Moved xent loss to nnet_ups
Joseph Turian <turian@gmail.com>
parents:
381
diff
changeset
|
389 @note: We do not sum, crossentropy is computed by component. |
b4efd192d880
Moved xent loss to nnet_ups
Joseph Turian <turian@gmail.com>
parents:
381
diff
changeset
|
390 @todo: Rewrite as a scalar, and then broadcast to tensor. |
b4efd192d880
Moved xent loss to nnet_ups
Joseph Turian <turian@gmail.com>
parents:
381
diff
changeset
|
391 """ |
383 | 392 return -(target * tensor.log(output) + (1 - target) * tensor.log(1 - output)) |
419
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
393 |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
394 |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
395 |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
396 class Prepend_scalar_constant_to_each_row(theano.Op): |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
397 def __init__(self, val = 0): |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
398 if isinstance(val, float): |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
399 val = scalar.constant(val) |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
400 self.val = val |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
401 |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
402 def make_node(self, mat): |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
403 #check type of input |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
404 if not isinstance(mat,theano.Result) or not mat.type==tensor.matrix().type: |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
405 raise TypeError("Expected a matrix as input") |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
406 x = tensor.as_tensor(mat) |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
407 y = tensor.as_tensor(self.val) |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
408 if x.type.dtype != y.type.dtype: |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
409 TypeError("the value to prepend don't have the same type as the matrix") |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
410 |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
411 node = theano.Apply(op=self, inputs=[mat], outputs=[tensor.matrix()]) |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
412 return node |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
413 |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
414 def perform(self, node, (mat, ), (output, )): |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
415 new_shape=(mat.shape[0],mat.shape[1]+1) |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
416 if output[0] == None: |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
417 output[0]=numpy.empty(new_shape,dtype=mat.dtype) |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
418 out=output[0] |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
419 else: |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
420 if output[0].shape!=new_shape: |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
421 try: |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
422 output[0].resize(new_shape) |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
423 except: |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
424 output[0]=numpy.empty(new_shape, dtype=mat.dtype) |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
425 out=output[0] |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
426 |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
427 out[:,0].fill(self.val.data) |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
428 out[:,1:]=mat |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
429 |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
430 def grad(self, (mat,), (goutput,)): |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
431 return goutput[:,1:] |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
432 |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
433 class Prepend_scalar_to_each_row(theano.Op): |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
434 def make_node(self, val, mat): |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
435 #check type of input |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
436 if isinstance(val, float): |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
437 val = scalar.constant(val) |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
438 if not isinstance(mat,theano.Result) or not mat.type==tensor.matrix().type: |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
439 raise TypeError("Expected a matrix as input") |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
440 x = tensor.as_tensor(mat) |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
441 y = tensor.as_tensor(val) |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
442 if x.type.dtype != y.type.dtype: |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
443 TypeError("the value to prepend don't have the same type as the matrix") |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
444 |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
445 node = theano.Apply(op=self, inputs=[val,mat], outputs=[tensor.matrix()]) |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
446 return node |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
447 |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
448 def perform(self, node, (val,mat), (output, )): |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
449 new_shape=(mat.shape[0],mat.shape[1]+1) |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
450 if output[0] == None: |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
451 output[0]=numpy.empty(new_shape,dtype=mat.dtype) |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
452 out=output[0] |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
453 else: |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
454 if output[0].shape!=new_shape: |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
455 try: |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
456 output[0].resize(new_shape) |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
457 except: |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
458 output[0]=numpy.empty(new_shape, dtype=mat.dtype) |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
459 out=output[0] |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
460 out[:,0].fill(val) |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
461 out[:,1:]=mat |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
462 |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
463 def grad(self, (val, mat), (goutput,)): |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
464 return goutput[:,0], goutput[:,1:] |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
465 |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
466 prepend_scalar_to_each_row = Prepend_scalar_to_each_row() |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
467 prepend_0_to_each_row = Prepend_scalar_constant_to_each_row(0.) |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
468 prepend_1_to_each_row = Prepend_scalar_constant_to_each_row(1.) |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
469 |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
470 class solve(theano.Op): |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
471 """ |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
472 Find the solution to the linear equation Ax=b, |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
473 where A is a 2d matrix and b is a 1d or 2d matrix. |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
474 It use numpy.solve to find the solution. |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
475 """ |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
476 |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
477 def make_node(self, A, b): |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
478 if not isinstance(A, theano.Result) or not A.type==tensor.matrix().type: |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
479 raise TypeError("We expected that A had a matrix type") |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
480 if not isinstance(B, theano.Result) or not B.type==tensor.matrix().type: |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
481 raise TypeError("We expected that B had a matrix type") |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
482 |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
483 node = theano.Apply(op=self, inputs=[A, B], outputs=[tensor.matrix()]) |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
484 return node |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
485 |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
486 def perform(self, node, (A, B), (output, )): |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
487 ret=numpy.solve(A,B) |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
488 output[0]=ret |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
489 |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
490 def grad(self, (theta, A, B), (gtheta,)): |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
491 raise NotImplementedError() |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
492 |
43d9aa93934e
added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
383
diff
changeset
|
493 |