Mercurial > pylearn

annotate nnet_ops.py @ 457:34acf8db186d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Deprecated pylearn.nnet_ops. Use theano.sandbox.nnet_ops instead.
author Joseph Turian <turian@iro.umontreal.ca>
date Tue, 07 Oct 2008 18:21:32 -0400
parents 2bb67e978c28
children 2bef0768bc27
rev   line source
419
43d9aa93934e added other_ops.py to nnet_ops; added basic tests, no docs.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 383
diff changeset
1
457
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
2 import sys
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
3 sys.stderr.write("Use theano.sandbox.nnet_ops instead of pylearn.nnet_ops.\n")
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
4 if 0:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
5 ## This file contain ops that are not currently integrated in the core of threano.
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
6 ## Not all of those ops have been thoroughly tested.
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
7
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
8 import theano
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
9 from theano import tensor, scalar
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
10 import numpy
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
11
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
12 ############
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
13 #
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
14 # SCALAR OPS
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
15 #
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
16
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
17 class ScalarSigmoid(scalar.UnaryScalarOp):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
18 @staticmethod
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
19 def st_impl(x):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
20 if x < -30.0:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
21 return 0.0
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
22 if x > 30.0:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
23 return 1.0
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
24 return 1.0 / (1.0 + numpy.exp(-x))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
25 def impl(self, x):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
26 return ScalarSigmoid.st_impl(x)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
27 def grad(self, (x,), (gz,)):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
28 y = scalar_sigmoid(x)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
29 return [gz * y * (1.0 - y)]
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
30 def c_code(self, node, name, (x,), (z,), sub):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
31 if node.inputs[0].type in [scalar.float32, scalar.float64]:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
32 return """%(z)s =
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
33 %(x)s < -30.0
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
34 ? 0.0
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
35 : %(x)s > 30.0
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
36 ? 1.0
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
37 : 1.0 /(1.0+exp(-%(x)s));""" % locals()
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
38 raise NotImplementedError('only floatingpoint is implemented')
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
39 scalar_sigmoid = ScalarSigmoid(scalar.upgrade_to_float, name='scalar_sigmoid')
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
40 sigmoid = tensor.Elemwise(scalar_sigmoid, name='sigmoid')
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
41
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
42 class ScalarSoftplus(scalar.UnaryScalarOp):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
43 @staticmethod
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
44 def static_impl(x):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
45 if x < -30.0:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
46 return 0.0
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
47 if x > 30.0:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
48 return x
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
49 return numpy.log1p(numpy.exp(x))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
50 def impl(self, x):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
51 return ScalarSoftplus.static_impl(x)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
52 def grad(self, (x,), (gz,)):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
53 return [gz * scalar_sigmoid(x)]
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
54 def c_code(self, node, name, (x,), (z,), sub):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
55 if node.inputs[0].type in [scalar.float32, scalar.float64]:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
56 return """%(z)s =
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
57 %(x)s < -30.0
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
58 ? 0.0
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
59 : %(x)s > 30.0
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
60 ? %(x)s
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
61 : log1p(exp(%(x)s));""" % locals()
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
62 raise NotImplementedError('only floating point x is implemented')
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
63 scalar_softplus = ScalarSoftplus(scalar.upgrade_to_float, name='scalar_softplus')
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
64 softplus = tensor.Elemwise(scalar_softplus, name='softplus')
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
65
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
66
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
67 ############
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
68 #
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
69 # TENSOR OPS
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
70 #
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
71
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
72
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
73 class SoftmaxWithBias(theano.Op):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
74 """
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
75 An L{Op} for the output of neural-net multiclass classifiers.
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
76
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
77 @type x: is a matrix of floats (32 or 64)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
78 @type b: is a [row] vector of floats (32 or 64), length is number of cols in x
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
79
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
80 This L{Op}'s output is softmax(x+b).
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
81 softmax(x[i]) is the i'th distribution over len(x[i]) options.
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
82 """
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
83
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
84 nin = 2
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
85 nout = 1
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
86 def __init__(self, **kwargs):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
87 theano.Op.__init__(self, **kwargs)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
88
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
89 def make_node(self, x, b):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
90 x = tensor.as_tensor(x)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
91 b = tensor.as_tensor(b)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
92 if x.type.ndim != 2 \
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
93 or x.type.dtype not in ['float32', 'float64']:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
94 raise ValueError('x must be 2-d tensor of floats')
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
95 if b.type.ndim != 1 \
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
96 or x.type.dtype not in ['float32', 'float64']:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
97 raise ValueError('b must be 1-d tensor of floats')
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
98
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
99 sm = x.type.make_result()
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
100 return theano.Apply(self, [x, b], [sm])
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
101
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
102 def perform(self, node, input_storage, output_storage):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
103 x, b = input_storage
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
104 if b.shape[0] != x.shape[1]:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
105 raise ValueError('b must have same number of columns as x')
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
106
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
107 sm = numpy.zeros_like(x)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
108 for i in xrange(sm.shape[0]):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
109 row = x[i] + b
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
110 sm[i] = numpy.exp(row - numpy.max(row))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
111 sm[i] *= 1.0 / numpy.sum(sm[i])
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
112 output_storage[0][0] = sm
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
113
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
114 def grad(self, (x, b), (g_sm,)):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
115 sm = softmax_with_bias(x, b)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
116 dx = SoftmaxWithBiasDx()(g_sm, sm)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
117 db = tensor.sum(dx, axis = 0)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
118 return dx, db
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
119
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
120 def c_headers(self):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
121 return ['<iostream>']
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
122
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
123 @staticmethod
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
124 def c_code_template():
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
125 # this implementation was lifted from
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
126 # /u/bergstrj/cvs/bergstrj/src/feb07/nn.cxx
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
127
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
128 #TODO: put this into a templated function, in the support code
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
129 #TODO: declare the max of each row as an Op output
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
130
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
131 #TODO: set error messages for failures in this code
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
132
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
133 #TODO: use this to accept float32 and int32: node.inputs[0].type.dtype_specs()[1]
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
134 init_decl = """
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
135 npy_intp* Nx = %(x)s->dimensions;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
136
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
137 if (%(x)s->nd != 2)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
138 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
139 PyErr_SetString(PyExc_ValueError, "a not 2d tensor");
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
140 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
141 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
142 if (%(b)s->nd != 1)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
143 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
144 PyErr_SetString(PyExc_ValueError, "b not 1d tensor");
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
145 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
146 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
147 if (%(x)s->descr->type_num != PyArray_DOUBLE)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
148 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
149 PyErr_SetString(PyExc_TypeError, "a not float64");
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
150 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
151 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
152 if (%(b)s->descr->type_num != PyArray_DOUBLE)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
153 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
154 PyErr_SetString(PyExc_TypeError, "b not float64");
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
155 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
156 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
157 if ((%(x)s->dimensions[1] != %(b)s->dimensions[0]))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
158 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
159 PyErr_SetString(PyExc_ValueError, "dimension mismatch in arguments");
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
160 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
161 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
162
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
163 if ((NULL == %(sm)s)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
164 || (%(sm)s->dimensions[0] != %(x)s->dimensions[0])
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
165 || (%(sm)s->dimensions[1] != %(x)s->dimensions[1]))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
166 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
167 if (NULL != %(sm)s) Py_XDECREF(%(sm)s);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
168 %(sm)s = (PyArrayObject*)PyArray_SimpleNew(2, PyArray_DIMS(%(x)s), type_num_%(x)s);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
169 if(!%(sm)s) {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
170 PyErr_SetString(PyExc_MemoryError, "failed to alloc sm output");
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
171 %(fail)s
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
172 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
173 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
174 """
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
175
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
176 begin_row_loop = """
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
177 for (size_t i = 0; i < Nx[0]; ++i)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
178 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
179 size_t j;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
180 double sum = 0.0;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
181 bool discount_max = false;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
182
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
183 const double* __restrict__ x_i = (double*)(%(x)s->data + %(x)s->strides[0] * i);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
184 const double* __restrict__ b_i = (double*)(%(b)s->data);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
185 double* __restrict__ sm_i = (double*)(%(sm)s->data + %(sm)s->strides[0] * i);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
186 """
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
187
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
188 inside_row_loop = """
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
189 npy_intp Sx = %(x)s->strides[1]/sizeof(double);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
190 npy_intp Sb = %(b)s->strides[0]/sizeof(double);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
191 npy_intp Ssm = %(sm)s->strides[1]/sizeof(double);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
192
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
193 size_t row_max_j=0;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
194 double row_max = x_i[0] + b_i[0];
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
195 // Get the maximum value of the row
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
196 for (j = 0; j < Nx[1]; ++j)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
197 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
198 double row_ij = x_i[j * Sx] + b_i[j * Sb];
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
199 row_max_j = (row_ij > row_max) ? j : row_max_j;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
200 row_max = (row_ij > row_max) ? row_ij : row_max;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
201 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
202
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
203 for (j = 0; j < Nx[1]; ++j)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
204 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
205 double row_ij = x_i[j * Sx] + b_i[j * Sb];
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
206 double sm_ij = exp(row_ij - row_max);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
207 sum += sm_ij;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
208 sm_i[j * Ssm] = sm_ij;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
209 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
210 if ( (0.0 == sum) || (isinf(sum)))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
211 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
212 //that was our best...
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
213 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
214 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
215
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
216 //cblas_dscal(x.N, 1.0 / sum, &mat_at(s,i,0), s.n);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
217 double sum_inv = 1.0 / sum;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
218 for (j = 0; j < Nx[1]; ++j)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
219 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
220 sm_i[j * Ssm] *= sum_inv;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
221 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
222
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
223 """
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
224
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
225 end_row_loop = """
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
226 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
227 """
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
228
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
229 return (init_decl, begin_row_loop, inside_row_loop, end_row_loop)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
230
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
231
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
232 def c_code(self, node, name, (x, b), (sm,), sub):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
233 code_template = ''.join(self.c_code_template())
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
234 return code_template % dict(locals(), **sub)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
235
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
236 softmax_with_bias = SoftmaxWithBias()
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
237
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
238
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
239 class SoftmaxWithBiasDx(theano.Op):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
240 nin = 2
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
241 nout = 1
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
242 """Gradient wrt x of the SoftmaxWithBias Op"""
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
243
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
244 def __init__(self, **kwargs):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
245 theano.Op.__init__(self, **kwargs)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
246
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
247 def make_node(self, dy, sm, **kwargs):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
248 dy = tensor.as_tensor(dy)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
249 sm = tensor.as_tensor(sm)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
250 return theano.Apply(self, [dy, sm], [sm.type.make_result()])
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
251
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
252 def perform(self, node, input_storage, output_storage):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
253 dy, sm = input_storage
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
254 dx = numpy.zeros_like(sm)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
255 #dx[i,j] = - (\sum_k dy[i,k] sm[i,k]) sm[i,j] + dy[i,j] sm[i,j]
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
256 for i in xrange(sm.shape[0]):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
257 dy_times_sm_i = dy[i] * sm[i]
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
258 dx[i] = dy_times_sm_i - sum(dy_times_sm_i) * sm[i]
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
259 output_storage[0][0] = dx
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
260
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
261 def grad(self, *args):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
262 raise NotImplementedError()
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
263
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
264 def c_code(self, node, name, (dy, sm), (dx,), sub):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
265 return '''
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
266 if ((%(dy)s->descr->type_num != PyArray_DOUBLE)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
267 || (%(sm)s->descr->type_num != PyArray_DOUBLE))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
268 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
269 PyErr_SetString(PyExc_TypeError, "types should be float64, float64");
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
270 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
271 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
272 if ((%(dy)s->nd != 2)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
273 || (%(sm)s->nd != 2))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
274 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
275 PyErr_SetString(PyExc_ValueError, "rank error");
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
276 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
277 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
278 if (%(dy)s->dimensions[0] != %(sm)s->dimensions[0])
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
279 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
280 PyErr_SetString(PyExc_ValueError, "dimension mismatch");
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
281 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
282 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
283 if ((NULL == %(dx)s)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
284 || (%(dx)s->dimensions[0] != %(sm)s->dimensions[0])
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
285 || (%(dx)s->dimensions[1] != %(sm)s->dimensions[1]))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
286 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
287 Py_XDECREF(%(dx)s);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
288 %(dx)s = (PyArrayObject*) PyArray_SimpleNew(2, PyArray_DIMS(%(sm)s),
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
289 type_num_%(sm)s);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
290 if (!%(dx)s)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
291 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
292 PyErr_SetString(PyExc_MemoryError, "failed to alloc dx output");
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
293 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
294 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
295 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
296
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
297 for (size_t i = 0; i < %(dx)s->dimensions[0]; ++i)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
298 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
299 const double* __restrict__ dy_i = (double*) (%(dy)s->data + %(dy)s->strides[0] * i);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
300 npy_intp Sdy = %(dy)s->strides[1]/sizeof(double);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
301 const double* __restrict__ sm_i = (double*) (%(sm)s->data + %(sm)s->strides[0] * i);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
302 npy_intp Ssm = %(sm)s->strides[1]/sizeof(double);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
303 double* __restrict__ dx_i = (double*) (%(dx)s->data + %(dx)s->strides[0] * i);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
304 npy_intp Sdx = %(dx)s->strides[1]/sizeof(double);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
305
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
306 double sum_dy_times_sm = 0.;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
307 for (size_t j = 0; j < %(dx)s->dimensions[1]; ++j)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
308 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
309 dx_i[j * Sdx] = dy_i[j * Sdy] * sm_i[j * Ssm];
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
310 sum_dy_times_sm += dx_i[j * Sdx];
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
311 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
312 for (size_t j = 0; j < %(dx)s->dimensions[1]; ++j)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
313 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
314 dx_i[j * Sdx] -= sum_dy_times_sm * sm_i[j * Ssm];
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
315 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
316 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
317 ''' % dict(locals(), **sub)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
318
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
319 def softmax(x, **kwargs):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
320 b = tensor.zeros_like(x[0,:])
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
321 return softmax_with_bias(x, b, **kwargs)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
322
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
323
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
324 class CrossentropySoftmaxArgmax1HotWithBias(theano.Op):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
325 """A special compound L{Op} for the output of neural-net classifiers.
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
326
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
327 @type x: is a matrix of floats (32 or 64)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
328 @type b: is a [row] vector of floats (32 or 64), length is number of cols in x
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
329 @type y_idx: a [column] vector of int (32 or 64), length is number of rows in x
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
330
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
331 @precondition: every entry in y_idx is a valid (non-negative) column index into x
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
332
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
333 This L{Op} has three outputs:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
334 - KL(softmax(x+b), y)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
335 - softmax(x+b)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
336 - argmax(x+b)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
337
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
338 softmax(x[i]) is the i'th distribution over len(x[i]) options
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
339 argmax(x) is the index of x's greatest element
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
340 y_idx[i] is an integer index, encoding a 1-hot distribution.
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
341
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
342 In practice, when we're trying to do classification, we have one row in x
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
343 and y_idx per example, and y[i] is the index of the (correct) class of the
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
344 i'th example.
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
345
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
346 """
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
347 nin=3
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
348 nout=3
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
349 def __init__(self, **kwargs):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
350 theano.Op.__init__(self, **kwargs)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
351
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
352 def make_node(self, x, b, y_idx):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
353 x = tensor.as_tensor(x)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
354 b = tensor.as_tensor(b)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
355 y_idx = tensor.as_tensor(y_idx)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
356 if x.type.ndim != 2 \
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
357 or x.type.dtype not in ['float32', 'float64']:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
358 raise ValueError('x must be 2-d tensor of floats')
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
359 if b.type.ndim != 1 \
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
360 or x.type.dtype not in ['float32', 'float64']:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
361 raise ValueError('b must be 1-d tensor of floats')
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
362 if y_idx.type.ndim != 1 \
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
363 or y_idx.type.dtype not in ['int8', 'int16', 'int32', 'int64']:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
364 raise ValueError('y_idx must be 1-d tensor of ints')
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
365
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
366 # TODO: Is this correct? It used to be y, not y_idx
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
367 nll = tensor.Tensor(x.type.dtype,
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
368 y_idx.type.broadcastable).make_result()
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
369 # nll = Tensor(x.dtype, y.broadcastable)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
370 sm = x.type.make_result()
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
371 am = y_idx.type.make_result()
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
372 return theano.Apply(self, [x, b, y_idx], [nll, sm, am])
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
373 def perform(self, node, input_storage, output_storage):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
374 """
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
375 The math, where x is an input vector, and t is a target index:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
376
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
377 softmax(x)[i] = exp(x[i]) / sum_j(exp(x[j]))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
378 nll(x,t) = -log(softmax(x)[t])
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
379
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
380 We compute this by subtracting off the max of x. This avoids numerical instability.
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
381
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
382 m = max_j x[j]
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
383 softmax(x)[i] = exp(x[i] -m) / sum_j(exp(x[j] - m))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
384
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
385 nll = -log(exp(x[t] -m) / sum_j(exp(x[j] - m)))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
386 = -x[t] + m + log( sum_j(exp(x[j] - m)))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
387
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
388 """
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
389 x, b, y_idx = input_storage
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
390 if b.shape[0] != x.shape[1]:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
391 raise ValueError('b must have same number of columns as x')
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
392 if y_idx.shape[0] != x.shape[0]:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
393 raise ValueError('y_idx must have same number of rows as x')
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
394
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
395 sm = numpy.zeros_like(x) # softmax
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
396 nll = numpy.zeros(x.shape[0]) #nll(y | softmax(x))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
397 am = numpy.zeros_like(y_idx)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
398 for i in xrange(sm.shape[0]):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
399 #add the bias vector to the i'th row of x
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
400 row = x[i] + b
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
401
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
402 #get the maximum value of i'th row for numerically safe softmax / nll
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
403 am[i] = numpy.argmax(row)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
404 m = row[am[i]]
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
405
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
406 #compute the unnormalized softmax, and normalization constant
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
407 sm[i] = numpy.exp(row - m)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
408 sum_j = numpy.sum(sm[i]) # sum_j(exp(x[j] - m))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
409
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
410 #normalized our softmax
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
411 sm[i] *= 1.0 / sum_j
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
412
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
413 # store the nll
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
414 nll[i] = -row[y_idx[i]] + m + numpy.log(sum_j)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
415
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
416 output_storage[0][0] = nll
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
417 output_storage[1][0] = sm
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
418 output_storage[2][0] = am
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
419 def grad(self, (x, b, y_idx), (g_nll, g_sm, g_am)):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
420 if g_sm is not None or g_am is not None:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
421 raise NotImplementedError()
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
422 nll, sm = crossentropy_softmax_1hot_with_bias(x, b, y_idx)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
423 dx = CrossentropySoftmax1HotWithBiasDx()(g_nll, sm, y_idx)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
424 db = tensor.sum(dx, axis = [0])
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
425 return dx, db, None
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
426
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
427 def c_headers(self): return ['<iostream>']
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
428
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
429 @staticmethod
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
430 def c_code_template():
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
431 # this implementation was lifted from
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
432 # /u/bergstrj/cvs/bergstrj/src/feb07/nn.cxx
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
433
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
434 #TODO: put this into a templated function, in the support code
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
435 #TODO: declare the max of each row as an Op output
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
436
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
437 #TODO: set error messages for failures in this code
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
438
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
439 #TODO: use this to accept float32 and int32: node.inputs[0].type.dtype_specs()[1]
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
440 (init_decl, begin_row_loop, inside_row_loop, end_row_loop) = \
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
441 SoftmaxWithBias.c_code_template()
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
442 return (init_decl,
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
443 """
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
444 if (%(y_idx)s->nd != 1)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
445 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
446 PyErr_SetString(PyExc_ValueError, "y_idx not 1d tensor");
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
447 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
448 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
449 if ((%(y_idx)s->descr->type_num != PyArray_INT64)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
450 && (%(y_idx)s->descr->type_num != PyArray_INT32)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
451 && (%(y_idx)s->descr->type_num != PyArray_INT16)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
452 && (%(y_idx)s->descr->type_num != PyArray_INT8))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
453 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
454 PyErr_SetString(PyExc_TypeError, "y_idx not int8, int16, int32, or int64");
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
455 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
456 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
457 if (%(x)s->dimensions[0] != %(y_idx)s->dimensions[0])
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
458 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
459 PyErr_SetString(PyExc_ValueError, "dimension mismatch in arguments");
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
460 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
461 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
462
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
463 if ((NULL == %(nll)s) //initial condition
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
464 || (%(nll)s->dimensions[0] != %(y_idx)s->dimensions[0]))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
465 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
466 if (NULL != %(nll)s) Py_XDECREF(%(nll)s);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
467 %(nll)s = (PyArrayObject*)PyArray_SimpleNew(1, PyArray_DIMS(%(y_idx)s), type_num_%(x)s);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
468 if(!%(nll)s)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
469 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
470 PyErr_SetString(PyExc_MemoryError, "failed to alloc nll output");
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
471 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
472 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
473 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
474 if ((NULL == %(am)s)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
475 || (%(am)s->dimensions[0] != %(y_idx)s->dimensions[0]))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
476 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
477 Py_XDECREF(%(am)s);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
478 %(am)s = (PyArrayObject*) PyArray_SimpleNew(1, PyArray_DIMS(%(y_idx)s), type_num_%(y_idx)s);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
479 if(!%(am)s)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
480 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
481 PyErr_SetString(PyExc_MemoryError, "failed to alloc am output");
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
482 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
483 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
484 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
485 """,
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
486 begin_row_loop,
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
487 """
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
488 const %(y_idx_type)s y_i = ((%(y_idx_type)s*)(%(y_idx)s->data + %(y_idx)s->strides[0] * i))[0];
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
489 double* __restrict__ nll_i = (double*)(%(nll)s->data + %(nll)s->strides[0] * i);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
490 %(am_type)s* __restrict__ am_i = (%(am_type)s*) (%(am)s->data + %(am)s->strides[0] * i);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
491 """,
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
492 inside_row_loop,
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
493 """
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
494 nll_i[0] = - x_i[y_i*Sx]
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
495 - b_i[y_i*Sb]
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
496 + row_max
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
497 + log(sum);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
498 am_i[0] = row_max_j;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
499 """,
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
500 end_row_loop)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
501
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
502
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
503 def c_code(self, node, name, (x, b, y_idx), (nll, sm, am), sub):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
504 y_idx_type = node.inputs[2].type.dtype_specs()[1]
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
505 am_type = y_idx_type
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
506 code_template = ''.join(self.c_code_template())
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
507 return code_template % dict(locals(), **sub)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
508
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
509 class CrossentropySoftmax1HotWithBiasDx (theano.Op):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
510 nin=3
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
511 nout=1
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
512 """Gradient wrt x of the CrossentropySoftmax1Hot Op"""
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
513 def __init__(self, **kwargs):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
514 theano.Op.__init__(self,**kwargs)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
515 def make_node(self, dy, sm, y_idx,**kwargs):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
516 dy = tensor.as_tensor(dy)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
517 sm = tensor.as_tensor(sm)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
518 y_idx = tensor.as_tensor(y_idx)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
519 return theano.Apply(self, [dy, sm, y_idx],[sm.type.make_result()])
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
520 def perform(self, node, input_storage, output_storage):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
521 dy,sm,y_idx = input_storage
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
522 dx = numpy.zeros_like(sm)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
523 for i in xrange(sm.shape[0]):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
524 dx[i] = dy[i] * sm[i] #vector scale
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
525 dx[i, y_idx[i]] -= dy[i] #scalar decrement
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
526 output_storage[0][0] = dx
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
527 def grad(self, *args):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
528 raise NotImplementedError()
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
529 def c_code(self, node, name, (dnll, sm, y_idx), (dx,), sub):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
530 y_idx_type = node.inputs[2].type.dtype_specs()[1]
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
531 return """
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
532
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
533 if ((%(dnll)s->descr->type_num != PyArray_DOUBLE)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
534 || (%(sm)s->descr->type_num != PyArray_DOUBLE)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
535 )
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
536 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
537 PyErr_SetString(PyExc_TypeError, "types should be float64, float64, int64");
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
538 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
539 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
540 if ((%(y_idx)s->descr->type_num != PyArray_INT64)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
541 && (%(y_idx)s->descr->type_num != PyArray_INT32)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
542 && (%(y_idx)s->descr->type_num != PyArray_INT16)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
543 && (%(y_idx)s->descr->type_num != PyArray_INT8))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
544 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
545 PyErr_SetString(PyExc_TypeError, "y_idx not int8, int16, int32, or int64");
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
546 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
547 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
548 if ((%(dnll)s->nd != 1)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
549 || (%(sm)s->nd != 2)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
550 || (%(y_idx)s->nd != 1))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
551 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
552 PyErr_SetString(PyExc_ValueError, "rank error");
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
553 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
554 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
555 if ((%(dnll)s->dimensions[0] != %(sm)s->dimensions[0])
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
556 || (%(dnll)s->dimensions[0] != %(y_idx)s->dimensions[0]))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
557 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
558 PyErr_SetString(PyExc_ValueError, "dimension mismatch");
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
559 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
560 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
561 if ((NULL == %(dx)s)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
562 || (%(dx)s->dimensions[0] != %(sm)s->dimensions[0])
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
563 || (%(dx)s->dimensions[1] != %(sm)s->dimensions[1]))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
564 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
565 if (NULL != %(dx)s) Py_XDECREF(%(dx)s);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
566 %(dx)s = (PyArrayObject*)PyArray_SimpleNew(2, PyArray_DIMS(%(sm)s), type_num_%(sm)s);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
567 if(!%(dx)s) {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
568 PyErr_SetString(PyExc_MemoryError, "failed to alloc dx output");
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
569 %(fail)s
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
570 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
571 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
572
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
573 for (size_t i = 0; i < %(dx)s->dimensions[0]; ++i)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
574 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
575 const double dnll_i = ((double*)(%(dnll)s->data + %(dnll)s->strides[0] * i))[0];
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
576
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
577 const %(y_idx_type)s y_i = ((%(y_idx_type)s*)(%(y_idx)s->data + %(y_idx)s->strides[0] * i))[0];
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
578
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
579 const double* __restrict__ sm_i = (double*)(%(sm)s->data + %(sm)s->strides[0] * i);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
580 npy_intp Ssm = %(sm)s->strides[1]/sizeof(double);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
581
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
582 double* __restrict__ dx_i = (double*)(%(dx)s->data + %(dx)s->strides[0] * i);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
583 npy_intp Sdx = %(dx)s->strides[1]/sizeof(double);
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
584
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
585 for (size_t j = 0; j < %(dx)s->dimensions[1]; ++j)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
586 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
587 dx_i[j * Sdx] = dnll_i * sm_i[j * Ssm];
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
588 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
589 if (y_i >= %(dx)s->dimensions[1])
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
590 {
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
591 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
592 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
593 dx_i[y_i * Sdx] -= dnll_i;
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
594 }
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
595 """ % dict(locals(), **sub)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
596
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
597 crossentropy_softmax_argmax_1hot_with_bias = \
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
598 CrossentropySoftmaxArgmax1HotWithBias()
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
599
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
600 def crossentropy_softmax_1hot_with_bias(x, b, y_idx, **kwargs):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
601 return crossentropy_softmax_argmax_1hot_with_bias(x, b, y_idx, **kwargs)[0:2]
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
602
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
603 def crossentropy_softmax_1hot(x, y_idx, **kwargs):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
604 b = tensor.zeros_like(x[0,:])
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
605 return crossentropy_softmax_1hot_with_bias(x, b, y_idx, **kwargs)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
606
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
607
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
608 class MultinomialCrossentropy1Hot(theano.Op):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
609 pass
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
610
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
611
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
612 def binary_crossentropy(output, target):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
613 """
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
614 Compute the crossentropy of binary output wrt binary target.
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
615 @note: We do not sum, crossentropy is computed by component.
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
616 @todo: Rewrite as a scalar, and then broadcast to tensor.
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
617 @todo: This is essentially duplicated as cost.cross_entropy
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
618 @warning: OUTPUT and TARGET are reversed in cost.cross_entropy
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
619 """
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
620 return -(target * tensor.log(output) + (1 - target) * tensor.log(1 - output))
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
621
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
622
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
623
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
624 class Prepend_scalar_constant_to_each_row(theano.Op):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
625 def __init__(self, val = 0):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
626 if isinstance(val, float):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
627 val = scalar.constant(val)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
628 self.val = val
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
629
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
630 def make_node(self, mat):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
631 #check type of input
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
632 if not isinstance(mat,theano.Result) or not mat.type==tensor.matrix().type:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
633 raise TypeError("Expected a matrix as input")
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
634 x = tensor.as_tensor(mat)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
635 y = tensor.as_tensor(self.val)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
636 if x.type.dtype != y.type.dtype:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
637 TypeError("the value to prepend don't have the same type as the matrix")
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
638
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
639 node = theano.Apply(op=self, inputs=[mat], outputs=[tensor.matrix()])
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
640 return node
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
641
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
642 def perform(self, node, (mat, ), (output, )):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
643 new_shape=(mat.shape[0],mat.shape[1]+1)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
644 if output[0] == None:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
645 output[0]=numpy.empty(new_shape,dtype=mat.dtype)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
646 out=output[0]
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
647 else:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
648 if output[0].shape!=new_shape:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
649 try:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
650 output[0].resize(new_shape)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
651 except:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
652 output[0]=numpy.empty(new_shape, dtype=mat.dtype)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
653 out=output[0]
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
654
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
655 out[:,0].fill(self.val.data)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
656 out[:,1:]=mat
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
657
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
658 def grad(self, (mat,), (goutput,)):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
659 return goutput[:,1:]
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
660
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
661 class Prepend_scalar_to_each_row(theano.Op):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
662 def make_node(self, val, mat):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
663 #check type of input
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
664 if isinstance(val, float):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
665 val = scalar.constant(val)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
666 if not isinstance(mat,theano.Result) or not mat.type==tensor.matrix().type:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
667 raise TypeError("Expected a matrix as input")
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
668 x = tensor.as_tensor(mat)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
669 y = tensor.as_tensor(val)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
670 if x.type.dtype != y.type.dtype:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
671 TypeError("the value to prepend don't have the same type as the matrix")
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
672
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
673 node = theano.Apply(op=self, inputs=[val,mat], outputs=[tensor.matrix()])
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
674 return node
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
675
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
676 def perform(self, node, (val,mat), (output, )):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
677 new_shape=(mat.shape[0],mat.shape[1]+1)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
678 if output[0] == None:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
679 output[0]=numpy.empty(new_shape,dtype=mat.dtype)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
680 out=output[0]
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
681 else:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
682 if output[0].shape!=new_shape:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
683 try:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
684 output[0].resize(new_shape)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
685 except:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
686 output[0]=numpy.empty(new_shape, dtype=mat.dtype)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
687 out=output[0]
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
688 out[:,0].fill(val)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
689 out[:,1:]=mat
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
690
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
691 def grad(self, (val, mat), (goutput,)):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
692 return goutput[:,0], goutput[:,1:]
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
693
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
694 prepend_scalar_to_each_row = Prepend_scalar_to_each_row()
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
695 prepend_0_to_each_row = Prepend_scalar_constant_to_each_row(0.)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
696 prepend_1_to_each_row = Prepend_scalar_constant_to_each_row(1.)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
697
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
698 class solve(theano.Op):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
699 """
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
700 Find the solution to the linear equation Ax=b,
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
701 where A is a 2d matrix and b is a 1d or 2d matrix.
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
702 It use numpy.solve to find the solution.
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
703 """
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
704
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
705 def make_node(self, A, b):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
706 if not isinstance(A, theano.Result) or not A.type==tensor.matrix().type:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
707 raise TypeError("We expected that A had a matrix type")
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
708 if not isinstance(B, theano.Result) or not B.type==tensor.matrix().type:
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
709 raise TypeError("We expected that B had a matrix type")
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
710
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
711 node = theano.Apply(op=self, inputs=[A, B], outputs=[tensor.matrix()])
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
712 return node
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
713
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
714 def perform(self, node, (A, B), (output, )):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
715 ret=numpy.solve(A,B)
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
716 output[0]=ret
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
717
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
718 def grad(self, (theta, A, B), (gtheta,)):
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
719 raise NotImplementedError()
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
720
34acf8db186d Deprecated pylearn.nnet_ops.
Joseph Turian <turian@iro.umontreal.ca>
parents: 449
diff changeset
721