pylearn: nnet_ops.py annotate

annotate nnet_ops.py @ 472:69c800af1370

changed weight initialization for logistic regression

author	James Bergstra <bergstrj@iro.umontreal.ca>
date	Thu, 23 Oct 2008 13:26:42 -0400
parents	34acf8db186d
children	2bef0768bc27

rev	line source
419 43d9aa93934e added other_ops.py to nnet_ops; added basic tests, no docs. James Bergstra <bergstrj@iro.umontreal.ca> parents: 383 diff changeset	1
457 34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	2 import sys
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	3 sys.stderr.write("Use theano.sandbox.nnet_ops instead of pylearn.nnet_ops.\n")
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	4 if 0:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	5 ## This file contain ops that are not currently integrated in the core of threano.
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	6 ## Not all of those ops have been thoroughly tested.
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	7
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	8 import theano
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	9 from theano import tensor, scalar
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	10 import numpy
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	11
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	12 ############
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	13 #
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	14 # SCALAR OPS
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	15 #
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	16
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	17 class ScalarSigmoid(scalar.UnaryScalarOp):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	18 @staticmethod
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	19 def st_impl(x):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	20 if x < -30.0:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	21 return 0.0
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	22 if x > 30.0:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	23 return 1.0
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	24 return 1.0 / (1.0 + numpy.exp(-x))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	25 def impl(self, x):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	26 return ScalarSigmoid.st_impl(x)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	27 def grad(self, (x,), (gz,)):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	28 y = scalar_sigmoid(x)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	29 return [gz * y * (1.0 - y)]
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	30 def c_code(self, node, name, (x,), (z,), sub):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	31 if node.inputs[0].type in [scalar.float32, scalar.float64]:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	32 return """%(z)s =
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	33 %(x)s < -30.0
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	34 ? 0.0
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	35 : %(x)s > 30.0
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	36 ? 1.0
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	37 : 1.0 /(1.0+exp(-%(x)s));""" % locals()
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	38 raise NotImplementedError('only floatingpoint is implemented')
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	39 scalar_sigmoid = ScalarSigmoid(scalar.upgrade_to_float, name='scalar_sigmoid')
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	40 sigmoid = tensor.Elemwise(scalar_sigmoid, name='sigmoid')
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	41
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	42 class ScalarSoftplus(scalar.UnaryScalarOp):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	43 @staticmethod
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	44 def static_impl(x):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	45 if x < -30.0:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	46 return 0.0
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	47 if x > 30.0:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	48 return x
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	49 return numpy.log1p(numpy.exp(x))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	50 def impl(self, x):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	51 return ScalarSoftplus.static_impl(x)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	52 def grad(self, (x,), (gz,)):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	53 return [gz * scalar_sigmoid(x)]
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	54 def c_code(self, node, name, (x,), (z,), sub):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	55 if node.inputs[0].type in [scalar.float32, scalar.float64]:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	56 return """%(z)s =
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	57 %(x)s < -30.0
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	58 ? 0.0
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	59 : %(x)s > 30.0
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	60 ? %(x)s
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	61 : log1p(exp(%(x)s));""" % locals()
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	62 raise NotImplementedError('only floating point x is implemented')
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	63 scalar_softplus = ScalarSoftplus(scalar.upgrade_to_float, name='scalar_softplus')
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	64 softplus = tensor.Elemwise(scalar_softplus, name='softplus')
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	65
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	66
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	67 ############
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	68 #
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	69 # TENSOR OPS
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	70 #
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	71
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	72
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	73 class SoftmaxWithBias(theano.Op):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	74 """
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	75 An L{Op} for the output of neural-net multiclass classifiers.
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	76
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	77 @type x: is a matrix of floats (32 or 64)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	78 @type b: is a [row] vector of floats (32 or 64), length is number of cols in x
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	79
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	80 This L{Op}'s output is softmax(x+b).
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	81 softmax(x[i]) is the i'th distribution over len(x[i]) options.
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	82 """
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	83
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	84 nin = 2
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	85 nout = 1
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	86 def __init__(self, **kwargs):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	87 theano.Op.__init__(self, **kwargs)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	88
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	89 def make_node(self, x, b):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	90 x = tensor.as_tensor(x)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	91 b = tensor.as_tensor(b)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	92 if x.type.ndim != 2 \
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	93 or x.type.dtype not in ['float32', 'float64']:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	94 raise ValueError('x must be 2-d tensor of floats')
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	95 if b.type.ndim != 1 \
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	96 or x.type.dtype not in ['float32', 'float64']:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	97 raise ValueError('b must be 1-d tensor of floats')
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	98
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	99 sm = x.type.make_result()
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	100 return theano.Apply(self, [x, b], [sm])
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	101
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	102 def perform(self, node, input_storage, output_storage):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	103 x, b = input_storage
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	104 if b.shape[0] != x.shape[1]:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	105 raise ValueError('b must have same number of columns as x')
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	106
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	107 sm = numpy.zeros_like(x)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	108 for i in xrange(sm.shape[0]):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	109 row = x[i] + b
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	110 sm[i] = numpy.exp(row - numpy.max(row))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	111 sm[i] *= 1.0 / numpy.sum(sm[i])
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	112 output_storage[0][0] = sm
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	113
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	114 def grad(self, (x, b), (g_sm,)):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	115 sm = softmax_with_bias(x, b)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	116 dx = SoftmaxWithBiasDx()(g_sm, sm)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	117 db = tensor.sum(dx, axis = 0)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	118 return dx, db
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	119
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	120 def c_headers(self):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	121 return ['<iostream>']
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	122
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	123 @staticmethod
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	124 def c_code_template():
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	125 # this implementation was lifted from
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	126 # /u/bergstrj/cvs/bergstrj/src/feb07/nn.cxx
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	127
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	128 #TODO: put this into a templated function, in the support code
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	129 #TODO: declare the max of each row as an Op output
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	130
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	131 #TODO: set error messages for failures in this code
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	132
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	133 #TODO: use this to accept float32 and int32: node.inputs[0].type.dtype_specs()[1]
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	134 init_decl = """
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	135 npy_intp* Nx = %(x)s->dimensions;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	136
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	137 if (%(x)s->nd != 2)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	138 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	139 PyErr_SetString(PyExc_ValueError, "a not 2d tensor");
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	140 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	141 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	142 if (%(b)s->nd != 1)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	143 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	144 PyErr_SetString(PyExc_ValueError, "b not 1d tensor");
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	145 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	146 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	147 if (%(x)s->descr->type_num != PyArray_DOUBLE)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	148 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	149 PyErr_SetString(PyExc_TypeError, "a not float64");
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	150 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	151 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	152 if (%(b)s->descr->type_num != PyArray_DOUBLE)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	153 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	154 PyErr_SetString(PyExc_TypeError, "b not float64");
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	155 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	156 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	157 if ((%(x)s->dimensions[1] != %(b)s->dimensions[0]))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	158 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	159 PyErr_SetString(PyExc_ValueError, "dimension mismatch in arguments");
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	160 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	161 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	162
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	163 if ((NULL == %(sm)s)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	164 \|\| (%(sm)s->dimensions[0] != %(x)s->dimensions[0])
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	165 \|\| (%(sm)s->dimensions[1] != %(x)s->dimensions[1]))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	166 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	167 if (NULL != %(sm)s) Py_XDECREF(%(sm)s);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	168 %(sm)s = (PyArrayObject*)PyArray_SimpleNew(2, PyArray_DIMS(%(x)s), type_num_%(x)s);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	169 if(!%(sm)s) {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	170 PyErr_SetString(PyExc_MemoryError, "failed to alloc sm output");
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	171 %(fail)s
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	172 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	173 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	174 """
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	175
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	176 begin_row_loop = """
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	177 for (size_t i = 0; i < Nx[0]; ++i)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	178 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	179 size_t j;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	180 double sum = 0.0;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	181 bool discount_max = false;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	182
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	183 const double* __restrict__ x_i = (double)(%(x)s->data + %(x)s->strides[0] i);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	184 const double* __restrict__ b_i = (double*)(%(b)s->data);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	185 double* __restrict__ sm_i = (double)(%(sm)s->data + %(sm)s->strides[0] i);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	186 """
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	187
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	188 inside_row_loop = """
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	189 npy_intp Sx = %(x)s->strides[1]/sizeof(double);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	190 npy_intp Sb = %(b)s->strides[0]/sizeof(double);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	191 npy_intp Ssm = %(sm)s->strides[1]/sizeof(double);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	192
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	193 size_t row_max_j=0;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	194 double row_max = x_i[0] + b_i[0];
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	195 // Get the maximum value of the row
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	196 for (j = 0; j < Nx[1]; ++j)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	197 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	198 double row_ij = x_i[j * Sx] + b_i[j * Sb];
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	199 row_max_j = (row_ij > row_max) ? j : row_max_j;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	200 row_max = (row_ij > row_max) ? row_ij : row_max;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	201 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	202
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	203 for (j = 0; j < Nx[1]; ++j)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	204 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	205 double row_ij = x_i[j * Sx] + b_i[j * Sb];
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	206 double sm_ij = exp(row_ij - row_max);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	207 sum += sm_ij;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	208 sm_i[j * Ssm] = sm_ij;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	209 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	210 if ( (0.0 == sum) \|\| (isinf(sum)))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	211 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	212 //that was our best...
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	213 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	214 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	215
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	216 //cblas_dscal(x.N, 1.0 / sum, &mat_at(s,i,0), s.n);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	217 double sum_inv = 1.0 / sum;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	218 for (j = 0; j < Nx[1]; ++j)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	219 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	220 sm_i[j * Ssm] *= sum_inv;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	221 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	222
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	223 """
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	224
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	225 end_row_loop = """
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	226 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	227 """
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	228
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	229 return (init_decl, begin_row_loop, inside_row_loop, end_row_loop)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	230
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	231
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	232 def c_code(self, node, name, (x, b), (sm,), sub):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	233 code_template = ''.join(self.c_code_template())
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	234 return code_template % dict(locals(), **sub)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	235
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	236 softmax_with_bias = SoftmaxWithBias()
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	237
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	238
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	239 class SoftmaxWithBiasDx(theano.Op):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	240 nin = 2
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	241 nout = 1
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	242 """Gradient wrt x of the SoftmaxWithBias Op"""
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	243
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	244 def __init__(self, **kwargs):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	245 theano.Op.__init__(self, **kwargs)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	246
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	247 def make_node(self, dy, sm, **kwargs):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	248 dy = tensor.as_tensor(dy)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	249 sm = tensor.as_tensor(sm)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	250 return theano.Apply(self, [dy, sm], [sm.type.make_result()])
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	251
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	252 def perform(self, node, input_storage, output_storage):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	253 dy, sm = input_storage
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	254 dx = numpy.zeros_like(sm)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	255 #dx[i,j] = - (\sum_k dy[i,k] sm[i,k]) sm[i,j] + dy[i,j] sm[i,j]
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	256 for i in xrange(sm.shape[0]):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	257 dy_times_sm_i = dy[i] * sm[i]
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	258 dx[i] = dy_times_sm_i - sum(dy_times_sm_i) * sm[i]
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	259 output_storage[0][0] = dx
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	260
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	261 def grad(self, *args):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	262 raise NotImplementedError()
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	263
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	264 def c_code(self, node, name, (dy, sm), (dx,), sub):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	265 return '''
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	266 if ((%(dy)s->descr->type_num != PyArray_DOUBLE)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	267 \|\| (%(sm)s->descr->type_num != PyArray_DOUBLE))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	268 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	269 PyErr_SetString(PyExc_TypeError, "types should be float64, float64");
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	270 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	271 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	272 if ((%(dy)s->nd != 2)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	273 \|\| (%(sm)s->nd != 2))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	274 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	275 PyErr_SetString(PyExc_ValueError, "rank error");
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	276 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	277 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	278 if (%(dy)s->dimensions[0] != %(sm)s->dimensions[0])
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	279 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	280 PyErr_SetString(PyExc_ValueError, "dimension mismatch");
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	281 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	282 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	283 if ((NULL == %(dx)s)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	284 \|\| (%(dx)s->dimensions[0] != %(sm)s->dimensions[0])
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	285 \|\| (%(dx)s->dimensions[1] != %(sm)s->dimensions[1]))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	286 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	287 Py_XDECREF(%(dx)s);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	288 %(dx)s = (PyArrayObject*) PyArray_SimpleNew(2, PyArray_DIMS(%(sm)s),
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	289 type_num_%(sm)s);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	290 if (!%(dx)s)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	291 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	292 PyErr_SetString(PyExc_MemoryError, "failed to alloc dx output");
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	293 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	294 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	295 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	296
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	297 for (size_t i = 0; i < %(dx)s->dimensions[0]; ++i)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	298 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	299 const double* __restrict__ dy_i = (double) (%(dy)s->data + %(dy)s->strides[0] i);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	300 npy_intp Sdy = %(dy)s->strides[1]/sizeof(double);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	301 const double* __restrict__ sm_i = (double) (%(sm)s->data + %(sm)s->strides[0] i);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	302 npy_intp Ssm = %(sm)s->strides[1]/sizeof(double);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	303 double* __restrict__ dx_i = (double) (%(dx)s->data + %(dx)s->strides[0] i);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	304 npy_intp Sdx = %(dx)s->strides[1]/sizeof(double);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	305
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	306 double sum_dy_times_sm = 0.;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	307 for (size_t j = 0; j < %(dx)s->dimensions[1]; ++j)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	308 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	309 dx_i[j * Sdx] = dy_i[j * Sdy] * sm_i[j * Ssm];
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	310 sum_dy_times_sm += dx_i[j * Sdx];
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	311 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	312 for (size_t j = 0; j < %(dx)s->dimensions[1]; ++j)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	313 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	314 dx_i[j * Sdx] -= sum_dy_times_sm * sm_i[j * Ssm];
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	315 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	316 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	317 ''' % dict(locals(), **sub)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	318
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	319 def softmax(x, **kwargs):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	320 b = tensor.zeros_like(x[0,:])
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	321 return softmax_with_bias(x, b, **kwargs)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	322
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	323
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	324 class CrossentropySoftmaxArgmax1HotWithBias(theano.Op):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	325 """A special compound L{Op} for the output of neural-net classifiers.
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	326
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	327 @type x: is a matrix of floats (32 or 64)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	328 @type b: is a [row] vector of floats (32 or 64), length is number of cols in x
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	329 @type y_idx: a [column] vector of int (32 or 64), length is number of rows in x
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	330
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	331 @precondition: every entry in y_idx is a valid (non-negative) column index into x
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	332
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	333 This L{Op} has three outputs:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	334 - KL(softmax(x+b), y)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	335 - softmax(x+b)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	336 - argmax(x+b)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	337
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	338 softmax(x[i]) is the i'th distribution over len(x[i]) options
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	339 argmax(x) is the index of x's greatest element
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	340 y_idx[i] is an integer index, encoding a 1-hot distribution.
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	341
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	342 In practice, when we're trying to do classification, we have one row in x
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	343 and y_idx per example, and y[i] is the index of the (correct) class of the
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	344 i'th example.
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	345
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	346 """
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	347 nin=3
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	348 nout=3
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	349 def __init__(self, **kwargs):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	350 theano.Op.__init__(self, **kwargs)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	351
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	352 def make_node(self, x, b, y_idx):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	353 x = tensor.as_tensor(x)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	354 b = tensor.as_tensor(b)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	355 y_idx = tensor.as_tensor(y_idx)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	356 if x.type.ndim != 2 \
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	357 or x.type.dtype not in ['float32', 'float64']:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	358 raise ValueError('x must be 2-d tensor of floats')
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	359 if b.type.ndim != 1 \
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	360 or x.type.dtype not in ['float32', 'float64']:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	361 raise ValueError('b must be 1-d tensor of floats')
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	362 if y_idx.type.ndim != 1 \
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	363 or y_idx.type.dtype not in ['int8', 'int16', 'int32', 'int64']:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	364 raise ValueError('y_idx must be 1-d tensor of ints')
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	365
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	366 # TODO: Is this correct? It used to be y, not y_idx
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	367 nll = tensor.Tensor(x.type.dtype,
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	368 y_idx.type.broadcastable).make_result()
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	369 # nll = Tensor(x.dtype, y.broadcastable)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	370 sm = x.type.make_result()
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	371 am = y_idx.type.make_result()
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	372 return theano.Apply(self, [x, b, y_idx], [nll, sm, am])
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	373 def perform(self, node, input_storage, output_storage):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	374 """
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	375 The math, where x is an input vector, and t is a target index:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	376
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	377 softmax(x)[i] = exp(x[i]) / sum_j(exp(x[j]))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	378 nll(x,t) = -log(softmax(x)[t])
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	379
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	380 We compute this by subtracting off the max of x. This avoids numerical instability.
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	381
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	382 m = max_j x[j]
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	383 softmax(x)[i] = exp(x[i] -m) / sum_j(exp(x[j] - m))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	384
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	385 nll = -log(exp(x[t] -m) / sum_j(exp(x[j] - m)))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	386 = -x[t] + m + log( sum_j(exp(x[j] - m)))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	387
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	388 """
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	389 x, b, y_idx = input_storage
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	390 if b.shape[0] != x.shape[1]:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	391 raise ValueError('b must have same number of columns as x')
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	392 if y_idx.shape[0] != x.shape[0]:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	393 raise ValueError('y_idx must have same number of rows as x')
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	394
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	395 sm = numpy.zeros_like(x) # softmax
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	396 nll = numpy.zeros(x.shape[0]) #nll(y \| softmax(x))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	397 am = numpy.zeros_like(y_idx)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	398 for i in xrange(sm.shape[0]):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	399 #add the bias vector to the i'th row of x
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	400 row = x[i] + b
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	401
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	402 #get the maximum value of i'th row for numerically safe softmax / nll
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	403 am[i] = numpy.argmax(row)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	404 m = row[am[i]]
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	405
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	406 #compute the unnormalized softmax, and normalization constant
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	407 sm[i] = numpy.exp(row - m)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	408 sum_j = numpy.sum(sm[i]) # sum_j(exp(x[j] - m))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	409
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	410 #normalized our softmax
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	411 sm[i] *= 1.0 / sum_j
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	412
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	413 # store the nll
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	414 nll[i] = -row[y_idx[i]] + m + numpy.log(sum_j)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	415
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	416 output_storage[0][0] = nll
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	417 output_storage[1][0] = sm
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	418 output_storage[2][0] = am
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	419 def grad(self, (x, b, y_idx), (g_nll, g_sm, g_am)):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	420 if g_sm is not None or g_am is not None:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	421 raise NotImplementedError()
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	422 nll, sm = crossentropy_softmax_1hot_with_bias(x, b, y_idx)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	423 dx = CrossentropySoftmax1HotWithBiasDx()(g_nll, sm, y_idx)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	424 db = tensor.sum(dx, axis = [0])
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	425 return dx, db, None
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	426
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	427 def c_headers(self): return ['<iostream>']
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	428
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	429 @staticmethod
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	430 def c_code_template():
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	431 # this implementation was lifted from
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	432 # /u/bergstrj/cvs/bergstrj/src/feb07/nn.cxx
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	433
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	434 #TODO: put this into a templated function, in the support code
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	435 #TODO: declare the max of each row as an Op output
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	436
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	437 #TODO: set error messages for failures in this code
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	438
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	439 #TODO: use this to accept float32 and int32: node.inputs[0].type.dtype_specs()[1]
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	440 (init_decl, begin_row_loop, inside_row_loop, end_row_loop) = \
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	441 SoftmaxWithBias.c_code_template()
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	442 return (init_decl,
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	443 """
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	444 if (%(y_idx)s->nd != 1)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	445 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	446 PyErr_SetString(PyExc_ValueError, "y_idx not 1d tensor");
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	447 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	448 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	449 if ((%(y_idx)s->descr->type_num != PyArray_INT64)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	450 && (%(y_idx)s->descr->type_num != PyArray_INT32)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	451 && (%(y_idx)s->descr->type_num != PyArray_INT16)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	452 && (%(y_idx)s->descr->type_num != PyArray_INT8))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	453 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	454 PyErr_SetString(PyExc_TypeError, "y_idx not int8, int16, int32, or int64");
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	455 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	456 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	457 if (%(x)s->dimensions[0] != %(y_idx)s->dimensions[0])
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	458 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	459 PyErr_SetString(PyExc_ValueError, "dimension mismatch in arguments");
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	460 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	461 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	462
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	463 if ((NULL == %(nll)s) //initial condition
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	464 \|\| (%(nll)s->dimensions[0] != %(y_idx)s->dimensions[0]))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	465 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	466 if (NULL != %(nll)s) Py_XDECREF(%(nll)s);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	467 %(nll)s = (PyArrayObject*)PyArray_SimpleNew(1, PyArray_DIMS(%(y_idx)s), type_num_%(x)s);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	468 if(!%(nll)s)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	469 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	470 PyErr_SetString(PyExc_MemoryError, "failed to alloc nll output");
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	471 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	472 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	473 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	474 if ((NULL == %(am)s)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	475 \|\| (%(am)s->dimensions[0] != %(y_idx)s->dimensions[0]))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	476 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	477 Py_XDECREF(%(am)s);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	478 %(am)s = (PyArrayObject*) PyArray_SimpleNew(1, PyArray_DIMS(%(y_idx)s), type_num_%(y_idx)s);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	479 if(!%(am)s)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	480 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	481 PyErr_SetString(PyExc_MemoryError, "failed to alloc am output");
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	482 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	483 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	484 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	485 """,
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	486 begin_row_loop,
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	487 """
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	488 const %(y_idx_type)s y_i = ((%(y_idx_type)s)(%(y_idx)s->data + %(y_idx)s->strides[0] i))[0];
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	489 double* __restrict__ nll_i = (double)(%(nll)s->data + %(nll)s->strides[0] i);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	490 %(am_type)s* __restrict__ am_i = (%(am_type)s) (%(am)s->data + %(am)s->strides[0] i);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	491 """,
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	492 inside_row_loop,
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	493 """
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	494 nll_i[0] = - x_i[y_i*Sx]
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	495 - b_i[y_i*Sb]
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	496 + row_max
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	497 + log(sum);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	498 am_i[0] = row_max_j;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	499 """,
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	500 end_row_loop)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	501
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	502
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	503 def c_code(self, node, name, (x, b, y_idx), (nll, sm, am), sub):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	504 y_idx_type = node.inputs[2].type.dtype_specs()[1]
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	505 am_type = y_idx_type
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	506 code_template = ''.join(self.c_code_template())
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	507 return code_template % dict(locals(), **sub)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	508
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	509 class CrossentropySoftmax1HotWithBiasDx (theano.Op):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	510 nin=3
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	511 nout=1
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	512 """Gradient wrt x of the CrossentropySoftmax1Hot Op"""
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	513 def __init__(self, **kwargs):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	514 theano.Op.__init__(self,**kwargs)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	515 def make_node(self, dy, sm, y_idx,**kwargs):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	516 dy = tensor.as_tensor(dy)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	517 sm = tensor.as_tensor(sm)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	518 y_idx = tensor.as_tensor(y_idx)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	519 return theano.Apply(self, [dy, sm, y_idx],[sm.type.make_result()])
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	520 def perform(self, node, input_storage, output_storage):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	521 dy,sm,y_idx = input_storage
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	522 dx = numpy.zeros_like(sm)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	523 for i in xrange(sm.shape[0]):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	524 dx[i] = dy[i] * sm[i] #vector scale
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	525 dx[i, y_idx[i]] -= dy[i] #scalar decrement
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	526 output_storage[0][0] = dx
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	527 def grad(self, *args):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	528 raise NotImplementedError()
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	529 def c_code(self, node, name, (dnll, sm, y_idx), (dx,), sub):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	530 y_idx_type = node.inputs[2].type.dtype_specs()[1]
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	531 return """
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	532
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	533 if ((%(dnll)s->descr->type_num != PyArray_DOUBLE)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	534 \|\| (%(sm)s->descr->type_num != PyArray_DOUBLE)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	535 )
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	536 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	537 PyErr_SetString(PyExc_TypeError, "types should be float64, float64, int64");
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	538 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	539 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	540 if ((%(y_idx)s->descr->type_num != PyArray_INT64)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	541 && (%(y_idx)s->descr->type_num != PyArray_INT32)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	542 && (%(y_idx)s->descr->type_num != PyArray_INT16)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	543 && (%(y_idx)s->descr->type_num != PyArray_INT8))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	544 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	545 PyErr_SetString(PyExc_TypeError, "y_idx not int8, int16, int32, or int64");
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	546 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	547 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	548 if ((%(dnll)s->nd != 1)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	549 \|\| (%(sm)s->nd != 2)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	550 \|\| (%(y_idx)s->nd != 1))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	551 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	552 PyErr_SetString(PyExc_ValueError, "rank error");
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	553 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	554 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	555 if ((%(dnll)s->dimensions[0] != %(sm)s->dimensions[0])
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	556 \|\| (%(dnll)s->dimensions[0] != %(y_idx)s->dimensions[0]))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	557 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	558 PyErr_SetString(PyExc_ValueError, "dimension mismatch");
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	559 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	560 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	561 if ((NULL == %(dx)s)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	562 \|\| (%(dx)s->dimensions[0] != %(sm)s->dimensions[0])
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	563 \|\| (%(dx)s->dimensions[1] != %(sm)s->dimensions[1]))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	564 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	565 if (NULL != %(dx)s) Py_XDECREF(%(dx)s);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	566 %(dx)s = (PyArrayObject*)PyArray_SimpleNew(2, PyArray_DIMS(%(sm)s), type_num_%(sm)s);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	567 if(!%(dx)s) {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	568 PyErr_SetString(PyExc_MemoryError, "failed to alloc dx output");
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	569 %(fail)s
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	570 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	571 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	572
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	573 for (size_t i = 0; i < %(dx)s->dimensions[0]; ++i)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	574 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	575 const double dnll_i = ((double)(%(dnll)s->data + %(dnll)s->strides[0] i))[0];
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	576
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	577 const %(y_idx_type)s y_i = ((%(y_idx_type)s)(%(y_idx)s->data + %(y_idx)s->strides[0] i))[0];
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	578
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	579 const double* __restrict__ sm_i = (double)(%(sm)s->data + %(sm)s->strides[0] i);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	580 npy_intp Ssm = %(sm)s->strides[1]/sizeof(double);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	581
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	582 double* __restrict__ dx_i = (double)(%(dx)s->data + %(dx)s->strides[0] i);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	583 npy_intp Sdx = %(dx)s->strides[1]/sizeof(double);
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	584
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	585 for (size_t j = 0; j < %(dx)s->dimensions[1]; ++j)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	586 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	587 dx_i[j * Sdx] = dnll_i * sm_i[j * Ssm];
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	588 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	589 if (y_i >= %(dx)s->dimensions[1])
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	590 {
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	591 %(fail)s;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	592 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	593 dx_i[y_i * Sdx] -= dnll_i;
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	594 }
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	595 """ % dict(locals(), **sub)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	596
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	597 crossentropy_softmax_argmax_1hot_with_bias = \
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	598 CrossentropySoftmaxArgmax1HotWithBias()
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	599
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	600 def crossentropy_softmax_1hot_with_bias(x, b, y_idx, **kwargs):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	601 return crossentropy_softmax_argmax_1hot_with_bias(x, b, y_idx, **kwargs)[0:2]
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	602
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	603 def crossentropy_softmax_1hot(x, y_idx, **kwargs):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	604 b = tensor.zeros_like(x[0,:])
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	605 return crossentropy_softmax_1hot_with_bias(x, b, y_idx, **kwargs)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	606
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	607
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	608 class MultinomialCrossentropy1Hot(theano.Op):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	609 pass
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	610
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	611
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	612 def binary_crossentropy(output, target):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	613 """
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	614 Compute the crossentropy of binary output wrt binary target.
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	615 @note: We do not sum, crossentropy is computed by component.
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	616 @todo: Rewrite as a scalar, and then broadcast to tensor.
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	617 @todo: This is essentially duplicated as cost.cross_entropy
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	618 @warning: OUTPUT and TARGET are reversed in cost.cross_entropy
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	619 """
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	620 return -(target * tensor.log(output) + (1 - target) * tensor.log(1 - output))
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	621
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	622
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	623
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	624 class Prepend_scalar_constant_to_each_row(theano.Op):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	625 def __init__(self, val = 0):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	626 if isinstance(val, float):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	627 val = scalar.constant(val)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	628 self.val = val
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	629
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	630 def make_node(self, mat):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	631 #check type of input
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	632 if not isinstance(mat,theano.Result) or not mat.type==tensor.matrix().type:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	633 raise TypeError("Expected a matrix as input")
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	634 x = tensor.as_tensor(mat)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	635 y = tensor.as_tensor(self.val)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	636 if x.type.dtype != y.type.dtype:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	637 TypeError("the value to prepend don't have the same type as the matrix")
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	638
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	639 node = theano.Apply(op=self, inputs=[mat], outputs=[tensor.matrix()])
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	640 return node
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	641
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	642 def perform(self, node, (mat, ), (output, )):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	643 new_shape=(mat.shape[0],mat.shape[1]+1)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	644 if output[0] == None:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	645 output[0]=numpy.empty(new_shape,dtype=mat.dtype)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	646 out=output[0]
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	647 else:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	648 if output[0].shape!=new_shape:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	649 try:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	650 output[0].resize(new_shape)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	651 except:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	652 output[0]=numpy.empty(new_shape, dtype=mat.dtype)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	653 out=output[0]
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	654
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	655 out[:,0].fill(self.val.data)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	656 out[:,1:]=mat
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	657
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	658 def grad(self, (mat,), (goutput,)):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	659 return goutput[:,1:]
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	660
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	661 class Prepend_scalar_to_each_row(theano.Op):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	662 def make_node(self, val, mat):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	663 #check type of input
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	664 if isinstance(val, float):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	665 val = scalar.constant(val)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	666 if not isinstance(mat,theano.Result) or not mat.type==tensor.matrix().type:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	667 raise TypeError("Expected a matrix as input")
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	668 x = tensor.as_tensor(mat)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	669 y = tensor.as_tensor(val)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	670 if x.type.dtype != y.type.dtype:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	671 TypeError("the value to prepend don't have the same type as the matrix")
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	672
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	673 node = theano.Apply(op=self, inputs=[val,mat], outputs=[tensor.matrix()])
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	674 return node
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	675
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	676 def perform(self, node, (val,mat), (output, )):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	677 new_shape=(mat.shape[0],mat.shape[1]+1)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	678 if output[0] == None:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	679 output[0]=numpy.empty(new_shape,dtype=mat.dtype)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	680 out=output[0]
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	681 else:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	682 if output[0].shape!=new_shape:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	683 try:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	684 output[0].resize(new_shape)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	685 except:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	686 output[0]=numpy.empty(new_shape, dtype=mat.dtype)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	687 out=output[0]
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	688 out[:,0].fill(val)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	689 out[:,1:]=mat
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	690
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	691 def grad(self, (val, mat), (goutput,)):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	692 return goutput[:,0], goutput[:,1:]
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	693
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	694 prepend_scalar_to_each_row = Prepend_scalar_to_each_row()
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	695 prepend_0_to_each_row = Prepend_scalar_constant_to_each_row(0.)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	696 prepend_1_to_each_row = Prepend_scalar_constant_to_each_row(1.)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	697
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	698 class solve(theano.Op):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	699 """
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	700 Find the solution to the linear equation Ax=b,
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	701 where A is a 2d matrix and b is a 1d or 2d matrix.
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	702 It use numpy.solve to find the solution.
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	703 """
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	704
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	705 def make_node(self, A, b):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	706 if not isinstance(A, theano.Result) or not A.type==tensor.matrix().type:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	707 raise TypeError("We expected that A had a matrix type")
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	708 if not isinstance(B, theano.Result) or not B.type==tensor.matrix().type:
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	709 raise TypeError("We expected that B had a matrix type")
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	710
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	711 node = theano.Apply(op=self, inputs=[A, B], outputs=[tensor.matrix()])
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	712 return node
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	713
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	714 def perform(self, node, (A, B), (output, )):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	715 ret=numpy.solve(A,B)
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	716 output[0]=ret
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	717
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	718 def grad(self, (theta, A, B), (gtheta,)):
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	719 raise NotImplementedError()
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	720
34acf8db186d Deprecated pylearn.nnet_ops. Joseph Turian <turian@iro.umontreal.ca> parents: 449 diff changeset	721

Mercurial > pylearn

annotate nnet_ops.py @ 472:69c800af1370