Mercurial > pylearn
view pylearn/formulas/activations.py @ 1315:f21693eecec7
added several activation formulas
author | Olivier Breuleux <breuleux@gmail.com> |
---|---|
date | Thu, 07 Oct 2010 16:42:12 -0400 |
parents | 970082c8e9de |
children | 00116be92710 |
line wrap: on
line source
""" Activation function for artificial neural units. """ __authors__ = "Razvan Pascanu, .." __copyright__ = "(c) 2010, Universite de Montreal" __license__ = "3-clause BSD License" __contact__ = "Razvan Pascanu <r.pascanu@gmail.com>" import theano from theano import tensor import tags @tags.tags('activation', 'unary', 'sigmoid', 'logistic', 'non-negative', 'increasing') def sigmoid(x): """ Return a symbolic variable representing the sigmoid (logistic) function of the input x. .. math:: \\textrm{sigmoid}(x) = \\frac{1}{1 + e^x} The image of :math:`\\textrm{sigmoid}(x)` is the open interval (0, 1), *in theory*. *In practice*, due to rounding errors in floating point representations, :math:`\\textrm{sigmoid}(x)` will lie in the closed range [0, 1]. :param x: tensor-like (a Theano variable with type theano.Tensor, or a value that can be converted to one) :math:`\in \mathbb{R}^n` :return: a Theano variable with the same shape as the input, where the sigmoid function is mapped to each element of the input x. """ return theano.tensor.nnet.sigmoid(x) @tags.tags('activation', 'unary', 'tanh', 'hyperbolic tangent', 'odd', 'increasing') def tanh(x): """ Return a symbolic variable representing the tanh (hyperbolic tangent) of the input x. .. math:: \\textrm{tanh}(x) = \\frac{e^{2x} - 1}{e^{2x} + 1} The image of :math:`\\textrm{tanh}(x)` is the open interval (-1, 1), *in theory*. *In practice*, due to rounding errors in floating point representations, :math:`\\textrm{tanh}(x)` will lie in the closed range [-1, 1]. :param x: tensor-like (a Theano variable with type theano.Tensor, or a value that can be converted to one) :math:`\in \mathbb{R}^n` :return: a Theano variable with the same shape as the input, where the tanh function is mapped to each element of the input x. """ return theano.tensor.tanh(x) @tags.tags('activation', 'unary', 'tanh', 'hyperbolic tangent', 'normalized', 'odd', 'increasing') def tanh_normalized(x): """ Return a symbolic variable representing a normalized tanh (hyperbolic tangent) of the input x. TODO: where does 1.759 come from? why is it normalized like that? .. math:: \\textrm{tanh\_normalized}(x) = 1.759\\textrm{ tanh}\left(\\frac{2x}{3}\\right) The image of :math:`\\textrm{tanh\_normalized}(x)` is the open interval (-1.759, 1.759), *in theory*. *In practice*, due to rounding errors in floating point representations, :math:`\\textrm{tanh\_normalized}(x)` will lie in the approximative closed range [-1.759, 1.759]. The exact bound depends on the precision of the floating point representation. :param x: tensor-like (a Theano variable with type theano.Tensor, or a value that can be converted to one) :math:`\in \mathbb{R}^n` :return: a Theano variable with the same shape as the input, where the tanh\_normalized function is mapped to each element of the input x. """ return 1.759*theano.tensor.tanh(0.6666*x) @tags.tags('activation', 'unary', 'abs_tanh', 'abs', 'tanh', 'hyperbolic tangent', 'non-negative', 'even') def abs_tanh(x): """ Return a symbolic variable representing the absolute value of the hyperbolic tangent of x. .. math:: \\textrm{abs\_tanh}(x) = |\\textrm{tanh}(x)| The image of :math:`\\textrm{abs\_tanh}(x)` is the interval [0, 1), *in theory*. *In practice*, due to rounding errors in floating point representations, :math:`\\textrm{abs\_tanh}(x)` will lie in the range [0, 1]. :param x: tensor-like (a Theano variable with type theano.Tensor, or a value that can be converted to one) :math:`\in \mathbb{R}^n` :return: a Theano variable with the same shape as the input, where the abs_tanh function is mapped to each element of the input x. """ return theano.tensor.abs_(theano.tensor.tanh(x)) @tags.tags('activation', 'unary', 'abs_tanh', 'abs', 'tanh', 'hyperbolic tangent', 'normalized', 'non-negative', 'even') def abs_tanh_normalized(x): """ Return a symbolic variable representing the absolute value of a normalized tanh (hyperbolic tangent) of the input x. TODO: where does 1.759 come from? why is it normalized like that? .. math:: \\textrm{abs\_tanh\_normalized}(x) = \left|1.759\\textrm{ tanh}\left(\\frac{2x}{3}\\right)\\right| The image of :math:`\\textrm{abs\_tanh\_normalized}(x)` is the range [0, 1.759), *in theory*. *In practice*, due to rounding errors in floating point representations, :math:`\\textrm{abs\_tanh\_normalized}(x)` will lie in the approximative closed range [0, 1.759]. The exact upper bound depends on the precision of the floating point representation. :param x: tensor-like (a Theano variable with type theano.Tensor, or a value that can be converted to one) :math:`\in \mathbb{R}^n` :return: a Theano variable with the same shape as the input, where the abs_tanh_normalized function is mapped to each element of the input x. """ return theano.tensor.abs_(1.759*theano.tensor.tanh(0.6666*x)) @tags.tags('activation','softsign') def softsign_act(input): """ Returns a symbolic variable that computes the softsign of ``input``. .. math:: f(input) = \\frac{input}{1.0 + |input|} :type input: tensor-like :param input: input tensor to which softsign should be applied :rtype: Theano variable :return: tensor obtained after applying the softsign function """ return input/(1.0 + tensor.abs_(input)) @tags.tags('activation','softsign','abs') def abssoftsign_act(input): """ Returns a symbolic variable that computes the absolute value of the softsign function on the input tensor ``input``. .. math:: f(input) = \left| \\frac{input}{1.0 +|input|} \\right| :type input: tensor-like :param input: input tensor to which softsign should be applied :rtype: Tensor variable :return: tensor obtained by taking the absolute value of softsign of the input """ return tensor.abs_(input)/(1.0 + tensor.abs_(input)) @tags.tags('activation','rectifier') def rectifier_act(input): """ Returns a symbolic variable that computes the value of the ``input`` if and only if it is positive, 0 otherwise. .. math:: f(input) = \left \lbrace \\begin{array}{l} input \quad \\text{ if } input > 0 \\ 0 \quad \\text{ else } \end{array} \\right \} :type input: tensor-like :param input: input tensor to which the rectifier activation function will be applied :rtype: Tensor variable :return: always positive tensor which equals with the input if it is also positive or to 0 otherwise """ return input*(input>=0) @tags.tags('activation','softplus') def softplus_act(input): """ Returns a symbolic variable that computes the softplus of ``input``. Note : (TODO) rescale in order to have a steady state regime close to 0 at initialization. .. math:: f(input) = ln \left( 1 + e^{input} \\right) :type input: tensor-like :param input: input tensor to which the softplus should be applied :rtype: Theano variable :return: tensor obtained by applying softsign on the input """ return tensor.nnet.softplus(input) @tags.tags('activation','abs') def abs_act(input): """ Returns the symbolic variable that represents the absolute value of ``input``. .. math:: f(input) = |input| :type input: tensor-like :param input: input tensor :rtype: Theano variable :return: tensor that represents the absolute value of the input """ return theano.tensor.abs_(input)