Mercurial > pylearn
comparison pylearn/shared/layers/exponential_mean.py @ 834:580087712f69
added shared.layers
| author | James Bergstra <bergstrj@iro.umontreal.ca> |
|---|---|
| date | Fri, 16 Oct 2009 12:14:43 -0400 |
| parents | |
| children | 912be602c3ac |
comparison
equal
deleted
inserted
replaced
| 833:039e93a95c20 | 834:580087712f69 |
|---|---|
| 1 | |
| 2 """Modules for maintaining statistics based on exponential decay""" | |
| 3 __docformat__ = "restructuredtext en" | |
| 4 | |
| 5 import copy | |
| 6 import numpy | |
| 7 import theano | |
| 8 import theano.tensor | |
| 9 from theano.compile.sandbox import shared | |
| 10 | |
| 11 class ExponentialMean(object): | |
| 12 """Maintain an exponentially-decaying estimate of the mean | |
| 13 | |
| 14 This module computes the exact mean of the first `max_denom` values of `x`. | |
| 15 After the first `max_denom` values, it tracks the mean using the formula: | |
| 16 | |
| 17 :math:`self.running <- (1.0 - (1.0/max_denom)) * self.running + (1.0/max_denom) * x` | |
| 18 | |
| 19 """ | |
| 20 | |
| 21 max_denom = None | |
| 22 """The average will be updated as if the current estimated average was estimated from at | |
| 23 most `max_denom-1` values.""" | |
| 24 | |
| 25 running = None | |
| 26 """Shared: The running mean statistic from which the output is computed.""" | |
| 27 | |
| 28 denom = None | |
| 29 """Shared: The number of examples we've updated from so far | |
| 30 """ | |
| 31 | |
| 32 def __init__(self, input, max_denom, ival): | |
| 33 """ | |
| 34 :param input: track the mean of this Variable | |
| 35 | |
| 36 :param max_denom: see `self.max_denom` | |
| 37 | |
| 38 :param ival: This should be a tensor of zeros with a shape that matches `input`'s runtime | |
| 39 value. | |
| 40 | |
| 41 """ | |
| 42 dtype=ival.dtype #dtype is an actual numpy dtype object, not a string | |
| 43 self.max_denom = max_denom | |
| 44 | |
| 45 if len(ival.shape) == 0: | |
| 46 input_type = theano.tensor.dscalar | |
| 47 elif len(ival.shape) == 1: | |
| 48 input_type = theano.tensor.dvector | |
| 49 elif len(ival.shape) == 2: | |
| 50 input_type = theano.tensor.dmatrix | |
| 51 else: | |
| 52 #TODO: x_type = theano.tensor.TensorType(...) | |
| 53 raise NotImplementedError() | |
| 54 | |
| 55 self.running = shared(numpy.array(ival, copy=True)) | |
| 56 # TODO: making this an lscalar caused different optimizations, followed by integer | |
| 57 # division somewhere were I wanted float division.... and the wrong answer. | |
| 58 self.denom = shared(numpy.asarray(1, dtype=dtype)) | |
| 59 | |
| 60 alpha = 1.0 / self.denom | |
| 61 self.output = (1.0 - alpha) * self.running + theano.tensor.cast(alpha * input, str(dtype)) | |
| 62 | |
| 63 self.updates = [ | |
| 64 (self.running, self.output), | |
| 65 (self.denom, theano.tensor.smallest(self.denom + 1, self.max_denom)), | |
| 66 ] | |
| 67 | |
| 68 assert self.output.type.dtype == dtype | |
| 69 | |
| 70 @classmethod | |
| 71 def new(cls, x, x_shape, max_denom, dtype='float64'): | |
| 72 """Return an `ExponentialMean` to track a Variable `x` with given shape | |
| 73 | |
| 74 :type x: Variable | |
| 75 :type x_shape: tuple | |
| 76 :type max_denom: int | |
| 77 :type dtype: string | |
| 78 :param dtype: the running average will be computed at this precision | |
| 79 | |
| 80 :rtype: ExponentialMean instance | |
| 81 """ | |
| 82 return cls(x, | |
| 83 max_denom=max_denom, | |
| 84 ival=numpy.zeros(x_shape, dtype=dtype)) |