Mercurial > pylearn

comparison pylearn/gd/sgd.py @ 1472:ddda8d93c162

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
dtype tweaks in sgd
author James Bergstra <bergstrj@iro.umontreal.ca>
date Wed, 18 May 2011 10:51:50 -0400
parents 86bf03990aad
children a57f4839a9d8
comparison
equal deleted inserted replaced
1471:281efa9a4463 1472:ddda8d93c162
1 """A stochastic gradient descent minimizer. 1 """A stochastic gradient descent minimizer.
2 """ 2 """
3 3 import numpy
4 import theano 4 import theano
5 5
6 def sgd_updates(params, grads, stepsizes): 6 def sgd_updates(params, grads, stepsizes):
7 """Return a list of (pairs) that can be used as updates in theano.function to implement 7 """Return a list of (pairs) that can be used as updates in theano.function to implement
8 stochastic gradient descent. 8 stochastic gradient descent.
33 iter(momentum) 33 iter(momentum)
34 except: 34 except:
35 momentum = [momentum for p in params] 35 momentum = [momentum for p in params]
36 if len(params) != len(grads): 36 if len(params) != len(grads):
37 raise ValueError('params and grads have different lens') 37 raise ValueError('params and grads have different lens')
38 headings = [theano.shared(p.get_value(borrow=False)*0) for p in params] 38 headings = [theano.shared(numpy.zeros_like(p.get_value(borrow=True))) for p in params]
39 updates = [] 39 updates = []
40 for s, p, gp, m, h in zip(stepsizes, params, grads, momentum, headings): 40 for s, p, gp, m, h in zip(stepsizes, params, grads, momentum, headings):
41 updates.append((p, p + s * h)) 41 updates.append((p, p + s * h))
42 updates.append((h, m*h - (1-m)*gp)) 42 updates.append((h, m*h - (1.0-m)*gp))
43 return updates 43 return updates
44 44
45 45
46 class StochasticGradientDescent(theano.Module): 46 class StochasticGradientDescent(theano.Module):
47 """Fixed stepsize gradient descent 47 """Fixed stepsize gradient descent