Mercurial > pylearn

comparison pylearn/gd/sgd.py @ 1474:a57f4839a9d8

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
merge
author James Bergstra <bergstrj@iro.umontreal.ca>
date Wed, 18 May 2011 10:52:42 -0400
parents ddda8d93c162 cac29ca79a74
children 0e6ca7eecc72
comparison
equal deleted inserted replaced
1473:91a475ca9b6d 1474:a57f4839a9d8
14 :param stepsizes: step by this amount times the negative gradient on each iteration 14 :param stepsizes: step by this amount times the negative gradient on each iteration
15 :type stepsizes: [symbolic] scalar or list of one [symbolic] scalar per param 15 :type stepsizes: [symbolic] scalar or list of one [symbolic] scalar per param
16 """ 16 """
17 try: 17 try:
18 iter(stepsizes) 18 iter(stepsizes)
19 except: 19 except Exception:
20 stepsizes = [stepsizes for p in params] 20 stepsizes = [stepsizes for p in params]
21 if len(params) != len(grads): 21 if len(params) != len(grads):
22 raise ValueError('params and grads have different lens') 22 raise ValueError('params and grads have different lens')
23 updates = [(p, p - step * gp) for (step, p, gp) in zip(stepsizes, params, grads)] 23 updates = [(p, p - step * gp) for (step, p, gp) in zip(stepsizes, params, grads)]
24 return updates 24 return updates
25 25
26 def sgd_momentum_updates(params, grads, stepsizes, momentum=0.9): 26 def sgd_momentum_updates(params, grads, stepsizes, momentum=0.9):
27 # if stepsizes is just a scalar, expand it to match params 27 # if stepsizes is just a scalar, expand it to match params
28 try: 28 try:
29 iter(stepsizes) 29 iter(stepsizes)
30 except: 30 except Exception:
31 stepsizes = [stepsizes for p in params] 31 stepsizes = [stepsizes for p in params]
32 try: 32 try:
33 iter(momentum) 33 iter(momentum)
34 except: 34 except Exception:
35 momentum = [momentum for p in params] 35 momentum = [momentum for p in params]
36 if len(params) != len(grads): 36 if len(params) != len(grads):
37 raise ValueError('params and grads have different lens') 37 raise ValueError('params and grads have different lens')
38 headings = [theano.shared(numpy.zeros_like(p.get_value(borrow=True))) for p in params] 38 headings = [theano.shared(numpy.zeros_like(p.get_value(borrow=True))) for p in params]
39 updates = [] 39 updates = []
77 77
78 if self.stepsize.ndim != 0: 78 if self.stepsize.ndim != 0:
79 raise TypeError('stepsize must be a scalar', stepsize) 79 raise TypeError('stepsize must be a scalar', stepsize)
80 80
81 self.params = params 81 self.params = params
82 self.gparams = theano.tensor.grad(cost, self.params) if gradients is None else gradients 82 self.gparams = [theano.tensor.grad(cost, self.params)] if gradients is None else gradients
83 assert len(self.params) == len(self.gparams)
83 84
84 self._updates = (dict((p, p - self.stepsize * g) for p, g in zip(self.params, self.gparams))) 85 self._updates = (dict((p, p - self.stepsize * g) for p, g in zip(self.params, self.gparams)))
85 if updates is not None: 86 if updates is not None:
86 self._updates.update(updates) 87 self._updates.update(updates)
87 88