Mercurial > pylearn
comparison pylearn/gd/dbd.py @ 1422:8c209c847087
adding delta-bar-delta optimization updates to gd module
| author | James Bergstra <bergstrj@iro.umontreal.ca> |
|---|---|
| date | Fri, 04 Feb 2011 16:07:27 -0500 |
| parents | |
| children |
comparison
equal
deleted
inserted
replaced
| 1421:3dee72c3055d | 1422:8c209c847087 |
|---|---|
| 1 """ | |
| 2 Delta-Bar-Delta gradient descent algorithm. | |
| 3 | |
| 4 Reference: TODO | |
| 5 | |
| 6 Math: TODO | |
| 7 """ | |
| 8 import sys | |
| 9 import numpy | |
| 10 from theano import shared, tensor | |
| 11 | |
| 12 def dbd_updates(params, grads, stepsizes, cost, | |
| 13 global_step_inc=1.1, | |
| 14 global_step_dec=0.5, | |
| 15 multiplier_min=.1, | |
| 16 multiplier_max=10, | |
| 17 multiplier_inc=.05, #additive | |
| 18 multiplier_dec=.95, #multiplicative | |
| 19 ): | |
| 20 """ | |
| 21 Parameter description TODO. | |
| 22 | |
| 23 Return value TODO. | |
| 24 """ | |
| 25 dtype = cost.dtype | |
| 26 if grads is None: | |
| 27 grads = tensor.grad(cost, params) | |
| 28 | |
| 29 paramvals = [p.get_value(borrow=False) for p in params] | |
| 30 | |
| 31 last_params = [shared(numpy.asarray(pv)) for pv in paramvals] | |
| 32 last_grads = [shared(numpy.zeros_like(pv)) for pv in paramvals] | |
| 33 multipliers = [shared(numpy.ones_like(pv)) for pv in paramvals] | |
| 34 global_stepsize = shared(numpy.asarray(1.0, dtype=dtype)) | |
| 35 | |
| 36 #DebugMode complains by default about inf | |
| 37 last_cost = shared(numpy.asarray(sys.maxint, dtype=dtype)) | |
| 38 | |
| 39 ups = dict() | |
| 40 cost_improvement = (cost < last_cost) | |
| 41 ups[last_cost] = tensor.switch(cost_improvement, cost, last_cost) | |
| 42 ups[global_stepsize] = new_gs = tensor.switch(cost_improvement, | |
| 43 global_step_inc*global_stepsize, | |
| 44 global_step_dec*global_stepsize,) | |
| 45 for lp, p in zip(last_params, params): | |
| 46 ups[lp] = tensor.switch(cost_improvement, p, lp) | |
| 47 for lg, g in zip(last_grads, grads): | |
| 48 ups[lg] = tensor.switch(cost_improvement, g, lg) | |
| 49 for m, lg, g in zip(multipliers, last_grads, grads): | |
| 50 ups[m] = tensor.switch(cost_improvement, | |
| 51 tensor.clip( | |
| 52 tensor.switch(g*lg >= 0, | |
| 53 m+multiplier_inc, | |
| 54 m*multiplier_dec), | |
| 55 multiplier_min, | |
| 56 multiplier_max), | |
| 57 m) | |
| 58 for lp, p, lg, g, stepsize, m in zip( | |
| 59 last_params, params, | |
| 60 last_grads, grads, | |
| 61 stepsizes, multipliers): | |
| 62 ups[p] = tensor.switch(cost_improvement, | |
| 63 p - new_gs*ups[m]*stepsize*g, | |
| 64 lp - new_gs*ups[m]*stepsize*lg) | |
| 65 return ups | |
| 66 |