# HG changeset patch
# User James Bergstra <bergstrj@iro.umontreal.ca>
# Date 1296853647 18000
# Node ID 8c209c8470873e389e78a6233652dc4fe5361eab
# Parent  3dee72c3055d184b24e987f37c0936400dfc0000
adding delta-bar-delta optimization updates to gd module

diff -r 3dee72c3055d -r 8c209c847087 pylearn/gd/dbd.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/pylearn/gd/dbd.py	Fri Feb 04 16:07:27 2011 -0500
@@ -0,0 +1,66 @@
+"""
+Delta-Bar-Delta gradient descent algorithm.
+
+Reference: TODO
+
+Math: TODO
+"""
+import sys
+import numpy
+from theano import shared, tensor
+
+def dbd_updates(params, grads, stepsizes, cost,
+        global_step_inc=1.1,
+        global_step_dec=0.5,
+        multiplier_min=.1,
+        multiplier_max=10,
+        multiplier_inc=.05, #additive
+        multiplier_dec=.95, #multiplicative
+        ):
+    """
+    Parameter description TODO.
+
+    Return value TODO.
+    """
+    dtype = cost.dtype
+    if grads is None:
+        grads = tensor.grad(cost, params)
+
+    paramvals = [p.get_value(borrow=False) for p in params]
+
+    last_params = [shared(numpy.asarray(pv)) for pv in paramvals]
+    last_grads = [shared(numpy.zeros_like(pv)) for pv in paramvals]
+    multipliers = [shared(numpy.ones_like(pv)) for pv in paramvals]
+    global_stepsize = shared(numpy.asarray(1.0, dtype=dtype))
+
+    #DebugMode complains by default about inf
+    last_cost = shared(numpy.asarray(sys.maxint, dtype=dtype))
+
+    ups = dict()
+    cost_improvement = (cost < last_cost)
+    ups[last_cost] = tensor.switch(cost_improvement, cost, last_cost)
+    ups[global_stepsize] = new_gs = tensor.switch(cost_improvement,
+            global_step_inc*global_stepsize,
+            global_step_dec*global_stepsize,)
+    for lp, p in zip(last_params, params):
+        ups[lp] = tensor.switch(cost_improvement, p, lp)
+    for lg, g in zip(last_grads, grads):
+        ups[lg] = tensor.switch(cost_improvement, g, lg)
+    for m, lg, g in zip(multipliers, last_grads, grads):
+        ups[m] = tensor.switch(cost_improvement,
+                tensor.clip(
+                    tensor.switch(g*lg >= 0,
+                        m+multiplier_inc,
+                        m*multiplier_dec),
+                    multiplier_min,
+                    multiplier_max),
+                m)
+    for lp, p, lg, g, stepsize, m in zip(
+            last_params, params,
+            last_grads, grads,
+            stepsizes, multipliers):
+        ups[p] = tensor.switch(cost_improvement,
+                p - new_gs*ups[m]*stepsize*g,
+                lp - new_gs*ups[m]*stepsize*lg)
+    return ups
+
diff -r 3dee72c3055d -r 8c209c847087 pylearn/gd/tests/test_dbd.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/pylearn/gd/tests/test_dbd.py	Fri Feb 04 16:07:27 2011 -0500
@@ -0,0 +1,23 @@
+import theano
+from theano.compile.debugmode import DebugMode
+from pylearn.gd import dbd
+
+mode = theano.compile.mode.get_default_mode()
+if isinstance(mode,DebugMode):
+    mode = 'FAST_RUN'
+
+def test_dbd_basic():
+
+    x = theano.shared(5.0)
+    y = theano.shared(3.0)
+
+    cost = (1.0 - x * y)**2
+    ups = dbd.dbd_updates([x,y], grads=None, stepsizes=[.01,.01],
+            cost = cost)
+    fn = theano.function([], cost, updates=ups)
+    c_i = fn()
+    assert c_i > 20
+    for i in xrange(20):
+        c_i = fn()
+    assert c_i < 1.0e-10
+