Mercurial > pylearn

changeset 405:be4209cd568f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added weight decay
author Joseph Turian <turian@gmail.com>
date Thu, 10 Jul 2008 01:17:40 -0400
parents 8cc11ac97087
children c2e6a8fcc35e
files sandbox/rbm/globals.py sandbox/rbm/model.py
diffstat 2 files changed, 8 insertions(+), 3 deletions(-) [+]
line wrap: on
line diff
--- a/sandbox/rbm/globals.py	Thu Jul 10 00:51:32 2008 -0400
+++ b/sandbox/rbm/globals.py	Thu Jul 10 01:17:40 2008 -0400
@@ -2,13 +2,14 @@
 Global variables.
 """
 
-INPUT_DIMENSION = 10
+INPUT_DIMENSION = 1000
 #INPUT_DIMENSION = 100
 HIDDEN_DIMENSION = 100
 #HIDDEN_DIMENSION = 10
 #HIDDEN_DIMENSION = 6
 LEARNING_RATE = 0.1
 LR = LEARNING_RATE
-#MOMENTUM = 0.9
-MOMENTUM = 0
+MOMENTUM = 0.9
+#MOMENTUM = 0
+WEIGHT_DECAY = 0.0002
 SEED = 666
--- a/sandbox/rbm/model.py	Thu Jul 10 00:51:32 2008 -0400
+++ b/sandbox/rbm/model.py	Thu Jul 10 01:17:40 2008 -0400
@@ -79,6 +79,8 @@
         @param instance: A dict from feature index to (non-zero) value.
         @todo: Should assert that nonzero_indices and zero_indices
         are correct (i.e. are truly nonzero/zero).
+        @todo: Multiply WEIGHT_DECAY by LEARNING_RATE, as done in Semantic Hashing?
+        @todo: Decay the biases too?
         """
         minibatch = len(instances)
         v0 = pylearn.sparse_instance.to_vector(instances, globals.INPUT_DIMENSION)
@@ -93,6 +95,8 @@
         db = LR * numpy.sum(h0 - q1, axis=0) / minibatch + globals.MOMENTUM * self.prev_db
         dc = LR * numpy.sum(v0 - v1, axis=0) / minibatch + globals.MOMENTUM * self.prev_dc
 
+        self.parameters.w *= (1 - globals.WEIGHT_DECAY)
+
         self.parameters.w += dw
         self.parameters.b += db
         self.parameters.c += dc