Mercurial > pylearn
changeset 406:c2e6a8fcc35e
Globals are now parameters for the RBM model
| author | Joseph Turian <turian@gmail.com> |
|---|---|
| date | Thu, 10 Jul 2008 02:10:23 -0400 |
| parents | be4209cd568f |
| children | b9f545594207 |
| files | sandbox/rbm/globals.py sandbox/rbm/main.py sandbox/rbm/model.py sandbox/rbm/parameters.py |
| diffstat | 4 files changed, 24 insertions(+), 35 deletions(-) [+] |
line wrap: on
line diff
--- a/sandbox/rbm/globals.py Thu Jul 10 01:17:40 2008 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,15 +0,0 @@ -""" -Global variables. -""" - -INPUT_DIMENSION = 1000 -#INPUT_DIMENSION = 100 -HIDDEN_DIMENSION = 100 -#HIDDEN_DIMENSION = 10 -#HIDDEN_DIMENSION = 6 -LEARNING_RATE = 0.1 -LR = LEARNING_RATE -MOMENTUM = 0.9 -#MOMENTUM = 0 -WEIGHT_DECAY = 0.0002 -SEED = 666
--- a/sandbox/rbm/main.py Thu Jul 10 01:17:40 2008 -0400 +++ b/sandbox/rbm/main.py Thu Jul 10 02:10:23 2008 -0400 @@ -16,7 +16,7 @@ nonzero_instances.append({1: 0.2, 2: 0.3, 5: 0.5}) import model -model = model.Model() +model = model.Model(input_dimension=10, hidden_dimension=6) for i in xrange(100000): # Select an instance
--- a/sandbox/rbm/model.py Thu Jul 10 01:17:40 2008 -0400 +++ b/sandbox/rbm/model.py Thu Jul 10 02:10:23 2008 -0400 @@ -5,13 +5,9 @@ import parameters -import globals -from globals import LR - import numpy from numpy import dot import random -random.seed(globals.SEED) import pylearn.nnet_ops import pylearn.sparse_instance @@ -53,8 +49,17 @@ """ @todo: input dimensions should be stored here! not as a global. """ - def __init__(self): - self.parameters = parameters.Parameters(randomly_initialize=True) + def __init__(self, input_dimension, hidden_dimension, learning_rate = 0.1, momentum = 0.9, weight_decay = 0.0002, random_seed = 666): + self.input_dimension = input_dimension + self.hidden_dimension = hidden_dimension + self.learning_rate = learning_rate + self.momentum = momentum + self.weight_decay = weight_decay + self.random_seed = random_seed + + random.seed(random_seed) + + self.parameters = parameters.Parameters(input_dimension=self.input_dimension, hidden_dimension=self.hidden_dimension, randomly_initialize=False, random_seed=self.random_seed) self.prev_dw = 0 self.prev_db = 0 self.prev_dc = 0 @@ -79,11 +84,11 @@ @param instance: A dict from feature index to (non-zero) value. @todo: Should assert that nonzero_indices and zero_indices are correct (i.e. are truly nonzero/zero). - @todo: Multiply WEIGHT_DECAY by LEARNING_RATE, as done in Semantic Hashing? + @todo: Multiply L{self.weight_decay} by L{self.learning_rate}, as done in Semantic Hashing? @todo: Decay the biases too? """ minibatch = len(instances) - v0 = pylearn.sparse_instance.to_vector(instances, globals.INPUT_DIMENSION) + v0 = pylearn.sparse_instance.to_vector(instances, self.input_dimension) print "old XENT:", numpy.sum(self.deterministic_reconstruction_error(v0)) q0 = sigmoid(self.parameters.b + dot(v0, self.parameters.w)) h0 = sample(q0) @@ -91,11 +96,11 @@ v1 = sample(p0) q1 = sigmoid(self.parameters.b + dot(v1, self.parameters.w)) - dw = LR * (dot(v0.T, h0) - dot(v1.T, q1)) / minibatch + globals.MOMENTUM * self.prev_dw - db = LR * numpy.sum(h0 - q1, axis=0) / minibatch + globals.MOMENTUM * self.prev_db - dc = LR * numpy.sum(v0 - v1, axis=0) / minibatch + globals.MOMENTUM * self.prev_dc + dw = self.learning_rate * (dot(v0.T, h0) - dot(v1.T, q1)) / minibatch + self.momentum * self.prev_dw + db = self.learning_rate * numpy.sum(h0 - q1, axis=0) / minibatch + self.momentum * self.prev_db + dc = self.learning_rate * numpy.sum(v0 - v1, axis=0) / minibatch + self.momentum * self.prev_dc - self.parameters.w *= (1 - globals.WEIGHT_DECAY) + self.parameters.w *= (1 - self.weight_decay) self.parameters.w += dw self.parameters.b += db @@ -121,14 +126,14 @@ # print h0.shape # print dot(v0.T, h0).shape # print self.parameters.w.shape -# self.parameters.w += LR * (dot(v0.T, h0) - dot(v1.T, q1)) / minibatch +# self.parameters.w += self.learning_rate * (dot(v0.T, h0) - dot(v1.T, q1)) / minibatch # print # print h0.shape # print q1.shape # print self.parameters.b.shape -# self.parameters.b += LR * numpy.sum(h0 - q1, axis=0) / minibatch +# self.parameters.b += self.learning_rate * numpy.sum(h0 - q1, axis=0) / minibatch # print v0.shape, v1.shape # print # print self.parameters.c.shape -# self.parameters.c += LR * numpy.sum(v0 - v1, axis=0) / minibatch +# self.parameters.c += self.learning_rate * numpy.sum(v0 - v1, axis=0) / minibatch # print self.parameters
--- a/sandbox/rbm/parameters.py Thu Jul 10 01:17:40 2008 -0400 +++ b/sandbox/rbm/parameters.py Thu Jul 10 02:10:23 2008 -0400 @@ -3,20 +3,19 @@ """ import numpy -import globals class Parameters: """ Parameters used by the L{Model}. """ - def __init__(self, input_dimension=globals.INPUT_DIMENSION, hidden_dimension=globals.HIDDEN_DIMENSION, randomly_initialize=False, seed=globals.SEED): + def __init__(self, input_dimension, hidden_dimension, randomly_initialize, random_seed): """ Initialize L{Model} parameters. @param randomly_initialize: If True, then randomly initialize - according to the given seed. If False, then just use zeroes. + according to the given random_seed. If False, then just use zeroes. """ if randomly_initialize: - numpy.random.seed(seed) + numpy.random.random_seed(random_seed) self.w = (numpy.random.rand(input_dimension, hidden_dimension)-0.5)/input_dimension self.b = numpy.zeros((1, hidden_dimension)) self.c = numpy.zeros((1, input_dimension))