# HG changeset patch
# User Joseph Turian <turian@gmail.com>
# Date 1215562461 14400
# Node ID 70019965f888f8a6711da26995bca3f0657df4d6
# Parent  f2d112dc53be560fc31c232d9a395911bd632a77
Basic, broken RBM implementation

diff -r f2d112dc53be -r 70019965f888 sandbox/rbm/__init__.py
diff -r f2d112dc53be -r 70019965f888 sandbox/rbm/globals.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sandbox/rbm/globals.py	Tue Jul 08 20:14:21 2008 -0400
@@ -0,0 +1,12 @@
+"""
+Global variables.
+"""
+
+#INPUT_DIMENSION = 1000
+#INPUT_DIMENSION = 100
+INPUT_DIMENSION = 4
+HIDDEN_DIMENSION = 10
+#HIDDEN_DIMENSION = 4
+LEARNING_RATE = 0.1
+LR = LEARNING_RATE
+SEED = 666
diff -r f2d112dc53be -r 70019965f888 sandbox/rbm/main.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sandbox/rbm/main.py	Tue Jul 08 20:14:21 2008 -0400
@@ -0,0 +1,28 @@
+#!/usr/bin/python
+"""
+    An RBM with binomial units trained with CD-1.
+
+    LIMITATIONS:
+       - Only does pure stochastic gradient (batchsize = 1).
+"""
+
+
+import numpy
+
+nonzero_instances = []
+nonzero_instances.append({0: 1, 1: 1})
+nonzero_instances.append({0: 1, 2: 1})
+
+#nonzero_instances.append({1: 0.1, 5: 0.5, 9: 1})
+#nonzero_instances.append({2: 0.3, 5: 0.5, 8: 0.8})
+##nonzero_instances.append({1: 0.2, 2: 0.3, 5: 0.5})
+
+import model
+model = model.Model()
+
+for i in xrange(100000):
+    # Select an instance
+    instance = nonzero_instances[i % len(nonzero_instances)]
+
+    # SGD update over instance
+    model.update(instance)
diff -r f2d112dc53be -r 70019965f888 sandbox/rbm/model.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sandbox/rbm/model.py	Tue Jul 08 20:14:21 2008 -0400
@@ -0,0 +1,66 @@
+"""
+The model for an autoassociator for sparse inputs, using Ronan Collobert + Jason
+Weston's sampling trick (2008).
+"""
+
+import parameters
+
+import globals
+from globals import LR
+
+import numpy
+from numpy import dot
+import random
+random.seed(globals.SEED)
+
+import pylearn.nnet_ops
+
+def sigmoid(v):
+#    if x < -30.0: return 0.0
+#    if x > 30.0: return 1.0 
+    return 1.0 / (1.0 + numpy.exp(-v))
+
+def sample(v):
+    assert len(v.shape) == 1
+    x = numpy.zeros(v.shape)
+    for i in range(v.shape[0]):
+        assert v[i] >= 0 and v[i] <= 1
+        if random.random() < v[i]: x[i] = 0
+        else: x[i] = 1
+    return x
+
+class Model:
+    def __init__(self):
+        self.parameters = parameters.Parameters(randomly_initialize=True)
+
+    def update(self, instance):
+        """
+        Update the L{Model} using one training instance.
+        @param instance: A dict from feature index to (non-zero) value.
+        @todo: Should assert that nonzero_indices and zero_indices
+        are correct (i.e. are truly nonzero/zero).
+        """
+        v0 = numpy.zeros(globals.INPUT_DIMENSION)
+        for idx in instance.keys():
+            v0[idx] = instance[idx]
+
+        q0 = sigmoid(self.parameters.b + dot(v0, self.parameters.w))
+        h0 = sample(q0)
+        p0 = sigmoid(self.parameters.c + dot(h0, self.parameters.w.T))
+        v1 = sample(p0)
+        q1 = sigmoid(self.parameters.b + dot(v1, self.parameters.w))
+        print
+        print "v[0]:", v0
+        print "Q(h[0][i] = 1 | v[0]):", q0
+        print "h[0]:", h0
+        print "P(v[1][j] = 1 | h[0]):", p0
+        print "v[1]:", v1
+        print "Q(h[1][i] = 1 | v[1]):", q1
+
+        print h0.shape
+        print v0.T.shape
+        print dot(h0, v0.T)
+        print dot(q1, v1.T)
+        self.parameters.w += LR * (dot(h0, v0.T) - dot(q1, v1.T))
+        self.parameters.b += LR * (h0 - q1)
+        self.parameters.c += LR * (v0 - v1)
diff -r f2d112dc53be -r 70019965f888 sandbox/rbm/parameters.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sandbox/rbm/parameters.py	Tue Jul 08 20:14:21 2008 -0400
@@ -0,0 +1,33 @@
+"""
+Parameters (weights) used by the L{Model}.
+"""
+
+import numpy
+import globals
+
+class Parameters:
+    """
+    Parameters used by the L{Model}.
+    """
+    def __init__(self, input_dimension=globals.INPUT_DIMENSION, hidden_dimension=globals.HIDDEN_DIMENSION, randomly_initialize=False, seed=globals.SEED):
+        """
+        Initialize L{Model} parameters.
+        @param randomly_initialize: If True, then randomly initialize
+        according to the given seed. If False, then just use zeroes.
+        """
+        if randomly_initialize:
+            numpy.random.seed(seed)
+            self.w = (numpy.random.rand(input_dimension, hidden_dimension)-0.5)/input_dimension
+            self.b = numpy.zeros(hidden_dimension)
+            self.c = numpy.zeros(input_dimension)
+        else:
+            self.w = numpy.zeros((input_dimension, hidden_dimension))
+            self.b = numpy.zeros(hidden_dimension)
+            self.c = numpy.zeros(input_dimension)
+
+    def __str__(self):
+        s = ""
+        s += "w: %s\n" % self.w
+        s += "b: %s\n" % self.b
+        s += "c: %s\n" % self.c
+        return s