diff sandbox/weights.py @ 466:23221eefb70e

Added pylearn.sandbox.weights.random_weights
author Joseph Turian <turian@iro.umontreal.ca>
date Wed, 15 Oct 2008 18:59:55 -0400
parents
children 3daabc7f94ff
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sandbox/weights.py	Wed Oct 15 18:59:55 2008 -0400
@@ -0,0 +1,25 @@
+"""
+Routine to initialize weights.
+
+@note: We assume that numpy.random.seed() has already been performed.
+"""
+
+from math import sqrt
+import numpy.random
+def random_weights(nin, nout, scale_by=sqrt(3)):
+    """
+    Generate an initial weight matrix with nin inputs (rows) and nout
+    outputs (cols).
+    Each weight is chosen uniformly at random to be in range:
+        [-scale_by/sqrt(nin), +scale_by/sqrt(nin)]
+    @note: Play with scale_by!
+    Ronan derives scale_by=sqrt(3) because that gives variance of
+    1 to something (I forget, ask Yoshua for the derivation). However,
+    Ronan got better results by accidentally using scale_by=1. Yoshua
+    hypothesizes this is because the variance will get telescopically
+    smaller as we go up the layers [need more explanation of this
+    argument].
+    @note: Things may get even trickier if the same weights are being
+    shared in multiple places.
+    """
+    return (numpy.random.rand(nin, nout) * 2.0 - 1) * scale_by / sqrt(nin)