# HG changeset patch
# User Joseph Turian <turian@gmail.com>
# Date 1227181086 18000
# Node ID 27b1344a57b17ede4e62f5b47ac3346e5ea1f231
# Parent  c6563c62998477690d87442b8c78a0dd9b56f38b
Added preprocessing back in

diff -r c6563c629984 -r 27b1344a57b1 embeddings/process.py
--- a/embeddings/process.py	Thu Nov 20 06:11:52 2008 -0500
+++ b/embeddings/process.py	Thu Nov 20 06:38:06 2008 -0500
@@ -50,17 +50,31 @@
 
 import re
 numberre = re.compile("[0-9]")
-
-def preprocess_word(w):
+slashre = re.compile("\\\/")
+ 
+def preprocess_word(origw):
     """
     Convert a word so that it can be embedded directly.
     Returned the preprocessed sequence.
-    @note: Perhaps run L{common.penntreebank.preprocess} on the word first.
+    @note: Preprocessing is appropriate for Penn Treebank style documents.
+    #@note: Perhaps run L{common.penntreebank.preprocess} on the word first.
     """
     read_embeddings()
-    if w not in __word_to_embedding:
-        w = string.lower(w)
-        w = numberre.sub("NUMBER", w)
+    if origw == "-LRB-": w = "("
+    elif origw == "-RRB-": w = ")"
+    elif origw == "-LCB-": w = "{"
+    elif origw == "-RCB-": w = "}"
+    elif origw == "-LSB-": w = "["
+    elif origw == "-RSB-": w = "]"
+    else:
+        w = origw
+        if w not in __word_to_embedding:
+            w = string.lower(w)
+            w = slashre.sub("/", w)
+            w = numberre.sub("NUMBER", w)
+#    if w not in __word_to_embedding:
+#        w = string.lower(w)
+#        w = numberre.sub("NUMBER", w)
     if w not in __word_to_embedding:
 #        sys.stderr.write("Word not in vocabulary, using %s: %s (original %s)\n" % (UNKNOWN, w, origw))
         w = UNKNOWN