# HG changeset patch
# User Joseph Turian <turian@gmail.com>
# Date 1227179512 18000
# Node ID c6563c62998477690d87442b8c78a0dd9b56f38b
# Parent  a6068dedfbd6dfe7d89de37dc52d160d7cdc61ed
Moved word preprocessing out

diff -r a6068dedfbd6 -r c6563c629984 embeddings/process.py
--- a/embeddings/process.py	Wed Nov 19 18:14:43 2008 -0500
+++ b/embeddings/process.py	Thu Nov 20 06:11:52 2008 -0500
@@ -50,27 +50,17 @@
 
 import re
 numberre = re.compile("[0-9]")
-slashre = re.compile("\\\/")
 
-def preprocess_word(origw):
+def preprocess_word(w):
     """
     Convert a word so that it can be embedded directly.
     Returned the preprocessed sequence.
-    @note: Preprocessing is appropriate for Penn Treebank style documents.
+    @note: Perhaps run L{common.penntreebank.preprocess} on the word first.
     """
     read_embeddings()
-    if origw == "-LRB-": w = "("
-    elif origw == "-RRB-": w = ")"
-    elif origw == "-LCB-": w = "{"
-    elif origw == "-RCB-": w = "}"
-    elif origw == "-LSB-": w = "["
-    elif origw == "-RSB-": w = "]"
-    else:
-        w = origw
-        if w not in __word_to_embedding:
-            w = string.lower(w)
-            w = slashre.sub("/", w)
-            w = numberre.sub("NUMBER", w)
+    if w not in __word_to_embedding:
+        w = string.lower(w)
+        w = numberre.sub("NUMBER", w)
     if w not in __word_to_embedding:
 #        sys.stderr.write("Word not in vocabulary, using %s: %s (original %s)\n" % (UNKNOWN, w, origw))
         w = UNKNOWN