# HG changeset patch # User Joseph Turian # Date 1224620684 14400 # Node ID a07948f780b9850880085ccbbedb4d73a7746746 # Parent f3711bcc467e6db7a807460d3d7409eef6ddc09a Moved embeddings out of sandbox diff -r f3711bcc467e -r a07948f780b9 embeddings/README.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/embeddings/README.txt Tue Oct 21 16:24:44 2008 -0400 @@ -0,0 +1,3 @@ +Messy scripts for working with Jason + Ronan's embeddings. + +Parameters are given in parameters.py diff -r f3711bcc467e -r a07948f780b9 embeddings/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/embeddings/__init__.py Tue Oct 21 16:24:44 2008 -0400 @@ -0,0 +1,1 @@ +from process import * diff -r f3711bcc467e -r a07948f780b9 embeddings/convert.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/embeddings/convert.py Tue Oct 21 16:24:44 2008 -0400 @@ -0,0 +1,15 @@ +#!/usr/bin/python +""" +Convert stdin sentences to word embeddings, and output YAML. +""" + +import sys, string +import read +import yaml + +output = [] +for l in sys.stdin: + l = string.strip(l) + output.append((l, read.convert_string(l))) + +print yaml.dump(output) diff -r f3711bcc467e -r a07948f780b9 embeddings/one-per-line.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/embeddings/one-per-line.py Tue Oct 21 16:24:44 2008 -0400 @@ -0,0 +1,27 @@ +#!/usr/bin/python + +import string +#import psyco + +weightsfile = "lm-weights.txt" +vocabfile = "words.asc" +size = 30000 +dimensions = 50 + +import numpy, math +import sys +from common.str import percent + +word_to_vector = {} + +f = open(weightsfile) +f.readline() +vals = [float(v) for v in string.split(f.readline())] +assert len(vals) == size * dimensions +vals.reverse() +#for i in range(size): +r = range(size) +r.reverse() +for i in r: + l = vals[dimensions*i:dimensions*(i+1)] + print string.join([`s` for s in l], "\t") diff -r f3711bcc467e -r a07948f780b9 embeddings/parameters.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/embeddings/parameters.py Tue Oct 21 16:24:44 2008 -0400 @@ -0,0 +1,8 @@ +""" +Locations of the embedding data files. +""" +WEIGHTSFILE = "/home/fringant2/lisa/data/word_embeddings.collobert-and-weston/lm-weights.txt" +VOCABFILE = "/home/fringant2/lisa/data/word_embeddings.collobert-and-weston/words.asc" +NUMBER_OF_WORDS = 30000 +DIMENSIONS = 50 +UNKNOWN = "UNKNOWN" diff -r f3711bcc467e -r a07948f780b9 embeddings/process.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/embeddings/process.py Tue Oct 21 16:24:44 2008 -0400 @@ -0,0 +1,114 @@ +""" +Read in the weights file +""" + +import string +import sys + +from parameters import * + +__words = None +__word_to_embedding = None +__read = False + +def word_to_embedding(w): + read_embeddings() + return __word_to_embedding[w] + +def read_embeddings(): + global __words + global __word_to_embedding + global __read + if __read: return + + __words = [string.strip(w) for w in open(VOCABFILE).readlines()] + assert len(__words) == NUMBER_OF_WORDS + + import numpy, math + from common.str import percent + + __word_to_embedding = {} + + sys.stderr.write("Reading %s...\n" % WEIGHTSFILE) + f = open(WEIGHTSFILE) + f.readline() + vals = [float(v) for v in string.split(f.readline())] + assert len(vals) == NUMBER_OF_WORDS * DIMENSIONS + for i in range(NUMBER_OF_WORDS): + l = vals[DIMENSIONS*i:DIMENSIONS*(i+1)] + w = __words[i] + __word_to_embedding[w] = l + __read = True + sys.stderr.write("...done reading %s\n" % WEIGHTSFILE) + +import re +numberre = re.compile("[0-9]") +slashre = re.compile("\\\/") + +def preprocess(l): + """ + Convert a sequence so that it can be embedded directly. + Returned the preprocessed sequence. + @note: Preprocessing is appropriate for Penn Treebank style documents. + """ + read_embeddings() + lnew = [] + for origw in l: + if origw == "-LRB-": w = "(" + elif origw == "-RRB-": w = ")" + elif origw == "-LCB-": w = "{" + elif origw == "-RCB-": w = "}" + elif origw == "-LSB-": w = "[" + elif origw == "-RSB-": w = "]" + else: + w = origw + w = string.lower(w) + w = slashre.sub("/", w) + w = numberre.sub("NUMBER", w) + if w not in __word_to_embedding: + sys.stderr.write("Word not in vocabulary, using %s: %s (original %s)\n" % (UNKNOWN, w, origw)) + w = UNKNOWN + assert w in __word_to_embedding + lnew.append(w) + return lnew + +#def convert_string(s, strict=False): +# """ +# Convert a string to a sequence of embeddings. +# @param strict: If strict, then words *must* be in the vocabulary. +# @todo: DEPRECATED Remove this function. +# """ +# read_embeddings() +# e = [] +# for origw in string.split(string.lower(s)): +# w = numberre.sub("NUMBER", origw) +# if w in __word_to_embedding: +# e.append(__word_to_embedding[w]) +# else: +# sys.stderr.write("Word not in vocabulary, using %s: %s (original %s)\n" % (UNKNOWN, w, origw)) +# assert not strict +# e.append(__word_to_embedding[UNKNOWN]) +# return e + +#def test(): +# """ +# Debugging code. +# """ +# read_embeddings() +# for w in __word_to_embedding: +# assert len(__word_to_embedding[w]) == 50 +# import numpy +# for w1 in __words: +# e1 = numpy.asarray(__word_to_embedding[w1]) +# lst = [] +# print w1, numpy.dot(e1, e1) +# for w2 in __word_to_embedding: +# if w1 >= w2: continue +# e2 = numpy.asarray(__word_to_embedding[w2]) +# d = (e1 - e2) +# l2 = numpy.dot(d, d) +# lst.append((l2, w1, w2)) +# lst.sort() +# print lst[:10] +# +#test() diff -r f3711bcc467e -r a07948f780b9 embeddings/read-original.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/embeddings/read-original.py Tue Oct 21 16:24:44 2008 -0400 @@ -0,0 +1,47 @@ +#!/usr/bin/python + +import string +#import psyco + +weightsfile = "lm-weights.txt" +vocabfile = "words.asc" +size = 30000 +dimensions = 50 + +words = [string.strip(w) for w in open(vocabfile).readlines()] +assert len(words) == 30000 + +import numpy, math +import sys +from common.str import percent + +word_to_vector = {} + +f = open(weightsfile) +f.readline() +vals = [float(v) for v in string.split(f.readline())] +assert len(vals) == size * dimensions +vals.reverse() +for i in range(size): + l = vals[dimensions*i:dimensions*(i+1)] + w = words[i] + word_to_vector[w] = l + +# l2 = numpy.asarray(l) +# print math.fabs(50 - numpy.sum(l2*l2)), w + +cnt = 0 +for i1 in range(len(words)): + for i2 in range(len(words)): + w1 = words[i1] + w2 = words[i2] + cnt += 1 + if i1 <= i2: continue + l1 = numpy.asarray(word_to_vector[w1]) + l2 = numpy.asarray(word_to_vector[w2]) + d = l2 - l1 + dist = numpy.sum(d * d) + if dist < 50: + print numpy.sum(d * d), w1, w2, i1, i2 + if cnt % 1000 == 0: + sys.stderr.write("%s done...\n" % percent(cnt, len(word_to_vector) * len(word_to_vector))) diff -r f3711bcc467e -r a07948f780b9 sandbox/embeddings/README.txt --- a/sandbox/embeddings/README.txt Mon Oct 20 19:14:06 2008 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,3 +0,0 @@ -Messy scripts for working with Jason + Ronan's embeddings. - -Parameters are given in parameters.py diff -r f3711bcc467e -r a07948f780b9 sandbox/embeddings/__init__.py --- a/sandbox/embeddings/__init__.py Mon Oct 20 19:14:06 2008 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1 +0,0 @@ -from process import * diff -r f3711bcc467e -r a07948f780b9 sandbox/embeddings/convert.py --- a/sandbox/embeddings/convert.py Mon Oct 20 19:14:06 2008 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,15 +0,0 @@ -#!/usr/bin/python -""" -Convert stdin sentences to word embeddings, and output YAML. -""" - -import sys, string -import read -import yaml - -output = [] -for l in sys.stdin: - l = string.strip(l) - output.append((l, read.convert_string(l))) - -print yaml.dump(output) diff -r f3711bcc467e -r a07948f780b9 sandbox/embeddings/one-per-line.py --- a/sandbox/embeddings/one-per-line.py Mon Oct 20 19:14:06 2008 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,27 +0,0 @@ -#!/usr/bin/python - -import string -#import psyco - -weightsfile = "lm-weights.txt" -vocabfile = "words.asc" -size = 30000 -dimensions = 50 - -import numpy, math -import sys -from common.str import percent - -word_to_vector = {} - -f = open(weightsfile) -f.readline() -vals = [float(v) for v in string.split(f.readline())] -assert len(vals) == size * dimensions -vals.reverse() -#for i in range(size): -r = range(size) -r.reverse() -for i in r: - l = vals[dimensions*i:dimensions*(i+1)] - print string.join([`s` for s in l], "\t") diff -r f3711bcc467e -r a07948f780b9 sandbox/embeddings/parameters.py --- a/sandbox/embeddings/parameters.py Mon Oct 20 19:14:06 2008 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,8 +0,0 @@ -""" -Locations of the embedding data files. -""" -WEIGHTSFILE = "/home/fringant2/lisa/data/word_embeddings.collobert-and-weston/lm-weights.txt" -VOCABFILE = "/home/fringant2/lisa/data/word_embeddings.collobert-and-weston/words.asc" -NUMBER_OF_WORDS = 30000 -DIMENSIONS = 50 -UNKNOWN = "UNKNOWN" diff -r f3711bcc467e -r a07948f780b9 sandbox/embeddings/process.py --- a/sandbox/embeddings/process.py Mon Oct 20 19:14:06 2008 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,114 +0,0 @@ -""" -Read in the weights file -""" - -import string -import sys - -from parameters import * - -__words = None -__word_to_embedding = None -__read = False - -def word_to_embedding(w): - read_embeddings() - return __word_to_embedding[w] - -def read_embeddings(): - global __words - global __word_to_embedding - global __read - if __read: return - - __words = [string.strip(w) for w in open(VOCABFILE).readlines()] - assert len(__words) == NUMBER_OF_WORDS - - import numpy, math - from common.str import percent - - __word_to_embedding = {} - - sys.stderr.write("Reading %s...\n" % WEIGHTSFILE) - f = open(WEIGHTSFILE) - f.readline() - vals = [float(v) for v in string.split(f.readline())] - assert len(vals) == NUMBER_OF_WORDS * DIMENSIONS - for i in range(NUMBER_OF_WORDS): - l = vals[DIMENSIONS*i:DIMENSIONS*(i+1)] - w = __words[i] - __word_to_embedding[w] = l - __read = True - sys.stderr.write("...done reading %s\n" % WEIGHTSFILE) - -import re -numberre = re.compile("[0-9]") -slashre = re.compile("\\\/") - -def preprocess(l): - """ - Convert a sequence so that it can be embedded directly. - Returned the preprocessed sequence. - @note: Preprocessing is appropriate for Penn Treebank style documents. - """ - read_embeddings() - lnew = [] - for origw in l: - if origw == "-LRB-": w = "(" - elif origw == "-RRB-": w = ")" - elif origw == "-LCB-": w = "{" - elif origw == "-RCB-": w = "}" - elif origw == "-LSB-": w = "[" - elif origw == "-RSB-": w = "]" - else: - w = origw - w = string.lower(w) - w = slashre.sub("/", w) - w = numberre.sub("NUMBER", w) - if w not in __word_to_embedding: - sys.stderr.write("Word not in vocabulary, using %s: %s (original %s)\n" % (UNKNOWN, w, origw)) - w = UNKNOWN - assert w in __word_to_embedding - lnew.append(w) - return lnew - -#def convert_string(s, strict=False): -# """ -# Convert a string to a sequence of embeddings. -# @param strict: If strict, then words *must* be in the vocabulary. -# @todo: DEPRECATED Remove this function. -# """ -# read_embeddings() -# e = [] -# for origw in string.split(string.lower(s)): -# w = numberre.sub("NUMBER", origw) -# if w in __word_to_embedding: -# e.append(__word_to_embedding[w]) -# else: -# sys.stderr.write("Word not in vocabulary, using %s: %s (original %s)\n" % (UNKNOWN, w, origw)) -# assert not strict -# e.append(__word_to_embedding[UNKNOWN]) -# return e - -#def test(): -# """ -# Debugging code. -# """ -# read_embeddings() -# for w in __word_to_embedding: -# assert len(__word_to_embedding[w]) == 50 -# import numpy -# for w1 in __words: -# e1 = numpy.asarray(__word_to_embedding[w1]) -# lst = [] -# print w1, numpy.dot(e1, e1) -# for w2 in __word_to_embedding: -# if w1 >= w2: continue -# e2 = numpy.asarray(__word_to_embedding[w2]) -# d = (e1 - e2) -# l2 = numpy.dot(d, d) -# lst.append((l2, w1, w2)) -# lst.sort() -# print lst[:10] -# -#test() diff -r f3711bcc467e -r a07948f780b9 sandbox/embeddings/read-original.py --- a/sandbox/embeddings/read-original.py Mon Oct 20 19:14:06 2008 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,47 +0,0 @@ -#!/usr/bin/python - -import string -#import psyco - -weightsfile = "lm-weights.txt" -vocabfile = "words.asc" -size = 30000 -dimensions = 50 - -words = [string.strip(w) for w in open(vocabfile).readlines()] -assert len(words) == 30000 - -import numpy, math -import sys -from common.str import percent - -word_to_vector = {} - -f = open(weightsfile) -f.readline() -vals = [float(v) for v in string.split(f.readline())] -assert len(vals) == size * dimensions -vals.reverse() -for i in range(size): - l = vals[dimensions*i:dimensions*(i+1)] - w = words[i] - word_to_vector[w] = l - -# l2 = numpy.asarray(l) -# print math.fabs(50 - numpy.sum(l2*l2)), w - -cnt = 0 -for i1 in range(len(words)): - for i2 in range(len(words)): - w1 = words[i1] - w2 = words[i2] - cnt += 1 - if i1 <= i2: continue - l1 = numpy.asarray(word_to_vector[w1]) - l2 = numpy.asarray(word_to_vector[w2]) - d = l2 - l1 - dist = numpy.sum(d * d) - if dist < 50: - print numpy.sum(d * d), w1, w2, i1, i2 - if cnt % 1000 == 0: - sys.stderr.write("%s done...\n" % percent(cnt, len(word_to_vector) * len(word_to_vector)))