Mercurial > pylearn

annotate _test_filetensor.py @ 399:8796b91a9f09

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
RBM minibatch works
author Joseph Turian <turian@gmail.com>
date Tue, 08 Jul 2008 21:42:21 -0400
parents b55c829695f1
children 040cb796f4e0
rev   line source
248
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
1 from filetensor import *
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
2 import filetensor
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
3
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
4 import unittest
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
5 import os
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
6
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
7 class T(unittest.TestCase):
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
8 fname = '/tmp/some_mat'
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
9
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
10 def setUp(self):
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
11 #TODO: test that /tmp/some_mat does not exist
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
12 try:
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
13 os.stat(self.fname)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
14 except OSError:
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
15 return #assume file was not found
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
16 raise Exception('autotest file "%s" exists!' % self.fname)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
17
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
18 def tearDown(self):
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
19 os.remove(self.fname)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
20
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
21 def test_file(self):
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
22 gen = numpy.random.rand(1)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
23 f = file(self.fname, 'w');
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
24 write(f, gen)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
25 f.flush()
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
26 f = file(self.fname, 'r');
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
27 mat = read(f, None, debug=False) #load from filename
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
28 self.failUnless(gen.shape == mat.shape)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
29 self.failUnless(numpy.all(gen == mat))
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
30
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
31 def test_filename(self):
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
32 gen = numpy.random.rand(1)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
33 write(self.fname, gen)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
34 mat = read(self.fname, None, debug=False) #load from filename
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
35 self.failUnless(gen.shape == mat.shape)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
36 self.failUnless(numpy.all(gen == mat))
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
37
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
38 def testNd(self):
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
39 """shape and values are stored correctly for tensors of rank 0 to 5"""
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
40 whole_shape = [5, 6, 7, 8, 9]
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
41 for i in xrange(5):
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
42 gen = numpy.asarray(numpy.random.rand(*whole_shape[:i]))
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
43 f = file(self.fname, 'w');
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
44 write(f, gen)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
45 f.flush()
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
46 f = file(self.fname, 'r');
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
47 mat = read(f, None, debug=False) #load from filename
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
48 self.failUnless(gen.shape == mat.shape)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
49 self.failUnless(numpy.all(gen == mat))
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
50
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
51 def test_dtypes(self):
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
52 """shape and values are stored correctly for all dtypes """
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
53 for dtype in filetensor._dtype_magic:
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
54 gen = numpy.asarray(
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
55 numpy.random.rand(4, 5, 2, 1) * 100,
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
56 dtype=dtype)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
57 f = file(self.fname, 'w');
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
58 write(f, gen)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
59 f.flush()
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
60 f = file(self.fname, 'r');
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
61 mat = read(f, None, debug=False) #load from filename
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
62 self.failUnless(gen.dtype == mat.dtype)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
63 self.failUnless(gen.shape == mat.shape)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
64 self.failUnless(numpy.all(gen == mat))
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
65
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
66 def test_dtype_invalid(self):
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
67 gen = numpy.zeros((3,4), dtype='uint16') #an unsupported dtype
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
68 f = file(self.fname, 'w')
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
69 passed = False
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
70 try:
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
71 write(f, gen)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
72 except TypeError, e:
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
73 if e[0].startswith('Invalid ndarray dtype'):
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
74 passed = True
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
75 f.close()
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
76 self.failUnless(passed)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
77
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
78
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
79 if __name__ == '__main__':
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
80 unittest.main()
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
81
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
82 #a small test script, starts by reading sys.argv[1]
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
83 #print 'rval', rval.shape, rval.size
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
84
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
85 if 0:
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
86 write(f, rval)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
87 print ''
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
88 f.close()
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
89 f = file('/tmp/some_mat', 'r');
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
90 rval2 = read(f) #load from file handle
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
91 print 'rval2', rval2.shape, rval2.size
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
92
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
93 assert rval.dtype == rval2.dtype
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
94 assert rval.shape == rval2.shape
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
95 assert numpy.all(rval == rval2)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
96 print 'ok'
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
97
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
98 def _unused():
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
99 f.seek(0,2) #seek to end
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
100 f_len = f.tell()
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
101 f.seek(f_data_start,0) #seek back to where we were
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
102
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
103 if debug: print 'length:', f_len
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
104
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
105
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
106 f_data_bytes = (f_len - f_data_start)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
107
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
108 if debug: print 'data bytes according to header: ', dim_size * elsize
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
109 if debug: print 'data bytes according to file : ', f_data_bytes
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
110
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
111 if debug: print 'reading data...'
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
112 sys.stdout.flush()
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
113
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
114 def read_ndarray(f, dim, dtype):
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
115 return numpy.fromfile(f, dtype=dtype, count=_prod(dim)).reshape(dim)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
116