Mercurial > pylearn

annotate _test_filetensor.py @ 464:121cc6db4481

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
More debug output
author Joseph Turian <turian@iro.umontreal.ca>
date Wed, 15 Oct 2008 17:00:26 -0400
parents 040cb796f4e0
children
rev   line source
248
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
1 from filetensor import *
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
2 import filetensor
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
3
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
4 import unittest
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
5 import os
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
6
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
7 class T(unittest.TestCase):
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
8 fname = '/tmp/some_mat'
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
9
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
10 def setUp(self):
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
11 #TODO: test that /tmp/some_mat does not exist
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
12 try:
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
13 os.stat(self.fname)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
14 except OSError:
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
15 return #assume file was not found
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
16 raise Exception('autotest file "%s" exists!' % self.fname)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
17
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
18 def tearDown(self):
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
19 os.remove(self.fname)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
20
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
21 def test_file(self):
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
22 gen = numpy.random.rand(1)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
23 f = file(self.fname, 'w');
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
24 write(f, gen)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
25 f.flush()
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
26 f = file(self.fname, 'r');
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
27 mat = read(f, None, debug=False) #load from filename
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
28 self.failUnless(gen.shape == mat.shape)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
29 self.failUnless(numpy.all(gen == mat))
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
30
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
31 def test_filename(self):
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
32 gen = numpy.random.rand(1)
420
040cb796f4e0 Removed feature of passing the file as a pathname from filetensor.{read, write}
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 282
diff changeset
33 f = file(self.fname, 'w')
040cb796f4e0 Removed feature of passing the file as a pathname from filetensor.{read, write}
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 282
diff changeset
34 write(f, gen)
040cb796f4e0 Removed feature of passing the file as a pathname from filetensor.{read, write}
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 282
diff changeset
35 f.close()
040cb796f4e0 Removed feature of passing the file as a pathname from filetensor.{read, write}
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 282
diff changeset
36 f = file(self.fname, 'r')
040cb796f4e0 Removed feature of passing the file as a pathname from filetensor.{read, write}
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 282
diff changeset
37 mat = read(f, None, debug=False) #load from filename
040cb796f4e0 Removed feature of passing the file as a pathname from filetensor.{read, write}
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 282
diff changeset
38 f.close()
248
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
39 self.failUnless(gen.shape == mat.shape)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
40 self.failUnless(numpy.all(gen == mat))
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
41
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
42 def testNd(self):
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
43 """shape and values are stored correctly for tensors of rank 0 to 5"""
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
44 whole_shape = [5, 6, 7, 8, 9]
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
45 for i in xrange(5):
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
46 gen = numpy.asarray(numpy.random.rand(*whole_shape[:i]))
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
47 f = file(self.fname, 'w');
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
48 write(f, gen)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
49 f.flush()
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
50 f = file(self.fname, 'r');
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
51 mat = read(f, None, debug=False) #load from filename
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
52 self.failUnless(gen.shape == mat.shape)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
53 self.failUnless(numpy.all(gen == mat))
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
54
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
55 def test_dtypes(self):
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
56 """shape and values are stored correctly for all dtypes """
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
57 for dtype in filetensor._dtype_magic:
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
58 gen = numpy.asarray(
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
59 numpy.random.rand(4, 5, 2, 1) * 100,
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
60 dtype=dtype)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
61 f = file(self.fname, 'w');
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
62 write(f, gen)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
63 f.flush()
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
64 f = file(self.fname, 'r');
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
65 mat = read(f, None, debug=False) #load from filename
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
66 self.failUnless(gen.dtype == mat.dtype)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
67 self.failUnless(gen.shape == mat.shape)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
68 self.failUnless(numpy.all(gen == mat))
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
69
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
70 def test_dtype_invalid(self):
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
71 gen = numpy.zeros((3,4), dtype='uint16') #an unsupported dtype
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
72 f = file(self.fname, 'w')
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
73 passed = False
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
74 try:
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
75 write(f, gen)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
76 except TypeError, e:
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
77 if e[0].startswith('Invalid ndarray dtype'):
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
78 passed = True
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
79 f.close()
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
80 self.failUnless(passed)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
81
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
82
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
83 if __name__ == '__main__':
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
84 unittest.main()
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
85
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
86 #a small test script, starts by reading sys.argv[1]
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
87 #print 'rval', rval.shape, rval.size
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
88
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
89 if 0:
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
90 write(f, rval)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
91 print ''
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
92 f.close()
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
93 f = file('/tmp/some_mat', 'r');
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
94 rval2 = read(f) #load from file handle
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
95 print 'rval2', rval2.shape, rval2.size
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
96
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
97 assert rval.dtype == rval2.dtype
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
98 assert rval.shape == rval2.shape
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
99 assert numpy.all(rval == rval2)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
100 print 'ok'
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
101
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
102 def _unused():
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
103 f.seek(0,2) #seek to end
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
104 f_len = f.tell()
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
105 f.seek(f_data_start,0) #seek back to where we were
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
106
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
107 if debug: print 'length:', f_len
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
108
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
109
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
110 f_data_bytes = (f_len - f_data_start)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
111
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
112 if debug: print 'data bytes according to header: ', dim_size * elsize
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
113 if debug: print 'data bytes according to file : ', f_data_bytes
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
114
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
115 if debug: print 'reading data...'
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
116 sys.stdout.flush()
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
117
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
118 def read_ndarray(f, dim, dtype):
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
119 return numpy.fromfile(f, dtype=dtype, count=_prod(dim)).reshape(dim)
82ba488b2c24 polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
120