Mercurial > pylearn

annotate _test_dataset.py @ 192:f62a03c9d485

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Redesign of StatsCollector
author Yoshua Bengio <bengioy@iro.umontreal.ca>
date Thu, 15 May 2008 12:46:21 -0400
parents 46c5c90019c2
children 5b3afda2f1ad
rev   line source
4
f7dcfb5f9d5b Added test for dataset.
bengioy@bengiomac.local
parents:
diff changeset
1 from dataset import *
f7dcfb5f9d5b Added test for dataset.
bengioy@bengiomac.local
parents:
diff changeset
2 from math import *
f7dcfb5f9d5b Added test for dataset.
bengioy@bengiomac.local
parents:
diff changeset
3 import unittest
f7dcfb5f9d5b Added test for dataset.
bengioy@bengiomac.local
parents:
diff changeset
4
f7dcfb5f9d5b Added test for dataset.
bengioy@bengiomac.local
parents:
diff changeset
5 def _sum_all(a):
f7dcfb5f9d5b Added test for dataset.
bengioy@bengiomac.local
parents:
diff changeset
6 s=a
f7dcfb5f9d5b Added test for dataset.
bengioy@bengiomac.local
parents:
diff changeset
7 while isinstance(s,numpy.ndarray):
f7dcfb5f9d5b Added test for dataset.
bengioy@bengiomac.local
parents:
diff changeset
8 s=sum(s)
f7dcfb5f9d5b Added test for dataset.
bengioy@bengiomac.local
parents:
diff changeset
9 return s
f7dcfb5f9d5b Added test for dataset.
bengioy@bengiomac.local
parents:
diff changeset
10
f7dcfb5f9d5b Added test for dataset.
bengioy@bengiomac.local
parents:
diff changeset
11 class T_arraydataset(unittest.TestCase):
f7dcfb5f9d5b Added test for dataset.
bengioy@bengiomac.local
parents:
diff changeset
12 def setUp(self):
f7dcfb5f9d5b Added test for dataset.
bengioy@bengiomac.local
parents:
diff changeset
13 numpy.random.seed(123456)
f7dcfb5f9d5b Added test for dataset.
bengioy@bengiomac.local
parents:
diff changeset
14
17
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
15
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
16 def test_ctor_len(self):
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
17 n = numpy.random.rand(8,3)
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
18 a=ArrayDataSet(n)
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
19 self.failUnless(a.data is n)
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
20 self.failUnless(a.fields is None)
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
21
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
22 self.failUnless(len(a) == n.shape[0])
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
23 self.failUnless(a[0].shape == (n.shape[1],))
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
24
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
25 def test_iter(self):
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
26 arr = numpy.random.rand(8,3)
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
27 a=ArrayDataSet(data=arr,fields={"x":slice(2),"y":slice(1,3)})
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
28 for i, example in enumerate(a):
21
fdf0abc490f7 Adapted _test_dataset.py to changes in LookupList
bengioy@bengiomac.local
parents: 19
diff changeset
29 self.failUnless(numpy.all( example['x'] == arr[i,:2]))
fdf0abc490f7 Adapted _test_dataset.py to changes in LookupList
bengioy@bengiomac.local
parents: 19
diff changeset
30 self.failUnless(numpy.all( example['y'] == arr[i,1:3]))
17
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
31
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
32 def test_zip(self):
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
33 arr = numpy.random.rand(8,3)
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
34 a=ArrayDataSet(data=arr,fields={"x":slice(2),"y":slice(1,3)})
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
35 for i, x in enumerate(a.zip("x")):
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
36 self.failUnless(numpy.all( x == arr[i,:2]))
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
37
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
38 def test_minibatch_basic(self):
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
39 arr = numpy.random.rand(10,4)
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
40 a=ArrayDataSet(data=arr,fields={"x":slice(2),"y":slice(1,4)})
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
41 for i, mb in enumerate(a.minibatches(minibatch_size=2)): #all fields
21
fdf0abc490f7 Adapted _test_dataset.py to changes in LookupList
bengioy@bengiomac.local
parents: 19
diff changeset
42 self.failUnless(numpy.all( mb['x'] == arr[i*2:i*2+2,0:2]))
fdf0abc490f7 Adapted _test_dataset.py to changes in LookupList
bengioy@bengiomac.local
parents: 19
diff changeset
43 self.failUnless(numpy.all( mb['y'] == arr[i*2:i*2+2,1:4]))
7
6f8f338686db Moved iterating counter into a FiniteDataSetIterator to allow embedded iterations and multiple threads iterating at the same time on a dataset.
bengioy@bengiomac.local
parents: 6
diff changeset
44
17
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
45 def test_getattr(self):
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
46 arr = numpy.random.rand(10,4)
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
47 a=ArrayDataSet(data=arr,fields={"x":slice(2),"y":slice(1,4)})
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
48 a_y = a.y
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
49 self.failUnless(numpy.all( a_y == arr[:,1:4]))
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
50
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
51 def test_minibatch_wraparound_even(self):
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
52 arr = numpy.random.rand(10,4)
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
53 arr2 = ArrayDataSet.Iterator.matcat(arr,arr)
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
54
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
55 a=ArrayDataSet(data=arr,fields={"x":slice(2),"y":slice(1,4)})
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
56
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
57 #print arr
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
58 for i, x in enumerate(a.minibatches(["x"], minibatch_size=2, n_batches=8)):
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
59 #print 'x' , x
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
60 self.failUnless(numpy.all( x == arr2[i*2:i*2+2,0:2]))
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
61
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
62 def test_minibatch_wraparound_odd(self):
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
63 arr = numpy.random.rand(10,4)
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
64 arr2 = ArrayDataSet.Iterator.matcat(arr,arr)
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
65
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
66 a=ArrayDataSet(data=arr,fields={"x":slice(2),"y":slice(1,4)})
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
67
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
68 for i, x in enumerate(a.minibatches(["x"], minibatch_size=3, n_batches=6)):
759d17112b23 more comments, looping ArrayDataSet iterator, bugfixes to lookup_list, more tests
bergstrj@iro.umontreal.ca
parents: 11
diff changeset
69 self.failUnless(numpy.all( x == arr2[i*3:i*3+3,0:2]))
19
57f4015e2e09 Iterators extend LookupList
bergstrj@iro.umontreal.ca
parents: 17
diff changeset
70
26
672fe4b23032 Fixed dataset errors so that _test_dataset.py works again.
bengioy@grenat.iro.umontreal.ca
parents: 22
diff changeset
71
672fe4b23032 Fixed dataset errors so that _test_dataset.py works again.
bengioy@grenat.iro.umontreal.ca
parents: 22
diff changeset
72 class T_renamingdataset(unittest.TestCase):
672fe4b23032 Fixed dataset errors so that _test_dataset.py works again.
bengioy@grenat.iro.umontreal.ca
parents: 22
diff changeset
73 def setUp(self):
672fe4b23032 Fixed dataset errors so that _test_dataset.py works again.
bengioy@grenat.iro.umontreal.ca
parents: 22
diff changeset
74 numpy.random.seed(123456)
672fe4b23032 Fixed dataset errors so that _test_dataset.py works again.
bengioy@grenat.iro.umontreal.ca
parents: 22
diff changeset
75
672fe4b23032 Fixed dataset errors so that _test_dataset.py works again.
bengioy@grenat.iro.umontreal.ca
parents: 22
diff changeset
76
672fe4b23032 Fixed dataset errors so that _test_dataset.py works again.
bengioy@grenat.iro.umontreal.ca
parents: 22
diff changeset
77 def test_hasfield(self):
672fe4b23032 Fixed dataset errors so that _test_dataset.py works again.
bengioy@grenat.iro.umontreal.ca
parents: 22
diff changeset
78 n = numpy.random.rand(3,8)
672fe4b23032 Fixed dataset errors so that _test_dataset.py works again.
bengioy@grenat.iro.umontreal.ca
parents: 22
diff changeset
79 a=ArrayDataSet(data=n,fields={"x":slice(2),"y":slice(1,4),"z":slice(4,6)})
672fe4b23032 Fixed dataset errors so that _test_dataset.py works again.
bengioy@grenat.iro.umontreal.ca
parents: 22
diff changeset
80 b=a.rename({'xx':'x','zz':'z'})
672fe4b23032 Fixed dataset errors so that _test_dataset.py works again.
bengioy@grenat.iro.umontreal.ca
parents: 22
diff changeset
81 self.failUnless(b.hasFields('xx','zz') and not b.hasFields('x') and not b.hasFields('y'))
672fe4b23032 Fixed dataset errors so that _test_dataset.py works again.
bengioy@grenat.iro.umontreal.ca
parents: 22
diff changeset
82
29
46c5c90019c2 Changed apply_function so that it propagates methods of the source.
bengioy@grenat.iro.umontreal.ca
parents: 28
diff changeset
83 class T_applyfunctiondataset(unittest.TestCase):
46c5c90019c2 Changed apply_function so that it propagates methods of the source.
bengioy@grenat.iro.umontreal.ca
parents: 28
diff changeset
84 def setUp(self):
46c5c90019c2 Changed apply_function so that it propagates methods of the source.
bengioy@grenat.iro.umontreal.ca
parents: 28
diff changeset
85 numpy.random.seed(123456)
46c5c90019c2 Changed apply_function so that it propagates methods of the source.
bengioy@grenat.iro.umontreal.ca
parents: 28
diff changeset
86
46c5c90019c2 Changed apply_function so that it propagates methods of the source.
bengioy@grenat.iro.umontreal.ca
parents: 28
diff changeset
87 def test_function(self):
46c5c90019c2 Changed apply_function so that it propagates methods of the source.
bengioy@grenat.iro.umontreal.ca
parents: 28
diff changeset
88 n = numpy.random.rand(3,8)
46c5c90019c2 Changed apply_function so that it propagates methods of the source.
bengioy@grenat.iro.umontreal.ca
parents: 28
diff changeset
89 a=ArrayDataSet(data=n,fields={"x":slice(2),"y":slice(1,4),"z":slice(4,6)})
46c5c90019c2 Changed apply_function so that it propagates methods of the source.
bengioy@grenat.iro.umontreal.ca
parents: 28
diff changeset
90 b=a.apply_function(lambda x,y: x+y,x+1, ['x','y'], ['x+y','x+1'], False,False,False)
46c5c90019c2 Changed apply_function so that it propagates methods of the source.
bengioy@grenat.iro.umontreal.ca
parents: 28
diff changeset
91 print b.fieldNames()
46c5c90019c2 Changed apply_function so that it propagates methods of the source.
bengioy@grenat.iro.umontreal.ca
parents: 28
diff changeset
92 print b('x+y')
46c5c90019c2 Changed apply_function so that it propagates methods of the source.
bengioy@grenat.iro.umontreal.ca
parents: 28
diff changeset
93
26
672fe4b23032 Fixed dataset errors so that _test_dataset.py works again.
bengioy@grenat.iro.umontreal.ca
parents: 22
diff changeset
94
4
f7dcfb5f9d5b Added test for dataset.
bengioy@bengiomac.local
parents:
diff changeset
95 if __name__ == '__main__':
f7dcfb5f9d5b Added test for dataset.
bengioy@bengiomac.local
parents:
diff changeset
96 unittest.main()
f7dcfb5f9d5b Added test for dataset.
bengioy@bengiomac.local
parents:
diff changeset
97