Mercurial > pylearn

annotate test_dataset.py @ 64:863da25a60f1

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
trying to fix infinite loop
author Yoshua Bengio <bengioy@iro.umontreal.ca>
date Fri, 02 May 2008 11:01:28 -0400
parents 14589f02a372
children d48eba49a2f4
rev   line source
51
59757365a057 the script can be autorun
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 45
diff changeset
1 #!/bin/env python
45
a5c70dc42972 Test functions for dataset.py
bengioy@grenat.iro.umontreal.ca
parents:
diff changeset
2 from dataset import *
a5c70dc42972 Test functions for dataset.py
bengioy@grenat.iro.umontreal.ca
parents:
diff changeset
3 from math import *
a5c70dc42972 Test functions for dataset.py
bengioy@grenat.iro.umontreal.ca
parents:
diff changeset
4 import numpy
a5c70dc42972 Test functions for dataset.py
bengioy@grenat.iro.umontreal.ca
parents:
diff changeset
5
a5c70dc42972 Test functions for dataset.py
bengioy@grenat.iro.umontreal.ca
parents:
diff changeset
6 def test1():
a5c70dc42972 Test functions for dataset.py
bengioy@grenat.iro.umontreal.ca
parents:
diff changeset
7 global a,ds
a5c70dc42972 Test functions for dataset.py
bengioy@grenat.iro.umontreal.ca
parents:
diff changeset
8 a = numpy.random.rand(10,4)
a5c70dc42972 Test functions for dataset.py
bengioy@grenat.iro.umontreal.ca
parents:
diff changeset
9 print a
a5c70dc42972 Test functions for dataset.py
bengioy@grenat.iro.umontreal.ca
parents:
diff changeset
10 ds = ArrayDataSet(a,{'x':slice(3),'y':3,'z':[0,2]})
a5c70dc42972 Test functions for dataset.py
bengioy@grenat.iro.umontreal.ca
parents:
diff changeset
11 print "len(ds)=",len(ds)
54
70147d00615a added assert
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 52
diff changeset
12 assert(len(ds)==10)
45
a5c70dc42972 Test functions for dataset.py
bengioy@grenat.iro.umontreal.ca
parents:
diff changeset
13 print "example 0 = ",ds[0]
54
70147d00615a added assert
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 52
diff changeset
14 # assert
45
a5c70dc42972 Test functions for dataset.py
bengioy@grenat.iro.umontreal.ca
parents:
diff changeset
15 print "x=",ds["x"]
a5c70dc42972 Test functions for dataset.py
bengioy@grenat.iro.umontreal.ca
parents:
diff changeset
16 print "x|y"
a5c70dc42972 Test functions for dataset.py
bengioy@grenat.iro.umontreal.ca
parents:
diff changeset
17 for x,y in ds("x","y"):
a5c70dc42972 Test functions for dataset.py
bengioy@grenat.iro.umontreal.ca
parents:
diff changeset
18 print x,y
a5c70dc42972 Test functions for dataset.py
bengioy@grenat.iro.umontreal.ca
parents:
diff changeset
19 minibatch_iterator = ds.minibatches(fieldnames=['z','y'],n_batches=1,minibatch_size=3,offset=4)
a5c70dc42972 Test functions for dataset.py
bengioy@grenat.iro.umontreal.ca
parents:
diff changeset
20 minibatch = minibatch_iterator.__iter__().next()
a5c70dc42972 Test functions for dataset.py
bengioy@grenat.iro.umontreal.ca
parents:
diff changeset
21 print "minibatch=",minibatch
a5c70dc42972 Test functions for dataset.py
bengioy@grenat.iro.umontreal.ca
parents:
diff changeset
22 for var in minibatch:
a5c70dc42972 Test functions for dataset.py
bengioy@grenat.iro.umontreal.ca
parents:
diff changeset
23 print "var=",var
48
b6730f9a336d Fixing MinibatchDataSet getitem
bengioy@grenat.iro.umontreal.ca
parents: 45
diff changeset
24 print "take a slice and look at field y",ds[1:6:2]["y"]
45
a5c70dc42972 Test functions for dataset.py
bengioy@grenat.iro.umontreal.ca
parents:
diff changeset
25
64
863da25a60f1 trying to fix infinite loop
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 63
diff changeset
26 def test2():
863da25a60f1 trying to fix infinite loop
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 63
diff changeset
27 a = numpy.random.rand(10,4)
863da25a60f1 trying to fix infinite loop
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 63
diff changeset
28 print a
863da25a60f1 trying to fix infinite loop
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 63
diff changeset
29 ds = ArrayDataSet(a,{'x':slice(3),'y':3,'z':[0,2]})
863da25a60f1 trying to fix infinite loop
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 63
diff changeset
30 for x,z in ds[:3]('x','z'):
863da25a60f1 trying to fix infinite loop
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 63
diff changeset
31 assert ds[i]['z'].all()==a[i][0:3:2].all()
863da25a60f1 trying to fix infinite loop
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 63
diff changeset
32
58
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
33 def test_ArrayDataSet():
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
34 #don't test stream
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
35 #tested only with float value
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
36 a = numpy.random.rand(10,4)
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
37 print a
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
38 ds = ArrayDataSet(a,{'x':slice(3),'y':3,'z':[0,2]})
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
39 assert len(ds)==10
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
40 #assert ds==a? should this work?
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
41 for i in range(len(ds)):
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
42 assert ds[i]['x'].all()==a[i][:2].all()
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
43 assert ds[i]['y']==a[i][3]
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
44 assert ds[i]['z'].all()==a[i][0:3:2].all()
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
45 print "x=",ds["x"]
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
46 print "x|y"
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
47 i=0
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
48 for x in ds('x','y'):
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
49 assert numpy.append(x['x'],x['y']).all()==a[i].all()
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
50 i+=1
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
51 # i=0
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
52 # for x in ds['x','y']: # don't work
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
53 # assert numpy.append(x['x'],x['y']).all()==a[i].all()
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
54 # i+=1
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
55 # for (x,y) in (ds('x','y'),a): #don't work # haven't found a variant that work.
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
56 # assert numpy.append(x,y)==z
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
57 i=0
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
58 for x,y in ds('x','y'):
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
59 assert numpy.append(x,y).all()==a[i].all()
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
60 i+=1
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
61 for minibatch in ds.minibatches(['x','z'], minibatch_size=3):
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
62 assert minibatch[0][:,0:3:2].all()==minibatch[1].all()
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
63 for x,z in ds.minibatches(['x','z'], minibatch_size=3):
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
64 assert x[:,0:3:2].all()==z.all()
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
65
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
66 # for minibatch in ds.minibatches(['z','y'], minibatch_size=3):
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
67 # print minibatch
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
68 # minibatch_iterator = ds.minibatches(fieldnames=['z','y'],n_batches=1,minibatch_size=3,offset=4)
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
69 # minibatch = minibatch_iterator.__iter__().next()
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
70 # print "minibatch=",minibatch
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
71 # for var in minibatch:
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
72 # print "var=",var
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
73 # print "take a slice and look at field y",ds[1:6:2]["y"]
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
74 have_thrown = False
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
75 try:
63
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
76 ds['h'] # h is not defined...
58
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
77 except :
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
78 have_thrown = True
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
79 assert have_thrown == True
63
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
80 assert len(ds.fields())==3
58
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
81 for field in ds.fields():
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
82 for field_value in field: # iterate over the values associated to that field for all the ds examples
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
83 pass
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
84 for field in ds('x','z').fields():
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
85 pass
60
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 58
diff changeset
86 for field in ds.fields('x','y'):
58
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
87 pass
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
88 for field_examples in ds.fields():
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
89 for example_value in field_examples:
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
90 pass
63
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
91
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
92 assert ds == ds.fields().examples()
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
93
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
94
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
95 #test missing value
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
96
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
97 assert len(ds[:3])==3
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
98 for x,z in ds[:3]('x','z'):
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
99 assert ds[i]['z'].all()==a[i][0:3:2].all()
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
100
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
101 #ds[i1:i2:s]# returns a ds with the examples i1,i1+s,...i2-s.
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
102
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
103 #ds[i]# returns an Example.
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
104
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
105 #ds[[i1,i2,...in]]# returns a ds with examples i1,i2,...in.
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
106
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
107 #ds[fieldname]# an iterable over the values of the field fieldname across
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
108 #the ds (the iterable is obtained by default by calling valuesVStack
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
109 #over the values for individual examples).
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
110
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
111 #ds.<property># returns the value of a property associated with
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
112 #the name <property>. The following properties should be supported:
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
113 # - 'description': a textual description or name for the ds
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
114 # - 'fieldtypes': a list of types (one per field)
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
115 #* ds1 | ds2 | ds3 == ds.hstack([ds1,ds2,ds3])
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
116 #* ds1 & ds2 & ds3 == ds.vstack([ds1,ds2,ds3])
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
117
14589f02a372 more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 60
diff changeset
118
64
863da25a60f1 trying to fix infinite loop
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 63
diff changeset
119 test2()
58
17729d7104fa added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents: 54
diff changeset
120
64
863da25a60f1 trying to fix infinite loop
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 63
diff changeset
121 #test_ArrayDataSet()
863da25a60f1 trying to fix infinite loop
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 63
diff changeset
122