Mercurial > pylearn
annotate test_dataset.py @ 120:5fa46297191b
Automated merge with ssh://p-omega1@lgcm.iro.umontreal.ca/tlearn
| author | Yoshua Bengio <bengioy@iro.umontreal.ca> |
|---|---|
| date | Wed, 07 May 2008 15:46:24 -0400 |
| parents | 9330d941fa1f |
| children | d3c72e412065 |
| rev | line source |
|---|---|
|
51
59757365a057
the script can be autorun
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
45
diff
changeset
|
1 #!/bin/env python |
| 45 | 2 from dataset import * |
| 3 from math import * | |
| 4 import numpy | |
| 5 | |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
6 def have_raised(to_eval): |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
7 have_thrown = False |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
8 try: |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
9 eval(to_eval) |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
10 except : |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
11 have_thrown = True |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
12 return have_thrown |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
13 |
| 45 | 14 def test1(): |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
15 print "test1" |
| 45 | 16 global a,ds |
| 17 a = numpy.random.rand(10,4) | |
| 18 print a | |
| 19 ds = ArrayDataSet(a,{'x':slice(3),'y':3,'z':[0,2]}) | |
| 20 print "len(ds)=",len(ds) | |
| 54 | 21 assert(len(ds)==10) |
| 45 | 22 print "example 0 = ",ds[0] |
| 54 | 23 # assert |
| 45 | 24 print "x=",ds["x"] |
| 25 print "x|y" | |
| 26 for x,y in ds("x","y"): | |
| 27 print x,y | |
| 28 minibatch_iterator = ds.minibatches(fieldnames=['z','y'],n_batches=1,minibatch_size=3,offset=4) | |
| 29 minibatch = minibatch_iterator.__iter__().next() | |
| 30 print "minibatch=",minibatch | |
| 31 for var in minibatch: | |
| 32 print "var=",var | |
|
48
b6730f9a336d
Fixing MinibatchDataSet getitem
bengioy@grenat.iro.umontreal.ca
parents:
45
diff
changeset
|
33 print "take a slice and look at field y",ds[1:6:2]["y"] |
| 45 | 34 |
|
58
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
35 def test_ArrayDataSet(): |
|
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
36 #don't test stream |
|
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
37 #tested only with float value |
| 106 | 38 #don't always test with y |
| 39 #don't test missing value | |
| 40 #don't test with tuple | |
| 41 #don't test proterties | |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
42 def test_iterate_over_examples(array,ds): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
43 #not in doc!!! |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
44 i=0 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
45 for example in range(len(ds)): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
46 assert (ds[example]['x']==a[example][:3]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
47 assert ds[example]['y']==a[example][3] |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
48 assert (ds[example]['z']==a[example][[0,2]]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
49 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
50 assert i==len(ds) |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
51 del example,i |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
52 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
53 # - for example in dataset: |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
54 i=0 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
55 for example in ds: |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
56 assert len(example)==3 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
57 assert (example['x']==array[i][:3]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
58 assert example['y']==array[i][3] |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
59 assert (example['z']==array[i][0:3:2]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
60 assert (numpy.append(example['x'],example['y'])==array[i]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
61 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
62 assert i==len(ds) |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
63 del example,i |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
64 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
65 # - for val1,val2,... in dataset: |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
66 i=0 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
67 for x,y,z in ds: |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
68 assert (x==array[i][:3]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
69 assert y==array[i][3] |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
70 assert (z==array[i][0:3:2]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
71 assert (numpy.append(x,y)==array[i]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
72 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
73 assert i==len(ds) |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
74 del x,y,z,i |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
75 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
76 # - for example in dataset(field1, field2,field3, ...): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
77 i=0 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
78 for example in ds('x','y','z'): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
79 assert len(example)==3 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
80 assert (example['x']==array[i][:3]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
81 assert example['y']==array[i][3] |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
82 assert (example['z']==array[i][0:3:2]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
83 assert (numpy.append(example['x'],example['y'])==array[i]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
84 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
85 assert i==len(ds) |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
86 del example,i |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
87 i=0 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
88 for example in ds('y','x'): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
89 assert len(example)==2 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
90 assert (example['x']==array[i][:3]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
91 assert example['y']==array[i][3] |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
92 assert (numpy.append(example['x'],example['y'])==array[i]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
93 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
94 assert i==len(ds) |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
95 del example,i |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
96 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
97 # - for val1,val2,val3 in dataset(field1, field2,field3): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
98 i=0 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
99 for x,y,z in ds('x','y','z'): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
100 assert (x==array[i][:3]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
101 assert y==array[i][3] |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
102 assert (z==array[i][0:3:2]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
103 assert (numpy.append(x,y)==array[i]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
104 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
105 assert i==len(ds) |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
106 del x,y,z,i |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
107 i=0 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
108 for y,x in ds('y','x',): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
109 assert (x==array[i][:3]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
110 assert y==array[i][3] |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
111 assert (numpy.append(x,y)==array[i]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
112 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
113 assert i==len(ds) |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
114 del x,y,i |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
115 |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
116 def test_minibatch_size(minibatch,minibatch_size,len_ds,nb_field,nb_iter_finished): |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
117 ##full minibatch or the last minibatch |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
118 for idx in range(nb_field): |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
119 test_minibatch_field_size(minibatch[idx],minibatch_size,len_ds,nb_iter_finished) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
120 del idx |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
121 def test_minibatch_field_size(minibatch_field,minibatch_size,len_ds,nb_iter_finished): |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
122 assert len(minibatch_field)==minibatch_size or ((nb_iter_finished*minibatch_size+len(minibatch_field))==len_ds and len(minibatch_field)<minibatch_size) |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
123 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
124 # - for minibatch in dataset.minibatches([field1, field2, ...],minibatch_size=N): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
125 i=0 |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
126 mi=0 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
127 m=ds.minibatches(['x','z'], minibatch_size=3) |
| 104 | 128 assert isinstance(m,DataSet.MinibatchWrapAroundIterator) |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
129 for minibatch in m: |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
130 assert len(minibatch)==2 |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
131 test_minibatch_size(minibatch,m.minibatch_size,len(ds),2,mi) |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
132 assert (minibatch[0][:,0:3:2]==minibatch[1]).all() |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
133 mi+=1 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
134 i+=len(minibatch[0]) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
135 assert i==len(ds) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
136 assert mi==4 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
137 del minibatch,i,m,mi |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
138 |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
139 i=0 |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
140 mi=0 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
141 m=ds.minibatches(['x','y'], minibatch_size=3) |
| 104 | 142 assert isinstance(m,DataSet.MinibatchWrapAroundIterator) |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
143 for minibatch in m: |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
144 assert len(minibatch)==2 |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
145 test_minibatch_size(minibatch,m.minibatch_size,len(ds),2,mi) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
146 mi+=1 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
147 for id in range(len(minibatch[0])): |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
148 assert (numpy.append(minibatch[0][id],minibatch[1][id])==a[i]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
149 i+=1 |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
150 assert i==len(ds) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
151 assert mi==4 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
152 del minibatch,i,id,m,mi |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
153 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
154 # - for mini1,mini2,mini3 in dataset.minibatches([field1, field2, field3], minibatch_size=N): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
155 i=0 |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
156 mi=0 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
157 m=ds.minibatches(['x','z'], minibatch_size=3) |
| 104 | 158 assert isinstance(m,DataSet.MinibatchWrapAroundIterator) |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
159 for x,z in m: |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
160 test_minibatch_field_size(x,m.minibatch_size,len(ds),mi) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
161 test_minibatch_field_size(z,m.minibatch_size,len(ds),mi) |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
162 assert (x[:,0:3:2]==z).all() |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
163 i+=len(x) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
164 mi+=1 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
165 assert i==len(ds) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
166 assert mi==4 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
167 del x,z,i,m,mi |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
168 i=0 |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
169 mi=0 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
170 m=ds.minibatches(['x','y'], minibatch_size=3) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
171 for x,y in m: |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
172 test_minibatch_field_size(x,m.minibatch_size,len(ds),mi) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
173 test_minibatch_field_size(y,m.minibatch_size,len(ds),mi) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
174 mi+=1 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
175 for id in range(len(x)): |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
176 assert (numpy.append(x[id],y[id])==a[i]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
177 i+=1 |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
178 assert i==len(ds) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
179 assert mi==4 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
180 del x,y,i,id,m,mi |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
181 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
182 #not in doc |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
183 i=0 |
| 104 | 184 m=ds.minibatches(['x','y'],n_batches=1,minibatch_size=3,offset=4) |
| 185 assert isinstance(m,DataSet.MinibatchWrapAroundIterator) | |
| 186 for x,y in m: | |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
187 assert len(x)==3 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
188 assert len(y)==3 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
189 for id in range(3): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
190 assert (numpy.append(x[id],y[id])==a[i+4]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
191 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
192 assert i==3 |
| 104 | 193 del x,y,i,id,m |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
194 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
195 i=0 |
| 104 | 196 m=ds.minibatches(['x','y'],n_batches=2,minibatch_size=3,offset=4) |
| 197 assert isinstance(m,DataSet.MinibatchWrapAroundIterator) | |
| 198 for x,y in m: | |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
199 assert len(x)==3 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
200 assert len(y)==3 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
201 for id in range(3): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
202 assert (numpy.append(x[id],y[id])==a[i+4]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
203 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
204 assert i==6 |
| 104 | 205 del x,y,i,id,m |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
206 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
207 i=0 |
|
103
a90d85fef3d4
clean up and a few more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
102
diff
changeset
|
208 m=ds.minibatches(['x','y'],n_batches=20,minibatch_size=3,offset=4) |
| 104 | 209 assert isinstance(m,DataSet.MinibatchWrapAroundIterator) |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
210 for x,y in m: |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
211 assert len(x)==3 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
212 assert len(y)==3 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
213 for id in range(3): |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
214 assert (numpy.append(x[id],y[id])==a[(i+4)%a.shape[0]]).all() |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
215 i+=1 |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
216 assert i==m.n_batches*m.minibatch_size |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
217 del x,y,i,id |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
218 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
219 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
220 def test_ds_iterator(array,iterator1,iterator2,iterator3): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
221 i=0 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
222 for x,y in iterator1: |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
223 assert (x==array[i][:3]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
224 assert y==array[i][3] |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
225 assert (numpy.append(x,y)==array[i]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
226 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
227 assert i==len(ds) |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
228 i=0 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
229 for y,z in iterator2: |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
230 assert y==array[i][3] |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
231 assert (z==array[i][0:3:2]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
232 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
233 assert i==len(ds) |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
234 i=0 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
235 for x,y,z in iterator3: |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
236 assert (x==array[i][:3]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
237 assert y==array[i][3] |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
238 assert (z==array[i][0:3:2]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
239 assert (numpy.append(x,y)==array[i]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
240 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
241 assert i==len(ds) |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
242 |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
243 def test_getitem(array,ds): |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
244 |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
245 def test_ds(orig,ds,index): |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
246 i=0 |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
247 assert len(ds)==len(index) |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
248 for x,z,y in ds('x','z','y'): |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
249 assert (orig[index[i]]['x']==array[index[i]][:3]).all() |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
250 assert (orig[index[i]]['x']==x).all() |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
251 assert orig[index[i]]['y']==array[index[i]][3] |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
252 assert orig[index[i]]['y']==y |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
253 assert (orig[index[i]]['z']==array[index[i]][0:3:2]).all() |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
254 assert (orig[index[i]]['z']==z).all() |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
255 i+=1 |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
256 del i |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
257 ds[0] |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
258 if len(ds)>2: |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
259 ds[:1] |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
260 ds[1:1] |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
261 ds[1:1:1] |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
262 if len(ds)>5: |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
263 ds[[1,2,3]] |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
264 for x in ds: |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
265 pass |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
266 |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
267 #ds[:n] returns a dataset with the n first examples. |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
268 ds2=ds[:3] |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
269 assert isinstance(ds2,DataSet) |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
270 test_ds(ds,ds2,index=[0,1,2]) |
| 106 | 271 del ds2 |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
272 |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
273 #ds[i1:i2:s]# returns a ds with the examples i1,i1+s,...i2-s. |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
274 ds2=ds[1:7:2] |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
275 assert isinstance(ds2,DataSet) |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
276 test_ds(ds,ds2,[1,3,5]) |
| 106 | 277 del ds2 |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
278 |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
279 #ds[i] |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
280 ds2=ds[5] |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
281 assert isinstance(ds2,Example) |
| 106 | 282 assert have_raised("ds["+str(len(ds))+"]") # index not defined |
| 283 assert not have_raised("ds["+str(len(ds)-1)+"]") | |
| 284 del ds2 | |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
285 |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
286 #ds[[i1,i2,...in]]# returns a ds with examples i1,i2,...in. |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
287 ds2=ds[[4,7,2,8]] |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
288 assert isinstance(ds2,DataSet) |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
289 test_ds(ds,ds2,[4,7,2,8]) |
| 106 | 290 del ds2 |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
291 |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
292 #ds[fieldname]# an iterable over the values of the field fieldname across |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
293 #the ds (the iterable is obtained by default by calling valuesVStack |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
294 #over the values for individual examples). |
| 106 | 295 assert have_raised("ds['h']") # h is not defined... |
| 296 assert have_raised("ds[['x']]") # bad syntax | |
| 297 assert not have_raised("ds['x']") | |
| 298 isinstance(ds['x'],DataSetFields) | |
| 299 ds2=ds['x'] | |
| 300 assert len(ds['x'])==10 | |
| 301 assert len(ds['y'])==10 | |
| 302 assert len(ds['z'])==10 | |
| 303 i=0 | |
| 304 for example in ds['x']: | |
| 305 assert (example==a[i][:3]).all() | |
| 306 i+=1 | |
| 307 i=0 | |
| 308 for example in ds['y']: | |
| 309 assert (example==a[i][3]).all() | |
| 310 i+=1 | |
| 311 i=0 | |
| 312 for example in ds['z']: | |
| 313 assert (example==a[i,0:3:2]).all() | |
| 314 i+=1 | |
| 315 del ds2,i | |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
316 |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
317 #ds.<property># returns the value of a property associated with |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
318 #the name <property>. The following properties should be supported: |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
319 # - 'description': a textual description or name for the ds |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
320 # - 'fieldtypes': a list of types (one per field) |
| 106 | 321 |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
322 #* ds1 | ds2 | ds3 == ds.hstack([ds1,ds2,ds3])#???? |
| 112 | 323 # i=0 |
| 324 # for example in hstack([ds('x'),ds('y'),ds('z')]): | |
| 325 # example==ds[i] | |
| 326 # i+=1 | |
| 327 # del i,example | |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
328 #* ds1 & ds2 & ds3 == ds.vstack([ds1,ds2,ds3])#???? |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
329 |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
330 |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
331 print "test_ArrayDataSet" |
|
58
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
332 a = numpy.random.rand(10,4) |
|
86
fdf72ea4f2bc
added function test_ds in test_ArrayDataSet who test a sub dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
84
diff
changeset
|
333 ds = ArrayDataSet(a,{'x':slice(3),'y':3,'z':[0,2]})###???tuple not tested |
|
91
eee739fefdff
corrected test from discution about the syntax with Yoshua
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
89
diff
changeset
|
334 ds = ArrayDataSet(a,LookupList(['x','y','z'],[slice(3),3,[0,2]]))###???tuple not tested |
|
58
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
335 assert len(ds)==10 |
|
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
336 #assert ds==a? should this work? |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
337 |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
338 test_iterate_over_examples(a, ds) |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
339 test_getitem(a, ds) |
|
91
eee739fefdff
corrected test from discution about the syntax with Yoshua
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
89
diff
changeset
|
340 |
|
eee739fefdff
corrected test from discution about the syntax with Yoshua
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
89
diff
changeset
|
341 # - for val1,val2,val3 in dataset(field1, field2,field3): |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
342 test_ds_iterator(a,ds('x','y'),ds('y','z'),ds('x','y','z')) |
|
89
05dc4804357b
more test and refactoring
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
87
diff
changeset
|
343 |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
344 |
| 63 | 345 assert len(ds.fields())==3 |
|
58
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
346 for field in ds.fields(): |
|
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
347 for field_value in field: # iterate over the values associated to that field for all the ds examples |
|
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
348 pass |
|
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
349 for field in ds('x','z').fields(): |
|
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
350 pass |
| 60 | 351 for field in ds.fields('x','y'): |
|
58
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
352 pass |
|
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
353 for field_examples in ds.fields(): |
|
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
354 for example_value in field_examples: |
|
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
355 pass |
| 63 | 356 |
| 357 assert ds == ds.fields().examples() | |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
358 # for ((x,y),a_v) in (ds('x','y'),a): #???don't work # haven't found a variant that work.# will not work |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
359 # assert numpy.append(x,y)==z |
|
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
360 |
|
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
361 def test_LookupList(): |
|
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
362 #test only the example in the doc??? |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
363 print "test_LookupList" |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
364 example = LookupList(['x','y','z'],[1,2,3]) |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
365 example['x'] = [1, 2, 3] # set or change a field |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
366 x, y, z = example |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
367 x = example[0] |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
368 x = example["x"] |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
369 assert example.keys()==['x','y','z'] |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
370 assert example.values()==[[1,2,3],2,3] |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
371 assert example.items()==[('x',[1,2,3]),('y',2),('z',3)] |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
372 example.append_keyval('u',0) # adds item with name 'u' and value 0 |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
373 assert len(example)==4 # number of items = 4 here |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
374 example2 = LookupList(['v','w'], ['a','b']) |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
375 example3 = LookupList(['x','y','z','u','v','w'], [[1, 2, 3],2,3,0,'a','b']) |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
376 assert example+example2==example3 |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
377 assert have_raised("example+example") |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
378 |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
379 def test_ApplyFunctionDataSet(): |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
380 print "test_ApplyFunctionDataSet" |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
381 raise NotImplementedError() |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
382 def test_CacheDataSet(): |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
383 print "test_CacheDataSet" |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
384 raise NotImplementedError() |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
385 def test_FieldsSubsetDataSet(): |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
386 print "test_FieldsSubsetDataSet" |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
387 raise NotImplementedError() |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
388 def test_DataSetFields(): |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
389 print "test_DataSetFields" |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
390 raise NotImplementedError() |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
391 def test_MinibatchDataSet(): |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
392 print "test_MinibatchDataSet" |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
393 raise NotImplementedError() |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
394 def test_HStackedDataSet(): |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
395 print "test_HStackedDataSet" |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
396 raise NotImplementedError() |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
397 def test_VStackedDataSet(): |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
398 print "test_VStackedDataSet" |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
399 raise NotImplementedError() |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
400 def test_ArrayFieldsDataSet(): |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
401 print "test_ArrayFieldsDataSet" |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
402 raise NotImplementedError() |
|
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
403 |
|
116
9330d941fa1f
added function load_pmat_as_array_dataset and save_array_dataset_as_pmat
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
112
diff
changeset
|
404 import pmat |
|
9330d941fa1f
added function load_pmat_as_array_dataset and save_array_dataset_as_pmat
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
112
diff
changeset
|
405 |
|
9330d941fa1f
added function load_pmat_as_array_dataset and save_array_dataset_as_pmat
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
112
diff
changeset
|
406 load_pmat_as_array_dataset("tmp.pmat") |
|
9330d941fa1f
added function load_pmat_as_array_dataset and save_array_dataset_as_pmat
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
112
diff
changeset
|
407 |
|
74
b4159cbdc06b
Fixed errors raised by test_dataset
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
66
diff
changeset
|
408 test1() |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
409 test_LookupList() |
|
65
d48eba49a2f4
fixed the infinite loop
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
64
diff
changeset
|
410 test_ArrayDataSet() |
|
64
863da25a60f1
trying to fix infinite loop
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
63
diff
changeset
|
411 |