Mercurial > pylearn
annotate _test_dataset.py @ 295:7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
| author | Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca> |
|---|---|
| date | Fri, 06 Jun 2008 17:11:25 -0400 |
| parents | f7924e13e426 |
| children | d08b71d186c8 |
| rev | line source |
|---|---|
|
51
59757365a057
the script can be autorun
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
45
diff
changeset
|
1 #!/bin/env python |
| 45 | 2 from dataset import * |
| 3 from math import * | |
|
289
3af204aa71e5
implemented test for FieldsSubsetDataSet. if executed, can take a --debug option to be used in a debugger
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
284
diff
changeset
|
4 import numpy, unittest, sys |
|
229
d7250ee86f72
Added speed test for ArraDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
202
diff
changeset
|
5 from misc import * |
| 45 | 6 |
|
145
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
7 def have_raised(to_eval, **var): |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
8 have_thrown = False |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
9 try: |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
10 eval(to_eval) |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
11 except : |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
12 have_thrown = True |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
13 return have_thrown |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
14 |
| 173 | 15 def have_raised2(f, *args, **kwargs): |
| 16 have_thrown = False | |
| 17 try: | |
| 18 f(*args, **kwargs) | |
| 19 except : | |
| 20 have_thrown = True | |
| 21 return have_thrown | |
| 22 | |
| 45 | 23 def test1(): |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
24 print "test1" |
| 45 | 25 global a,ds |
| 26 a = numpy.random.rand(10,4) | |
| 27 print a | |
| 28 ds = ArrayDataSet(a,{'x':slice(3),'y':3,'z':[0,2]}) | |
| 29 print "len(ds)=",len(ds) | |
| 54 | 30 assert(len(ds)==10) |
| 45 | 31 print "example 0 = ",ds[0] |
| 54 | 32 # assert |
| 45 | 33 print "x=",ds["x"] |
| 34 print "x|y" | |
| 35 for x,y in ds("x","y"): | |
| 36 print x,y | |
| 37 minibatch_iterator = ds.minibatches(fieldnames=['z','y'],n_batches=1,minibatch_size=3,offset=4) | |
| 38 minibatch = minibatch_iterator.__iter__().next() | |
| 39 print "minibatch=",minibatch | |
| 40 for var in minibatch: | |
| 41 print "var=",var | |
|
48
b6730f9a336d
Fixing MinibatchDataSet getitem
bengioy@grenat.iro.umontreal.ca
parents:
45
diff
changeset
|
42 print "take a slice and look at field y",ds[1:6:2]["y"] |
| 45 | 43 |
|
165
2a12e7437c56
small refactoring
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
164
diff
changeset
|
44 del a,ds,x,y,minibatch_iterator,minibatch,var |
|
2a12e7437c56
small refactoring
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
164
diff
changeset
|
45 |
|
145
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
46 def test_iterate_over_examples(array,ds): |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
47 #not in doc!!! |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
48 i=0 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
49 for example in range(len(ds)): |
| 293 | 50 wanted = array[example][:3] |
| 51 returned = ds[example]['x'] | |
| 52 if (wanted != returned).all(): | |
| 53 print 'returned:', returned | |
| 54 print 'wanted:', wanted | |
|
145
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
55 assert (ds[example]['x']==array[example][:3]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
56 assert ds[example]['y']==array[example][3] |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
57 assert (ds[example]['z']==array[example][[0,2]]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
58 i+=1 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
59 assert i==len(ds) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
60 del example,i |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
61 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
62 # - for example in dataset: |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
63 i=0 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
64 for example in ds: |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
65 assert len(example)==3 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
66 assert (example['x']==array[i][:3]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
67 assert example['y']==array[i][3] |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
68 assert (example['z']==array[i][0:3:2]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
69 assert (numpy.append(example['x'],example['y'])==array[i]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
70 i+=1 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
71 assert i==len(ds) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
72 del example,i |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
73 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
74 # - for val1,val2,... in dataset: |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
75 i=0 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
76 for x,y,z in ds: |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
77 assert (x==array[i][:3]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
78 assert y==array[i][3] |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
79 assert (z==array[i][0:3:2]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
80 assert (numpy.append(x,y)==array[i]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
81 i+=1 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
82 assert i==len(ds) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
83 del x,y,z,i |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
84 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
85 # - for example in dataset(field1, field2,field3, ...): |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
86 i=0 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
87 for example in ds('x','y','z'): |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
88 assert len(example)==3 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
89 assert (example['x']==array[i][:3]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
90 assert example['y']==array[i][3] |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
91 assert (example['z']==array[i][0:3:2]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
92 assert (numpy.append(example['x'],example['y'])==array[i]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
93 i+=1 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
94 assert i==len(ds) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
95 del example,i |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
96 i=0 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
97 for example in ds('y','x'): |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
98 assert len(example)==2 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
99 assert (example['x']==array[i][:3]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
100 assert example['y']==array[i][3] |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
101 assert (numpy.append(example['x'],example['y'])==array[i]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
102 i+=1 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
103 assert i==len(ds) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
104 del example,i |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
105 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
106 # - for val1,val2,val3 in dataset(field1, field2,field3): |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
107 i=0 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
108 for x,y,z in ds('x','y','z'): |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
109 assert (x==array[i][:3]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
110 assert y==array[i][3] |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
111 assert (z==array[i][0:3:2]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
112 assert (numpy.append(x,y)==array[i]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
113 i+=1 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
114 assert i==len(ds) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
115 del x,y,z,i |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
116 i=0 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
117 for y,x in ds('y','x',): |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
118 assert (x==array[i][:3]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
119 assert y==array[i][3] |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
120 assert (numpy.append(x,y)==array[i]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
121 i+=1 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
122 assert i==len(ds) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
123 del x,y,i |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
124 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
125 def test_minibatch_size(minibatch,minibatch_size,len_ds,nb_field,nb_iter_finished): |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
126 ##full minibatch or the last minibatch |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
127 for idx in range(nb_field): |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
128 test_minibatch_field_size(minibatch[idx],minibatch_size,len_ds,nb_iter_finished) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
129 del idx |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
130 def test_minibatch_field_size(minibatch_field,minibatch_size,len_ds,nb_iter_finished): |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
131 assert len(minibatch_field)==minibatch_size or ((nb_iter_finished*minibatch_size+len(minibatch_field))==len_ds and len(minibatch_field)<minibatch_size) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
132 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
133 # - for minibatch in dataset.minibatches([field1, field2, ...],minibatch_size=N): |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
134 i=0 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
135 mi=0 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
136 m=ds.minibatches(['x','z'], minibatch_size=3) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
137 assert isinstance(m,DataSet.MinibatchWrapAroundIterator) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
138 for minibatch in m: |
|
161
60e00cce3492
bugfix test in case it is not an ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
158
diff
changeset
|
139 assert isinstance(minibatch,DataSetFields) |
|
145
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
140 assert len(minibatch)==2 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
141 test_minibatch_size(minibatch,m.minibatch_size,len(ds),2,mi) |
|
161
60e00cce3492
bugfix test in case it is not an ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
158
diff
changeset
|
142 if type(ds)==ArrayDataSet: |
|
60e00cce3492
bugfix test in case it is not an ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
158
diff
changeset
|
143 assert (minibatch[0][:,::2]==minibatch[1]).all() |
|
60e00cce3492
bugfix test in case it is not an ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
158
diff
changeset
|
144 else: |
|
162
45427d4d64b3
bugfix in test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
161
diff
changeset
|
145 for j in xrange(len(minibatch[0])): |
|
45427d4d64b3
bugfix in test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
161
diff
changeset
|
146 (minibatch[0][j][::2]==minibatch[1][j]).all() |
|
145
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
147 mi+=1 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
148 i+=len(minibatch[0]) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
149 assert i==len(ds) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
150 assert mi==4 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
151 del minibatch,i,m,mi |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
152 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
153 i=0 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
154 mi=0 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
155 m=ds.minibatches(['x','y'], minibatch_size=3) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
156 assert isinstance(m,DataSet.MinibatchWrapAroundIterator) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
157 for minibatch in m: |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
158 assert len(minibatch)==2 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
159 test_minibatch_size(minibatch,m.minibatch_size,len(ds),2,mi) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
160 mi+=1 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
161 for id in range(len(minibatch[0])): |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
162 assert (numpy.append(minibatch[0][id],minibatch[1][id])==array[i]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
163 i+=1 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
164 assert i==len(ds) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
165 assert mi==4 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
166 del minibatch,i,id,m,mi |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
167 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
168 # - for mini1,mini2,mini3 in dataset.minibatches([field1, field2, field3], minibatch_size=N): |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
169 i=0 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
170 mi=0 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
171 m=ds.minibatches(['x','z'], minibatch_size=3) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
172 assert isinstance(m,DataSet.MinibatchWrapAroundIterator) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
173 for x,z in m: |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
174 test_minibatch_field_size(x,m.minibatch_size,len(ds),mi) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
175 test_minibatch_field_size(z,m.minibatch_size,len(ds),mi) |
|
164
3518710e16ec
don't assume we have an ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
162
diff
changeset
|
176 for id in range(len(x)): |
|
3518710e16ec
don't assume we have an ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
162
diff
changeset
|
177 assert (x[id][::2]==z[id]).all() |
|
3518710e16ec
don't assume we have an ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
162
diff
changeset
|
178 i+=1 |
|
145
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
179 mi+=1 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
180 assert i==len(ds) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
181 assert mi==4 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
182 del x,z,i,m,mi |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
183 i=0 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
184 mi=0 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
185 m=ds.minibatches(['x','y'], minibatch_size=3) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
186 for x,y in m: |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
187 test_minibatch_field_size(x,m.minibatch_size,len(ds),mi) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
188 test_minibatch_field_size(y,m.minibatch_size,len(ds),mi) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
189 mi+=1 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
190 for id in range(len(x)): |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
191 assert (numpy.append(x[id],y[id])==array[i]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
192 i+=1 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
193 assert i==len(ds) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
194 assert mi==4 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
195 del x,y,i,id,m,mi |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
196 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
197 #not in doc |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
198 i=0 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
199 m=ds.minibatches(['x','y'],n_batches=1,minibatch_size=3,offset=4) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
200 assert isinstance(m,DataSet.MinibatchWrapAroundIterator) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
201 for x,y in m: |
|
239
77b362a23f8e
more general test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
236
diff
changeset
|
202 assert len(x)==m.minibatch_size |
|
77b362a23f8e
more general test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
236
diff
changeset
|
203 assert len(y)==m.minibatch_size |
|
77b362a23f8e
more general test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
236
diff
changeset
|
204 for id in range(m.minibatch_size): |
|
145
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
205 assert (numpy.append(x[id],y[id])==array[i+4]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
206 i+=1 |
|
239
77b362a23f8e
more general test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
236
diff
changeset
|
207 assert i==m.n_batches*m.minibatch_size |
|
145
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
208 del x,y,i,id,m |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
209 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
210 i=0 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
211 m=ds.minibatches(['x','y'],n_batches=2,minibatch_size=3,offset=4) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
212 assert isinstance(m,DataSet.MinibatchWrapAroundIterator) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
213 for x,y in m: |
|
239
77b362a23f8e
more general test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
236
diff
changeset
|
214 assert len(x)==m.minibatch_size |
|
77b362a23f8e
more general test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
236
diff
changeset
|
215 assert len(y)==m.minibatch_size |
|
77b362a23f8e
more general test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
236
diff
changeset
|
216 for id in range(m.minibatch_size): |
|
145
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
217 assert (numpy.append(x[id],y[id])==array[i+4]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
218 i+=1 |
|
239
77b362a23f8e
more general test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
236
diff
changeset
|
219 assert i==m.n_batches*m.minibatch_size |
|
145
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
220 del x,y,i,id,m |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
221 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
222 i=0 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
223 m=ds.minibatches(['x','y'],n_batches=20,minibatch_size=3,offset=4) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
224 assert isinstance(m,DataSet.MinibatchWrapAroundIterator) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
225 for x,y in m: |
|
239
77b362a23f8e
more general test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
236
diff
changeset
|
226 assert len(x)==m.minibatch_size |
|
77b362a23f8e
more general test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
236
diff
changeset
|
227 assert len(y)==m.minibatch_size |
|
77b362a23f8e
more general test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
236
diff
changeset
|
228 for id in range(m.minibatch_size): |
|
145
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
229 assert (numpy.append(x[id],y[id])==array[(i+4)%array.shape[0]]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
230 i+=1 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
231 assert i==m.n_batches*m.minibatch_size |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
232 del x,y,i,id |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
233 |
| 293 | 234 assert not have_raised2(ds.minibatches,['x','y'],n_batches=1,minibatch_size=len(array)+1,offset=0) |
|
240
97f35d586727
more test. Are we wanting to limit the size of minibatch to the size of the dataset?
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
239
diff
changeset
|
235 assert not have_raised2(ds.minibatches,['x','y'],n_batches=1,minibatch_size=len(array),offset=0) |
|
145
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
236 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
237 def test_ds_iterator(array,iterator1,iterator2,iterator3): |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
238 l=len(iterator1) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
239 i=0 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
240 for x,y in iterator1: |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
241 assert (x==array[i][:3]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
242 assert y==array[i][3] |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
243 assert (numpy.append(x,y)==array[i]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
244 i+=1 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
245 assert i==l |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
246 i=0 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
247 for y,z in iterator2: |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
248 assert y==array[i][3] |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
249 assert (z==array[i][0:3:2]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
250 i+=1 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
251 assert i==l |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
252 i=0 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
253 for x,y,z in iterator3: |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
254 assert (x==array[i][:3]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
255 assert y==array[i][3] |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
256 assert (z==array[i][0:3:2]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
257 assert (numpy.append(x,y)==array[i]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
258 i+=1 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
259 assert i==l |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
260 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
261 def test_getitem(array,ds): |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
262 def test_ds(orig,ds,index): |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
263 i=0 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
264 assert len(ds)==len(index) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
265 for x,z,y in ds('x','z','y'): |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
266 assert (orig[index[i]]['x']==array[index[i]][:3]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
267 assert (orig[index[i]]['x']==x).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
268 assert orig[index[i]]['y']==array[index[i]][3] |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
269 assert orig[index[i]]['y']==y |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
270 assert (orig[index[i]]['z']==array[index[i]][0:3:2]).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
271 assert (orig[index[i]]['z']==z).all() |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
272 i+=1 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
273 del i |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
274 ds[0] |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
275 if len(ds)>2: |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
276 ds[:1] |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
277 ds[1:1] |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
278 ds[1:1:1] |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
279 if len(ds)>5: |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
280 ds[[1,2,3]] |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
281 for x in ds: |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
282 pass |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
283 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
284 #ds[:n] returns a dataset with the n first examples. |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
285 ds2=ds[:3] |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
286 assert isinstance(ds2,DataSet) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
287 test_ds(ds,ds2,index=[0,1,2]) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
288 del ds2 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
289 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
290 #ds[i1:i2:s]# returns a ds with the examples i1,i1+s,...i2-s. |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
291 ds2=ds[1:7:2] |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
292 assert isinstance(ds2,DataSet) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
293 test_ds(ds,ds2,[1,3,5]) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
294 del ds2 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
295 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
296 #ds[i] |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
297 ds2=ds[5] |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
298 assert isinstance(ds2,Example) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
299 assert have_raised("var['ds']["+str(len(ds))+"]",ds=ds) # index not defined |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
300 assert not have_raised("var['ds']["+str(len(ds)-1)+"]",ds=ds) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
301 del ds2 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
302 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
303 #ds[[i1,i2,...in]]# returns a ds with examples i1,i2,...in. |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
304 ds2=ds[[4,7,2,8]] |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
305 assert isinstance(ds2,DataSet) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
306 test_ds(ds,ds2,[4,7,2,8]) |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
307 del ds2 |
|
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
308 |
|
269
fdce496c3b56
deprecating __getitem__[fieldname] syntax
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
268
diff
changeset
|
309 #ds.<property># returns the value of a property associated with |
|
fdce496c3b56
deprecating __getitem__[fieldname] syntax
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
268
diff
changeset
|
310 #the name <property>. The following properties should be supported: |
|
fdce496c3b56
deprecating __getitem__[fieldname] syntax
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
268
diff
changeset
|
311 # - 'description': a textual description or name for the ds |
|
fdce496c3b56
deprecating __getitem__[fieldname] syntax
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
268
diff
changeset
|
312 # - 'fieldtypes': a list of types (one per field) |
|
145
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
313 |
|
269
fdce496c3b56
deprecating __getitem__[fieldname] syntax
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
268
diff
changeset
|
314 #* ds1 | ds2 | ds3 == ds.hstack([ds1,ds2,ds3])#???? |
|
fdce496c3b56
deprecating __getitem__[fieldname] syntax
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
268
diff
changeset
|
315 #assert hstack([ds('x','y'),ds('z')])==ds |
|
fdce496c3b56
deprecating __getitem__[fieldname] syntax
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
268
diff
changeset
|
316 #hstack([ds('z','y'),ds('x')])==ds |
| 173 | 317 assert have_raised2(hstack,[ds('x'),ds('x')]) |
| 318 assert have_raised2(hstack,[ds('y','x'),ds('x')]) | |
| 319 assert not have_raised2(hstack,[ds('x'),ds('y')]) | |
|
269
fdce496c3b56
deprecating __getitem__[fieldname] syntax
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
268
diff
changeset
|
320 |
|
fdce496c3b56
deprecating __getitem__[fieldname] syntax
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
268
diff
changeset
|
321 # i=0 |
|
fdce496c3b56
deprecating __getitem__[fieldname] syntax
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
268
diff
changeset
|
322 # for example in hstack([ds('x'),ds('y'),ds('z')]): |
|
fdce496c3b56
deprecating __getitem__[fieldname] syntax
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
268
diff
changeset
|
323 # example==ds[i] |
|
fdce496c3b56
deprecating __getitem__[fieldname] syntax
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
268
diff
changeset
|
324 # i+=1 |
|
fdce496c3b56
deprecating __getitem__[fieldname] syntax
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
268
diff
changeset
|
325 # del i,example |
|
fdce496c3b56
deprecating __getitem__[fieldname] syntax
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
268
diff
changeset
|
326 #* ds1 & ds2 & ds3 == ds.vstack([ds1,ds2,ds3])#???? |
|
145
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
327 |
|
165
2a12e7437c56
small refactoring
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
164
diff
changeset
|
328 def test_fields_fct(ds): |
|
2a12e7437c56
small refactoring
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
164
diff
changeset
|
329 #@todo, fill correctly |
|
2a12e7437c56
small refactoring
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
164
diff
changeset
|
330 assert len(ds.fields())==3 |
| 168 | 331 i=0 |
| 332 v=0 | |
|
165
2a12e7437c56
small refactoring
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
164
diff
changeset
|
333 for field in ds.fields(): |
|
2a12e7437c56
small refactoring
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
164
diff
changeset
|
334 for field_value in field: # iterate over the values associated to that field for all the ds examples |
| 168 | 335 v+=1 |
| 336 i+=1 | |
| 337 assert i==3 | |
| 338 assert v==3*10 | |
| 339 del i,v | |
| 340 | |
| 341 i=0 | |
| 342 v=0 | |
|
165
2a12e7437c56
small refactoring
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
164
diff
changeset
|
343 for field in ds('x','z').fields(): |
| 168 | 344 i+=1 |
| 345 for val in field: | |
| 346 v+=1 | |
| 347 assert i==2 | |
| 348 assert v==2*10 | |
| 349 del i,v | |
| 350 | |
| 351 i=0 | |
| 352 v=0 | |
|
165
2a12e7437c56
small refactoring
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
164
diff
changeset
|
353 for field in ds.fields('x','y'): |
| 168 | 354 i+=1 |
| 355 for val in field: | |
| 356 v+=1 | |
| 357 assert i==2 | |
| 358 assert v==2*10 | |
| 359 del i,v | |
| 360 | |
| 361 i=0 | |
| 362 v=0 | |
|
165
2a12e7437c56
small refactoring
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
164
diff
changeset
|
363 for field_examples in ds.fields(): |
|
2a12e7437c56
small refactoring
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
164
diff
changeset
|
364 for example_value in field_examples: |
| 168 | 365 v+=1 |
| 366 i+=1 | |
| 367 assert i==3 | |
| 368 assert v==3*10 | |
| 369 del i,v | |
| 370 | |
|
165
2a12e7437c56
small refactoring
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
164
diff
changeset
|
371 assert ds == ds.fields().examples() |
| 168 | 372 assert len(ds('x','y').fields()) == 2 |
| 373 assert len(ds('x','z').fields()) == 2 | |
| 374 assert len(ds('y').fields()) == 1 | |
|
165
2a12e7437c56
small refactoring
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
164
diff
changeset
|
375 |
| 168 | 376 del field |
|
194
0040ba0f0666
small refactoring
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
173
diff
changeset
|
377 def test_all(array,ds): |
|
0040ba0f0666
small refactoring
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
173
diff
changeset
|
378 assert len(ds)==10 |
|
0040ba0f0666
small refactoring
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
173
diff
changeset
|
379 |
|
0040ba0f0666
small refactoring
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
173
diff
changeset
|
380 test_iterate_over_examples(array, ds) |
|
0040ba0f0666
small refactoring
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
173
diff
changeset
|
381 test_getitem(array, ds) |
|
0040ba0f0666
small refactoring
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
173
diff
changeset
|
382 test_ds_iterator(array,ds('x','y'),ds('y','z'),ds('x','y','z')) |
|
0040ba0f0666
small refactoring
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
173
diff
changeset
|
383 test_fields_fct(ds) |
|
145
933db7ece663
make some function global to reuse them to test other dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
139
diff
changeset
|
384 |
|
279
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
385 class T_DataSet(unittest.TestCase): |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
386 def test_ArrayDataSet(self): |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
387 #don't test stream |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
388 #tested only with float value |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
389 #don't always test with y |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
390 #don't test missing value |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
391 #don't test with tuple |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
392 #don't test proterties |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
393 a2 = numpy.random.rand(10,4) |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
394 ds = ArrayDataSet(a2,{'x':slice(3),'y':3,'z':[0,2]})###???tuple not tested |
| 292 | 395 ds = ArrayDataSet(a2,Example(['x','y','z'],[slice(3),3,[0,2]]))###???tuple not tested |
|
279
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
396 #assert ds==a? should this work? |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
397 |
|
279
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
398 test_all(a2,ds) |
|
169
051e07807554
added test for CachedDataSet(ds,cache_all_upon_construction=True)
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
168
diff
changeset
|
399 |
|
279
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
400 del a2, ds |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
401 |
|
279
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
402 def test_CachedDataSet(self): |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
403 a = numpy.random.rand(10,4) |
| 292 | 404 ds1 = ArrayDataSet(a,Example(['x','y','z'],[slice(3),3,[0,2]]))###???tuple not tested |
|
279
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
405 ds2 = CachedDataSet(ds1) |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
406 ds3 = CachedDataSet(ds1,cache_all_upon_construction=True) |
|
148
a5329e719229
added test for CachedDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
145
diff
changeset
|
407 |
|
279
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
408 test_all(a,ds2) |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
409 test_all(a,ds3) |
|
169
051e07807554
added test for CachedDataSet(ds,cache_all_upon_construction=True)
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
168
diff
changeset
|
410 |
|
279
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
411 del a,ds1,ds2,ds3 |
|
148
a5329e719229
added test for CachedDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
145
diff
changeset
|
412 |
|
a5329e719229
added test for CachedDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
145
diff
changeset
|
413 |
|
279
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
414 def test_DataSetFields(self): |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
415 raise NotImplementedError() |
|
161
60e00cce3492
bugfix test in case it is not an ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
158
diff
changeset
|
416 |
|
279
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
417 def test_ApplyFunctionDataSet(self): |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
418 a = numpy.random.rand(10,4) |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
419 a2 = a+1 |
| 292 | 420 ds1 = ArrayDataSet(a,Example(['x','y','z'],[slice(3),3,[0,2]]))###???tuple not tested |
|
202
b9950ae5e54b
Added test for ApplyFunctionDataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
195
diff
changeset
|
421 |
|
279
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
422 ds2 = ApplyFunctionDataSet(ds1,lambda x,y,z: (x+1,y+1,z+1), ['x','y','z'],minibatch_mode=False) |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
423 ds3 = ApplyFunctionDataSet(ds1,lambda x,y,z: (numpy.array(x)+1,numpy.array(y)+1,numpy.array(z)+1), |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
424 ['x','y','z'], |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
425 minibatch_mode=True) |
|
202
b9950ae5e54b
Added test for ApplyFunctionDataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
195
diff
changeset
|
426 |
|
279
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
427 test_all(a2,ds2) |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
428 test_all(a2,ds3) |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
429 |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
430 del a,ds1,ds2,ds3 |
|
202
b9950ae5e54b
Added test for ApplyFunctionDataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
195
diff
changeset
|
431 |
|
279
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
432 def test_FieldsSubsetDataSet(self): |
|
289
3af204aa71e5
implemented test for FieldsSubsetDataSet. if executed, can take a --debug option to be used in a debugger
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
284
diff
changeset
|
433 a = numpy.random.rand(10,4) |
|
3af204aa71e5
implemented test for FieldsSubsetDataSet. if executed, can take a --debug option to be used in a debugger
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
284
diff
changeset
|
434 ds = ArrayDataSet(a,LookupList(['x','y','z','w'],[slice(3),3,[0,2],0])) |
|
3af204aa71e5
implemented test for FieldsSubsetDataSet. if executed, can take a --debug option to be used in a debugger
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
284
diff
changeset
|
435 ds = FieldsSubsetDataSet(ds,['x','y','z']) |
|
3af204aa71e5
implemented test for FieldsSubsetDataSet. if executed, can take a --debug option to be used in a debugger
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
284
diff
changeset
|
436 |
|
3af204aa71e5
implemented test for FieldsSubsetDataSet. if executed, can take a --debug option to be used in a debugger
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
284
diff
changeset
|
437 test_all(a,ds) |
|
3af204aa71e5
implemented test for FieldsSubsetDataSet. if executed, can take a --debug option to be used in a debugger
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
284
diff
changeset
|
438 |
|
3af204aa71e5
implemented test for FieldsSubsetDataSet. if executed, can take a --debug option to be used in a debugger
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
284
diff
changeset
|
439 del a, ds |
|
295
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
440 |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
441 def test_MultiLengthDataSet(self): |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
442 class MultiLengthDataSet(DataSet): |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
443 """ Dummy dataset, where one field is a ndarray of variables size. """ |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
444 def __len__(self) : |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
445 return 100 |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
446 def fieldNames(self) : |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
447 return 'input','target','name' |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
448 def minibatches_nowrap(self,fieldnames,minibatch_size,n_batches,offset): |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
449 class MultiLengthDataSetIterator(object): |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
450 def __init__(self,dataset,fieldnames,minibatch_size,n_batches,offset): |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
451 if fieldnames is None: fieldnames = dataset.fieldNames() |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
452 self.minibatch = LookupList(fieldnames,range(len(fieldnames))) |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
453 self.dataset, self.minibatch_size, self.current = dataset, minibatch_size, offset |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
454 def __iter__(self): |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
455 return self |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
456 def next(self): |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
457 for k in self.minibatch._names : |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
458 self.minibatch[k] = [] |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
459 for ex in range(self.minibatch_size) : |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
460 if 'input' in self.minibatch._names: |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
461 self.minibatch['input'].append( numpy.array( range(self.current + 1) ) ) |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
462 if 'target' in self.minibatch._names: |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
463 self.minibatch['target'].append( self.current % 2 ) |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
464 if 'name' in self.minibatch._names: |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
465 self.minibatch['name'].append( str(self.current) ) |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
466 self.current += 1 |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
467 return self.minibatch |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
468 return MultiLengthDataSetIterator(self,fieldnames,minibatch_size,n_batches,offset) |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
469 ds = MultiLengthDataSet() |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
470 for k in range(len(ds)): |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
471 x = ds[k] |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
472 dsa = ApplyFunctionDataset(ds,lambda x,y,z: (x[-1],y*10,int(z)),['input','target','name'],minibatch_mode=True) |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
473 # needs more testing using ds, dsa, dscache, ... |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
474 raise NotImplementedError() |
|
7380376816e5
started a test for datasets where one field has a variable length. Not obvious, all tests requires a matrix as a reference
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents:
294
diff
changeset
|
475 |
|
279
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
476 def test_MinibatchDataSet(self): |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
477 raise NotImplementedError() |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
478 def test_HStackedDataSet(self): |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
479 raise NotImplementedError() |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
480 def test_VStackedDataSet(self): |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
481 raise NotImplementedError() |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
482 def test_ArrayFieldsDataSet(self): |
|
2e22cc120688
passed test_dataset.py to unittest
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
278
diff
changeset
|
483 raise NotImplementedError() |
|
229
d7250ee86f72
Added speed test for ArraDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
202
diff
changeset
|
484 |
|
d7250ee86f72
Added speed test for ArraDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
202
diff
changeset
|
485 |
|
157
e8c8022582f4
only exec the test if we are the main
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
149
diff
changeset
|
486 if __name__=='__main__': |
|
289
3af204aa71e5
implemented test for FieldsSubsetDataSet. if executed, can take a --debug option to be used in a debugger
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
284
diff
changeset
|
487 if len(sys.argv)==2: |
|
3af204aa71e5
implemented test for FieldsSubsetDataSet. if executed, can take a --debug option to be used in a debugger
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
284
diff
changeset
|
488 if sys.argv[1]=="--debug": |
|
3af204aa71e5
implemented test for FieldsSubsetDataSet. if executed, can take a --debug option to be used in a debugger
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
284
diff
changeset
|
489 module = __import__("_test_dataset") |
|
3af204aa71e5
implemented test for FieldsSubsetDataSet. if executed, can take a --debug option to be used in a debugger
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
284
diff
changeset
|
490 tests = unittest.TestLoader().loadTestsFromModule(module) |
|
3af204aa71e5
implemented test for FieldsSubsetDataSet. if executed, can take a --debug option to be used in a debugger
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
284
diff
changeset
|
491 tests.debug() |
|
3af204aa71e5
implemented test for FieldsSubsetDataSet. if executed, can take a --debug option to be used in a debugger
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
284
diff
changeset
|
492 print "bad argument: only --debug is accepted" |
|
3af204aa71e5
implemented test for FieldsSubsetDataSet. if executed, can take a --debug option to be used in a debugger
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
284
diff
changeset
|
493 elif len(sys.argv)==1: |
|
3af204aa71e5
implemented test for FieldsSubsetDataSet. if executed, can take a --debug option to be used in a debugger
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
284
diff
changeset
|
494 unittest.main() |
|
3af204aa71e5
implemented test for FieldsSubsetDataSet. if executed, can take a --debug option to be used in a debugger
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
284
diff
changeset
|
495 else: |
|
3af204aa71e5
implemented test for FieldsSubsetDataSet. if executed, can take a --debug option to be used in a debugger
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
284
diff
changeset
|
496 print "bad argument: only --debug is accepted" |
|
229
d7250ee86f72
Added speed test for ArraDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
202
diff
changeset
|
497 |