Mercurial > pylearn
annotate test_dataset.py @ 105:8c0a1b11b007
bugfix, we keep all the line, but only a some columns
| author | Frederic Bastien <bastienf@iro.umontreal.ca> |
|---|---|
| date | Tue, 06 May 2008 16:34:48 -0400 |
| parents | e1a004b21daa |
| children | cf9bdb1d9656 |
| rev | line source |
|---|---|
|
51
59757365a057
the script can be autorun
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
45
diff
changeset
|
1 #!/bin/env python |
| 45 | 2 from dataset import * |
| 3 from math import * | |
| 4 import numpy | |
| 5 | |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
6 def have_raised(to_eval): |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
7 have_thrown = False |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
8 try: |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
9 eval(to_eval) |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
10 except : |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
11 have_thrown = True |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
12 return have_thrown |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
13 |
| 45 | 14 def test1(): |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
15 print "test1" |
| 45 | 16 global a,ds |
| 17 a = numpy.random.rand(10,4) | |
| 18 print a | |
| 19 ds = ArrayDataSet(a,{'x':slice(3),'y':3,'z':[0,2]}) | |
| 20 print "len(ds)=",len(ds) | |
| 54 | 21 assert(len(ds)==10) |
| 45 | 22 print "example 0 = ",ds[0] |
| 54 | 23 # assert |
| 45 | 24 print "x=",ds["x"] |
| 25 print "x|y" | |
| 26 for x,y in ds("x","y"): | |
| 27 print x,y | |
| 28 minibatch_iterator = ds.minibatches(fieldnames=['z','y'],n_batches=1,minibatch_size=3,offset=4) | |
| 29 minibatch = minibatch_iterator.__iter__().next() | |
| 30 print "minibatch=",minibatch | |
| 31 for var in minibatch: | |
| 32 print "var=",var | |
|
48
b6730f9a336d
Fixing MinibatchDataSet getitem
bengioy@grenat.iro.umontreal.ca
parents:
45
diff
changeset
|
33 print "take a slice and look at field y",ds[1:6:2]["y"] |
| 45 | 34 |
|
58
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
35 def test_ArrayDataSet(): |
|
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
36 #don't test stream |
|
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
37 #tested only with float value |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
38 #test with y too |
|
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
39 #test missing value |
|
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
40 |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
41 def test_iterate_over_examples(array,ds): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
42 #not in doc!!! |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
43 i=0 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
44 for example in range(len(ds)): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
45 assert (ds[example]['x']==a[example][:3]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
46 assert ds[example]['y']==a[example][3] |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
47 assert (ds[example]['z']==a[example][[0,2]]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
48 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
49 assert i==len(ds) |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
50 del example,i |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
51 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
52 # - for example in dataset: |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
53 i=0 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
54 for example in ds: |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
55 assert len(example)==3 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
56 assert (example['x']==array[i][:3]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
57 assert example['y']==array[i][3] |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
58 assert (example['z']==array[i][0:3:2]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
59 assert (numpy.append(example['x'],example['y'])==array[i]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
60 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
61 assert i==len(ds) |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
62 del example,i |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
63 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
64 # - for val1,val2,... in dataset: |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
65 i=0 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
66 for x,y,z in ds: |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
67 assert (x==array[i][:3]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
68 assert y==array[i][3] |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
69 assert (z==array[i][0:3:2]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
70 assert (numpy.append(x,y)==array[i]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
71 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
72 assert i==len(ds) |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
73 del x,y,z,i |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
74 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
75 # - for example in dataset(field1, field2,field3, ...): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
76 i=0 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
77 for example in ds('x','y','z'): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
78 assert len(example)==3 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
79 assert (example['x']==array[i][:3]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
80 assert example['y']==array[i][3] |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
81 assert (example['z']==array[i][0:3:2]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
82 assert (numpy.append(example['x'],example['y'])==array[i]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
83 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
84 assert i==len(ds) |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
85 del example,i |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
86 i=0 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
87 for example in ds('y','x'): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
88 assert len(example)==2 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
89 assert (example['x']==array[i][:3]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
90 assert example['y']==array[i][3] |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
91 assert (numpy.append(example['x'],example['y'])==array[i]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
92 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
93 assert i==len(ds) |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
94 del example,i |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
95 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
96 # - for val1,val2,val3 in dataset(field1, field2,field3): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
97 i=0 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
98 for x,y,z in ds('x','y','z'): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
99 assert (x==array[i][:3]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
100 assert y==array[i][3] |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
101 assert (z==array[i][0:3:2]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
102 assert (numpy.append(x,y)==array[i]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
103 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
104 assert i==len(ds) |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
105 del x,y,z,i |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
106 i=0 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
107 for y,x in ds('y','x',): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
108 assert (x==array[i][:3]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
109 assert y==array[i][3] |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
110 assert (numpy.append(x,y)==array[i]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
111 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
112 assert i==len(ds) |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
113 del x,y,i |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
114 |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
115 def test_minibatch_size(minibatch,minibatch_size,len_ds,nb_field,nb_iter_finished): |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
116 ##full minibatch or the last minibatch |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
117 for idx in range(nb_field): |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
118 test_minibatch_field_size(minibatch[idx],minibatch_size,len_ds,nb_iter_finished) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
119 del idx |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
120 def test_minibatch_field_size(minibatch_field,minibatch_size,len_ds,nb_iter_finished): |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
121 assert len(minibatch_field)==minibatch_size or ((nb_iter_finished*minibatch_size+len(minibatch_field))==len_ds and len(minibatch_field)<minibatch_size) |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
122 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
123 # - for minibatch in dataset.minibatches([field1, field2, ...],minibatch_size=N): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
124 i=0 |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
125 mi=0 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
126 m=ds.minibatches(['x','z'], minibatch_size=3) |
| 104 | 127 assert isinstance(m,DataSet.MinibatchWrapAroundIterator) |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
128 for minibatch in m: |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
129 assert len(minibatch)==2 |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
130 test_minibatch_size(minibatch,m.minibatch_size,len(ds),2,mi) |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
131 assert (minibatch[0][:,0:3:2]==minibatch[1]).all() |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
132 mi+=1 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
133 i+=len(minibatch[0]) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
134 assert i==len(ds) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
135 assert mi==4 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
136 del minibatch,i,m,mi |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
137 |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
138 i=0 |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
139 mi=0 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
140 m=ds.minibatches(['x','y'], minibatch_size=3) |
| 104 | 141 assert isinstance(m,DataSet.MinibatchWrapAroundIterator) |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
142 for minibatch in m: |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
143 assert len(minibatch)==2 |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
144 test_minibatch_size(minibatch,m.minibatch_size,len(ds),2,mi) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
145 mi+=1 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
146 for id in range(len(minibatch[0])): |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
147 assert (numpy.append(minibatch[0][id],minibatch[1][id])==a[i]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
148 i+=1 |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
149 assert i==len(ds) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
150 assert mi==4 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
151 del minibatch,i,id,m,mi |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
152 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
153 # - for mini1,mini2,mini3 in dataset.minibatches([field1, field2, field3], minibatch_size=N): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
154 i=0 |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
155 mi=0 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
156 m=ds.minibatches(['x','z'], minibatch_size=3) |
| 104 | 157 assert isinstance(m,DataSet.MinibatchWrapAroundIterator) |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
158 for x,z in m: |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
159 test_minibatch_field_size(x,m.minibatch_size,len(ds),mi) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
160 test_minibatch_field_size(z,m.minibatch_size,len(ds),mi) |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
161 assert (x[:,0:3:2]==z).all() |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
162 i+=len(x) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
163 mi+=1 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
164 assert i==len(ds) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
165 assert mi==4 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
166 del x,z,i,m,mi |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
167 i=0 |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
168 mi=0 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
169 m=ds.minibatches(['x','y'], minibatch_size=3) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
170 for x,y in m: |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
171 test_minibatch_field_size(x,m.minibatch_size,len(ds),mi) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
172 test_minibatch_field_size(y,m.minibatch_size,len(ds),mi) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
173 mi+=1 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
174 for id in range(len(x)): |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
175 assert (numpy.append(x[id],y[id])==a[i]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
176 i+=1 |
|
102
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
177 assert i==len(ds) |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
178 assert mi==4 |
|
4537ac630348
modifed test to accomodate the last change in dataset.py.
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
100
diff
changeset
|
179 del x,y,i,id,m,mi |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
180 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
181 #not in doc |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
182 i=0 |
| 104 | 183 m=ds.minibatches(['x','y'],n_batches=1,minibatch_size=3,offset=4) |
| 184 assert isinstance(m,DataSet.MinibatchWrapAroundIterator) | |
| 185 for x,y in m: | |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
186 assert len(x)==3 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
187 assert len(y)==3 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
188 for id in range(3): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
189 assert (numpy.append(x[id],y[id])==a[i+4]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
190 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
191 assert i==3 |
| 104 | 192 del x,y,i,id,m |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
193 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
194 i=0 |
| 104 | 195 m=ds.minibatches(['x','y'],n_batches=2,minibatch_size=3,offset=4) |
| 196 assert isinstance(m,DataSet.MinibatchWrapAroundIterator) | |
| 197 for x,y in m: | |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
198 assert len(x)==3 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
199 assert len(y)==3 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
200 for id in range(3): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
201 assert (numpy.append(x[id],y[id])==a[i+4]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
202 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
203 assert i==6 |
| 104 | 204 del x,y,i,id,m |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
205 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
206 i=0 |
|
103
a90d85fef3d4
clean up and a few more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
102
diff
changeset
|
207 m=ds.minibatches(['x','y'],n_batches=20,minibatch_size=3,offset=4) |
| 104 | 208 assert isinstance(m,DataSet.MinibatchWrapAroundIterator) |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
209 for x,y in m: |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
210 assert len(x)==3 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
211 assert len(y)==3 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
212 for id in range(3): |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
213 assert (numpy.append(x[id],y[id])==a[(i+4)%a.shape[0]]).all() |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
214 i+=1 |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
215 assert i==m.n_batches*m.minibatch_size |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
216 del x,y,i,id |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
217 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
218 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
219 def test_ds_iterator(array,iterator1,iterator2,iterator3): |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
220 i=0 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
221 for x,y in iterator1: |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
222 assert (x==array[i][:3]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
223 assert y==array[i][3] |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
224 assert (numpy.append(x,y)==array[i]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
225 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
226 assert i==len(ds) |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
227 i=0 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
228 for y,z in iterator2: |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
229 assert y==array[i][3] |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
230 assert (z==array[i][0:3:2]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
231 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
232 assert i==len(ds) |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
233 i=0 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
234 for x,y,z in iterator3: |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
235 assert (x==array[i][:3]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
236 assert y==array[i][3] |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
237 assert (z==array[i][0:3:2]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
238 assert (numpy.append(x,y)==array[i]).all() |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
239 i+=1 |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
240 assert i==len(ds) |
|
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
241 |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
242 def test_getitem(array,ds): |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
243 |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
244 def test_ds(orig,ds,index): |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
245 i=0 |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
246 assert len(ds)==len(index) |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
247 for x,z,y in ds('x','z','y'): |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
248 assert (orig[index[i]]['x']==array[index[i]][:3]).all() |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
249 assert (orig[index[i]]['x']==x).all() |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
250 assert orig[index[i]]['y']==array[index[i]][3] |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
251 assert orig[index[i]]['y']==y |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
252 assert (orig[index[i]]['z']==array[index[i]][0:3:2]).all() |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
253 assert (orig[index[i]]['z']==z).all() |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
254 i+=1 |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
255 del i |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
256 ds[0] |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
257 if len(ds)>2: |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
258 ds[:1] |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
259 ds[1:1] |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
260 ds[1:1:1] |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
261 if len(ds)>5: |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
262 ds[[1,2,3]] |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
263 for x in ds: |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
264 pass |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
265 |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
266 assert have_raised("ds['h']") # h is not defined... |
|
103
a90d85fef3d4
clean up and a few more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
102
diff
changeset
|
267 assert have_raised("ds[['x']]") # h is not defined... |
|
a90d85fef3d4
clean up and a few more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
102
diff
changeset
|
268 assert not have_raised("ds['x']") |
|
a90d85fef3d4
clean up and a few more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
102
diff
changeset
|
269 assert have_raised("ds["+str(len(ds))+"]") # index not defined |
|
a90d85fef3d4
clean up and a few more test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
102
diff
changeset
|
270 assert not have_raised("ds["+str(len(ds)-1)+"]") |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
271 |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
272 #ds[:n] returns a dataset with the n first examples. |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
273 ds2=ds[:3] |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
274 assert isinstance(ds2,DataSet) |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
275 test_ds(ds,ds2,index=[0,1,2]) |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
276 |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
277 #ds[i1:i2:s]# returns a ds with the examples i1,i1+s,...i2-s. |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
278 ds2=ds[1:7:2] |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
279 assert isinstance(ds2,DataSet) |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
280 test_ds(ds,ds2,[1,3,5]) |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
281 |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
282 #ds[i] |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
283 ds2=ds[5] |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
284 assert isinstance(ds2,Example) |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
285 |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
286 #ds[[i1,i2,...in]]# returns a ds with examples i1,i2,...in. |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
287 ds2=ds[[4,7,2,8]] |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
288 assert isinstance(ds2,DataSet) |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
289 test_ds(ds,ds2,[4,7,2,8]) |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
290 |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
291 #ds[fieldname]# an iterable over the values of the field fieldname across |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
292 #the ds (the iterable is obtained by default by calling valuesVStack |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
293 #over the values for individual examples). |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
294 |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
295 #ds.<property># returns the value of a property associated with |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
296 #the name <property>. The following properties should be supported: |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
297 # - 'description': a textual description or name for the ds |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
298 # - 'fieldtypes': a list of types (one per field) |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
299 #* ds1 | ds2 | ds3 == ds.hstack([ds1,ds2,ds3])#???? |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
300 #* ds1 & ds2 & ds3 == ds.vstack([ds1,ds2,ds3])#???? |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
301 |
|
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
302 |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
303 print "test_ArrayDataSet" |
|
58
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
304 a = numpy.random.rand(10,4) |
|
86
fdf72ea4f2bc
added function test_ds in test_ArrayDataSet who test a sub dataset
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
84
diff
changeset
|
305 ds = ArrayDataSet(a,{'x':slice(3),'y':3,'z':[0,2]})###???tuple not tested |
|
91
eee739fefdff
corrected test from discution about the syntax with Yoshua
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
89
diff
changeset
|
306 ds = ArrayDataSet(a,LookupList(['x','y','z'],[slice(3),3,[0,2]]))###???tuple not tested |
|
58
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
307 assert len(ds)==10 |
|
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
308 #assert ds==a? should this work? |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
309 |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
310 test_iterate_over_examples(a, ds) |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
311 test_getitem(a, ds) |
|
91
eee739fefdff
corrected test from discution about the syntax with Yoshua
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
89
diff
changeset
|
312 |
|
eee739fefdff
corrected test from discution about the syntax with Yoshua
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
89
diff
changeset
|
313 # - for val1,val2,val3 in dataset(field1, field2,field3): |
|
96
352910e0dbf5
added test and some restructuring for futur use
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
94
diff
changeset
|
314 test_ds_iterator(a,ds('x','y'),ds('y','z'),ds('x','y','z')) |
|
89
05dc4804357b
more test and refactoring
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
87
diff
changeset
|
315 |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
316 |
| 63 | 317 assert len(ds.fields())==3 |
|
58
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
318 for field in ds.fields(): |
|
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
319 for field_value in field: # iterate over the values associated to that field for all the ds examples |
|
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
320 pass |
|
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
321 for field in ds('x','z').fields(): |
|
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
322 pass |
| 60 | 323 for field in ds.fields('x','y'): |
|
58
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
324 pass |
|
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
325 for field_examples in ds.fields(): |
|
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
326 for example_value in field_examples: |
|
17729d7104fa
added function test_ArrayDataSet
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
54
diff
changeset
|
327 pass |
| 63 | 328 |
| 329 assert ds == ds.fields().examples() | |
|
100
574f4db76022
restructuring and added test
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
96
diff
changeset
|
330 # for ((x,y),a_v) in (ds('x','y'),a): #???don't work # haven't found a variant that work.# will not work |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
331 # assert numpy.append(x,y)==z |
|
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
332 |
|
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
333 def test_LookupList(): |
|
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
334 #test only the example in the doc??? |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
335 print "test_LookupList" |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
336 example = LookupList(['x','y','z'],[1,2,3]) |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
337 example['x'] = [1, 2, 3] # set or change a field |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
338 x, y, z = example |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
339 x = example[0] |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
340 x = example["x"] |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
341 assert example.keys()==['x','y','z'] |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
342 assert example.values()==[[1,2,3],2,3] |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
343 assert example.items()==[('x',[1,2,3]),('y',2),('z',3)] |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
344 example.append_keyval('u',0) # adds item with name 'u' and value 0 |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
345 assert len(example)==4 # number of items = 4 here |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
346 example2 = LookupList(['v','w'], ['a','b']) |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
347 example3 = LookupList(['x','y','z','u','v','w'], [[1, 2, 3],2,3,0,'a','b']) |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
348 assert example+example2==example3 |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
349 assert have_raised("example+example") |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
350 |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
351 def test_ApplyFunctionDataSet(): |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
352 print "test_ApplyFunctionDataSet" |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
353 raise NotImplementedError() |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
354 def test_CacheDataSet(): |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
355 print "test_CacheDataSet" |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
356 raise NotImplementedError() |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
357 def test_FieldsSubsetDataSet(): |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
358 print "test_FieldsSubsetDataSet" |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
359 raise NotImplementedError() |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
360 def test_DataSetFields(): |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
361 print "test_DataSetFields" |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
362 raise NotImplementedError() |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
363 def test_MinibatchDataSet(): |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
364 print "test_MinibatchDataSet" |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
365 raise NotImplementedError() |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
366 def test_HStackedDataSet(): |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
367 print "test_HStackedDataSet" |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
368 raise NotImplementedError() |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
369 def test_VStackedDataSet(): |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
370 print "test_VStackedDataSet" |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
371 raise NotImplementedError() |
|
84
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
372 def test_ArrayFieldsDataSet(): |
|
aa9e786ee849
added function have_raised that evaluate the string in parameter and return true if the function have raised an exception
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
82
diff
changeset
|
373 print "test_ArrayFieldsDataSet" |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
374 raise NotImplementedError() |
|
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
375 |
|
74
b4159cbdc06b
Fixed errors raised by test_dataset
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
66
diff
changeset
|
376 test1() |
|
81
4b0859606d05
Added test for ArrayDataSet and LookUpList
Frederic Bastien <bastienf@iro.umontreal.ca>
parents:
66
diff
changeset
|
377 test_LookupList() |
|
65
d48eba49a2f4
fixed the infinite loop
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
64
diff
changeset
|
378 test_ArrayDataSet() |
|
64
863da25a60f1
trying to fix infinite loop
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
63
diff
changeset
|
379 |