--- a/test_dataset.py	Mon May 12 15:08:18 2008 -0400
+++ b/test_dataset.py	Mon May 12 15:35:18 2008 -0400
@@ -3,7 +3,8 @@
 from math import *
 import numpy
 
-def have_raised(to_eval):
+def have_raised(to_eval, **var):
+    
     have_thrown = False
     try:
         eval(to_eval)
@@ -32,6 +33,301 @@
         print "var=",var
     print "take a slice and look at field y",ds[1:6:2]["y"]
 
+def test_iterate_over_examples(array,ds):
+#not in doc!!!
+    i=0
+    for example in range(len(ds)):
+        assert (ds[example]['x']==array[example][:3]).all()
+        assert ds[example]['y']==array[example][3]
+        assert (ds[example]['z']==array[example][[0,2]]).all()
+        i+=1
+    assert i==len(ds)
+    del example,i
+
+#     - for example in dataset:
+    i=0
+    for example in ds:
+        assert len(example)==3
+        assert (example['x']==array[i][:3]).all()
+        assert example['y']==array[i][3]
+        assert (example['z']==array[i][0:3:2]).all()
+        assert (numpy.append(example['x'],example['y'])==array[i]).all()
+        i+=1
+    assert i==len(ds)
+    del example,i
+
+#     - for val1,val2,... in dataset:
+    i=0
+    for x,y,z in ds:
+        assert (x==array[i][:3]).all()
+        assert y==array[i][3]
+        assert (z==array[i][0:3:2]).all()
+        assert (numpy.append(x,y)==array[i]).all()
+        i+=1
+    assert i==len(ds)
+    del x,y,z,i
+
+#     - for example in dataset(field1, field2,field3, ...):
+    i=0
+    for example in ds('x','y','z'):
+        assert len(example)==3
+        assert (example['x']==array[i][:3]).all()
+        assert example['y']==array[i][3]
+        assert (example['z']==array[i][0:3:2]).all()
+        assert (numpy.append(example['x'],example['y'])==array[i]).all()
+        i+=1
+    assert i==len(ds)
+    del example,i
+    i=0
+    for example in ds('y','x'):
+        assert len(example)==2
+        assert (example['x']==array[i][:3]).all()
+        assert example['y']==array[i][3]
+        assert (numpy.append(example['x'],example['y'])==array[i]).all()
+        i+=1
+    assert i==len(ds)
+    del example,i
+
+#     - for val1,val2,val3 in dataset(field1, field2,field3):
+    i=0
+    for x,y,z in ds('x','y','z'):
+        assert (x==array[i][:3]).all()
+        assert y==array[i][3]
+        assert (z==array[i][0:3:2]).all()
+        assert (numpy.append(x,y)==array[i]).all()
+        i+=1
+    assert i==len(ds)
+    del x,y,z,i
+    i=0
+    for y,x in ds('y','x',):
+        assert (x==array[i][:3]).all()
+        assert y==array[i][3]
+        assert (numpy.append(x,y)==array[i]).all()
+        i+=1
+    assert i==len(ds)
+    del x,y,i
+
+    def test_minibatch_size(minibatch,minibatch_size,len_ds,nb_field,nb_iter_finished):
+        ##full minibatch or the last minibatch
+        for idx in range(nb_field):
+            test_minibatch_field_size(minibatch[idx],minibatch_size,len_ds,nb_iter_finished)
+        del idx
+    def test_minibatch_field_size(minibatch_field,minibatch_size,len_ds,nb_iter_finished):
+        assert len(minibatch_field)==minibatch_size or ((nb_iter_finished*minibatch_size+len(minibatch_field))==len_ds and len(minibatch_field)<minibatch_size)
+
+#     - for minibatch in dataset.minibatches([field1, field2, ...],minibatch_size=N):
+    i=0
+    mi=0
+    m=ds.minibatches(['x','z'], minibatch_size=3)
+    assert isinstance(m,DataSet.MinibatchWrapAroundIterator)
+    for minibatch in m:
+        assert len(minibatch)==2
+        test_minibatch_size(minibatch,m.minibatch_size,len(ds),2,mi)
+        assert (minibatch[0][:,0:3:2]==minibatch[1]).all()
+        mi+=1
+        i+=len(minibatch[0])
+    assert i==len(ds)
+    assert mi==4
+    del minibatch,i,m,mi
+
+    i=0
+    mi=0
+    m=ds.minibatches(['x','y'], minibatch_size=3)
+    assert isinstance(m,DataSet.MinibatchWrapAroundIterator)
+    for minibatch in m:
+        assert len(minibatch)==2
+        test_minibatch_size(minibatch,m.minibatch_size,len(ds),2,mi)
+        mi+=1
+        for id in range(len(minibatch[0])):
+            assert (numpy.append(minibatch[0][id],minibatch[1][id])==array[i]).all()
+            i+=1
+    assert i==len(ds)
+    assert mi==4
+    del minibatch,i,id,m,mi
+
+#     - for mini1,mini2,mini3 in dataset.minibatches([field1, field2, field3], minibatch_size=N):
+    i=0
+    mi=0
+    m=ds.minibatches(['x','z'], minibatch_size=3)
+    assert isinstance(m,DataSet.MinibatchWrapAroundIterator)
+    for x,z in m:
+        test_minibatch_field_size(x,m.minibatch_size,len(ds),mi)
+        test_minibatch_field_size(z,m.minibatch_size,len(ds),mi)
+        assert (x[:,0:3:2]==z).all()
+        i+=len(x)
+        mi+=1
+    assert i==len(ds)
+    assert mi==4
+    del x,z,i,m,mi
+    i=0
+    mi=0
+    m=ds.minibatches(['x','y'], minibatch_size=3)
+    for x,y in m:
+        test_minibatch_field_size(x,m.minibatch_size,len(ds),mi)
+        test_minibatch_field_size(y,m.minibatch_size,len(ds),mi)
+        mi+=1
+        for id in range(len(x)):
+            assert (numpy.append(x[id],y[id])==array[i]).all()
+            i+=1
+    assert i==len(ds)
+    assert mi==4
+    del x,y,i,id,m,mi
+
+#not in doc
+    i=0
+    m=ds.minibatches(['x','y'],n_batches=1,minibatch_size=3,offset=4)
+    assert isinstance(m,DataSet.MinibatchWrapAroundIterator)
+    for x,y in m:
+        assert len(x)==3
+        assert len(y)==3
+        for id in range(3):
+            assert (numpy.append(x[id],y[id])==array[i+4]).all()
+            i+=1
+    assert i==3
+    del x,y,i,id,m
+
+    i=0
+    m=ds.minibatches(['x','y'],n_batches=2,minibatch_size=3,offset=4)
+    assert isinstance(m,DataSet.MinibatchWrapAroundIterator)
+    for x,y in m:
+        assert len(x)==3
+        assert len(y)==3
+        for id in range(3):
+            assert (numpy.append(x[id],y[id])==array[i+4]).all()
+            i+=1
+    assert i==6
+    del x,y,i,id,m
+
+    i=0
+    m=ds.minibatches(['x','y'],n_batches=20,minibatch_size=3,offset=4)
+    assert isinstance(m,DataSet.MinibatchWrapAroundIterator)
+    for x,y in m:
+        assert len(x)==3
+        assert len(y)==3
+        for id in range(3):
+            assert (numpy.append(x[id],y[id])==array[(i+4)%array.shape[0]]).all()
+            i+=1
+    assert i==m.n_batches*m.minibatch_size
+    del x,y,i,id
+
+
+def test_ds_iterator(array,iterator1,iterator2,iterator3):
+    l=len(iterator1)
+    i=0
+    for x,y in iterator1:
+        assert (x==array[i][:3]).all()
+        assert y==array[i][3]
+        assert (numpy.append(x,y)==array[i]).all()
+        i+=1
+    assert i==l
+    i=0
+    for y,z in iterator2:
+        assert y==array[i][3]
+        assert (z==array[i][0:3:2]).all()
+        i+=1
+    assert i==l
+    i=0
+    for x,y,z in iterator3:
+        assert (x==array[i][:3]).all()
+        assert y==array[i][3]
+        assert (z==array[i][0:3:2]).all()
+        assert (numpy.append(x,y)==array[i]).all()
+        i+=1
+    assert i==l
+
+def test_getitem(array,ds):
+    def test_ds(orig,ds,index):
+        i=0
+        assert len(ds)==len(index)
+        for x,z,y in ds('x','z','y'):
+            assert (orig[index[i]]['x']==array[index[i]][:3]).all()
+            assert (orig[index[i]]['x']==x).all()
+            assert orig[index[i]]['y']==array[index[i]][3]
+            assert orig[index[i]]['y']==y
+            assert (orig[index[i]]['z']==array[index[i]][0:3:2]).all()
+            assert (orig[index[i]]['z']==z).all()
+            i+=1
+        del i
+        ds[0]
+        if len(ds)>2:
+            ds[:1]
+            ds[1:1]
+            ds[1:1:1]
+        if len(ds)>5:
+            ds[[1,2,3]]
+        for x in ds:
+            pass
+
+#ds[:n] returns a dataset with the n first examples.
+    ds2=ds[:3]
+    assert isinstance(ds2,DataSet)
+    test_ds(ds,ds2,index=[0,1,2])
+    del ds2
+
+#ds[i1:i2:s]# returns a ds with the examples i1,i1+s,...i2-s.
+    ds2=ds[1:7:2]
+    assert isinstance(ds2,DataSet)
+    test_ds(ds,ds2,[1,3,5])
+    del ds2
+
+#ds[i]
+    ds2=ds[5]
+    assert isinstance(ds2,Example)
+    assert have_raised("var['ds']["+str(len(ds))+"]",ds=ds)  # index not defined
+    assert not have_raised("var['ds']["+str(len(ds)-1)+"]",ds=ds)
+    del ds2
+
+#ds[[i1,i2,...in]]# returns a ds with examples i1,i2,...in.
+    ds2=ds[[4,7,2,8]]
+    assert isinstance(ds2,DataSet)
+    test_ds(ds,ds2,[4,7,2,8])
+    del ds2
+
+#ds[fieldname]# an iterable over the values of the field fieldname across
+  #the ds (the iterable is obtained by default by calling valuesVStack
+  #over the values for individual examples).
+    assert have_raised("ds['h']")  # h is not defined...
+    assert have_raised("ds[['x']]")  # bad syntax
+    assert not have_raised("var['ds']['x']",ds=ds)
+    isinstance(ds['x'],DataSetFields)
+    ds2=ds['x']
+    assert len(ds['x'])==10
+    assert len(ds['y'])==10
+    assert len(ds['z'])==10
+    i=0
+    for example in ds['x']:
+        assert (example==array[i][:3]).all()
+        i+=1
+    i=0
+    for example in ds['y']:
+        assert (example==array[i][3]).all()
+        i+=1
+    i=0
+    for example in ds['z']:
+        assert (example==array[i,0:3:2]).all()
+        i+=1
+    del ds2,i
+
+#ds.<property># returns the value of a property associated with
+  #the name <property>. The following properties should be supported:
+  #    - 'description': a textual description or name for the ds
+  #    - 'fieldtypes': a list of types (one per field)
+
+#* ds1 | ds2 | ds3 == ds.hstack([ds1,ds2,ds3])#????
+    #hstack([ds('x','y'),ds('z')]
+    #hstack([ds('z','y'),ds('x')]
+    #assert have_thrown("hstack([ds('x'),ds('x')]")
+    #assert not have_thrown("hstack([ds('x'),ds('x')]")
+    #accept_nonunique_names
+    #assert have_thrown("hstack([ds('y','x'),ds('x')]")
+#        i=0
+#        for example in hstack([ds('x'),ds('y'),ds('z')]):
+#            example==ds[i]
+#            i+=1 
+#        del i,example
+#* ds1 & ds2 & ds3 == ds.vstack([ds1,ds2,ds3])#????
+
+
 def test_ArrayDataSet():
     #don't test stream
     #tested only with float value
@@ -39,313 +335,18 @@
     #don't test missing value
     #don't test with tuple
     #don't test proterties
-    def test_iterate_over_examples(array,ds):
-#not in doc!!!
-        i=0
-        for example in range(len(ds)):
-            assert (ds[example]['x']==a[example][:3]).all()
-            assert ds[example]['y']==a[example][3]
-            assert (ds[example]['z']==a[example][[0,2]]).all()
-            i+=1
-        assert i==len(ds)
-        del example,i
-
-#     - for example in dataset:
-        i=0
-        for example in ds:
-            assert len(example)==3
-            assert (example['x']==array[i][:3]).all()
-            assert example['y']==array[i][3]
-            assert (example['z']==array[i][0:3:2]).all()
-            assert (numpy.append(example['x'],example['y'])==array[i]).all()
-            i+=1
-        assert i==len(ds)
-        del example,i
-
-#     - for val1,val2,... in dataset:
-        i=0
-        for x,y,z in ds:
-            assert (x==array[i][:3]).all()
-            assert y==array[i][3]
-            assert (z==array[i][0:3:2]).all()
-            assert (numpy.append(x,y)==array[i]).all()
-            i+=1
-        assert i==len(ds)
-        del x,y,z,i
-
-#     - for example in dataset(field1, field2,field3, ...):
-        i=0
-        for example in ds('x','y','z'):
-            assert len(example)==3
-            assert (example['x']==array[i][:3]).all()
-            assert example['y']==array[i][3]
-            assert (example['z']==array[i][0:3:2]).all()
-            assert (numpy.append(example['x'],example['y'])==array[i]).all()
-            i+=1
-        assert i==len(ds)
-        del example,i
-        i=0
-        for example in ds('y','x'):
-            assert len(example)==2
-            assert (example['x']==array[i][:3]).all()
-            assert example['y']==array[i][3]
-            assert (numpy.append(example['x'],example['y'])==array[i]).all()
-            i+=1
-        assert i==len(ds)
-        del example,i
-
-#     - for val1,val2,val3 in dataset(field1, field2,field3):
-        i=0
-        for x,y,z in ds('x','y','z'):
-            assert (x==array[i][:3]).all()
-            assert y==array[i][3]
-            assert (z==array[i][0:3:2]).all()
-            assert (numpy.append(x,y)==array[i]).all()
-            i+=1
-        assert i==len(ds)
-        del x,y,z,i
-        i=0
-        for y,x in ds('y','x',):
-            assert (x==array[i][:3]).all()
-            assert y==array[i][3]
-            assert (numpy.append(x,y)==array[i]).all()
-            i+=1
-        assert i==len(ds)
-        del x,y,i
-
-        def test_minibatch_size(minibatch,minibatch_size,len_ds,nb_field,nb_iter_finished):
-            ##full minibatch or the last minibatch
-            for idx in range(nb_field):
-                test_minibatch_field_size(minibatch[idx],minibatch_size,len_ds,nb_iter_finished)
-            del idx
-        def test_minibatch_field_size(minibatch_field,minibatch_size,len_ds,nb_iter_finished):
-            assert len(minibatch_field)==minibatch_size or ((nb_iter_finished*minibatch_size+len(minibatch_field))==len_ds and len(minibatch_field)<minibatch_size)
-
-#     - for minibatch in dataset.minibatches([field1, field2, ...],minibatch_size=N):
-        i=0
-        mi=0
-        m=ds.minibatches(['x','z'], minibatch_size=3)
-        assert isinstance(m,DataSet.MinibatchWrapAroundIterator)
-        for minibatch in m:
-            assert len(minibatch)==2
-            test_minibatch_size(minibatch,m.minibatch_size,len(ds),2,mi)
-            assert (minibatch[0][:,0:3:2]==minibatch[1]).all()
-            mi+=1
-            i+=len(minibatch[0])
-        assert i==len(ds)
-        assert mi==4
-        del minibatch,i,m,mi
-
-        i=0
-        mi=0
-        m=ds.minibatches(['x','y'], minibatch_size=3)
-        assert isinstance(m,DataSet.MinibatchWrapAroundIterator)
-        for minibatch in m:
-            assert len(minibatch)==2
-            test_minibatch_size(minibatch,m.minibatch_size,len(ds),2,mi)
-            mi+=1
-            for id in range(len(minibatch[0])):
-                assert (numpy.append(minibatch[0][id],minibatch[1][id])==a[i]).all()
-                i+=1
-        assert i==len(ds)
-        assert mi==4
-        del minibatch,i,id,m,mi
-
-#     - for mini1,mini2,mini3 in dataset.minibatches([field1, field2, field3], minibatch_size=N):
-        i=0
-        mi=0
-        m=ds.minibatches(['x','z'], minibatch_size=3)
-        assert isinstance(m,DataSet.MinibatchWrapAroundIterator)
-        for x,z in m:
-            test_minibatch_field_size(x,m.minibatch_size,len(ds),mi)
-            test_minibatch_field_size(z,m.minibatch_size,len(ds),mi)
-            assert (x[:,0:3:2]==z).all()
-            i+=len(x)
-            mi+=1
-        assert i==len(ds)
-        assert mi==4
-        del x,z,i,m,mi
-        i=0
-        mi=0
-        m=ds.minibatches(['x','y'], minibatch_size=3)
-        for x,y in m:
-            test_minibatch_field_size(x,m.minibatch_size,len(ds),mi)
-            test_minibatch_field_size(y,m.minibatch_size,len(ds),mi)
-            mi+=1
-            for id in range(len(x)):
-                assert (numpy.append(x[id],y[id])==a[i]).all()
-                i+=1
-        assert i==len(ds)
-        assert mi==4
-        del x,y,i,id,m,mi
-
-#not in doc
-        i=0
-        m=ds.minibatches(['x','y'],n_batches=1,minibatch_size=3,offset=4)
-        assert isinstance(m,DataSet.MinibatchWrapAroundIterator)
-        for x,y in m:
-            assert len(x)==3
-            assert len(y)==3
-            for id in range(3):
-                assert (numpy.append(x[id],y[id])==a[i+4]).all()
-                i+=1
-        assert i==3
-        del x,y,i,id,m
-
-        i=0
-        m=ds.minibatches(['x','y'],n_batches=2,minibatch_size=3,offset=4)
-        assert isinstance(m,DataSet.MinibatchWrapAroundIterator)
-        for x,y in m:
-            assert len(x)==3
-            assert len(y)==3
-            for id in range(3):
-                assert (numpy.append(x[id],y[id])==a[i+4]).all()
-                i+=1
-        assert i==6
-        del x,y,i,id,m
-
-        i=0
-        m=ds.minibatches(['x','y'],n_batches=20,minibatch_size=3,offset=4)
-        assert isinstance(m,DataSet.MinibatchWrapAroundIterator)
-        for x,y in m:
-            assert len(x)==3
-            assert len(y)==3
-            for id in range(3):
-                assert (numpy.append(x[id],y[id])==a[(i+4)%a.shape[0]]).all()
-                i+=1
-        assert i==m.n_batches*m.minibatch_size
-        del x,y,i,id
-
-
-    def test_ds_iterator(array,iterator1,iterator2,iterator3):
-        i=0
-        for x,y in iterator1:
-            assert (x==array[i][:3]).all()
-            assert y==array[i][3]
-            assert (numpy.append(x,y)==array[i]).all()
-            i+=1
-        assert i==len(ds)
-        i=0
-        for y,z in iterator2:
-            assert y==array[i][3]
-            assert (z==array[i][0:3:2]).all()
-            i+=1
-        assert i==len(ds)
-        i=0
-        for x,y,z in iterator3:
-            assert (x==array[i][:3]).all()
-            assert y==array[i][3]
-            assert (z==array[i][0:3:2]).all()
-            assert (numpy.append(x,y)==array[i]).all()
-            i+=1
-        assert i==len(ds)
-
-    def test_getitem(array,ds):
-        
-        def test_ds(orig,ds,index):
-            i=0
-            assert len(ds)==len(index)
-            for x,z,y in ds('x','z','y'):
-                assert (orig[index[i]]['x']==array[index[i]][:3]).all()
-                assert (orig[index[i]]['x']==x).all()
-                assert orig[index[i]]['y']==array[index[i]][3]
-                assert orig[index[i]]['y']==y
-                assert (orig[index[i]]['z']==array[index[i]][0:3:2]).all()
-                assert (orig[index[i]]['z']==z).all()
-                i+=1
-            del i
-            ds[0]
-            if len(ds)>2:
-                ds[:1]
-                ds[1:1]
-                ds[1:1:1]
-            if len(ds)>5:
-                ds[[1,2,3]]
-            for x in ds:
-                pass
-
-    #ds[:n] returns a dataset with the n first examples.
-        ds2=ds[:3]
-        assert isinstance(ds2,DataSet)
-        test_ds(ds,ds2,index=[0,1,2])
-        del ds2
-
-    #ds[i1:i2:s]# returns a ds with the examples i1,i1+s,...i2-s.
-        ds2=ds[1:7:2]
-        assert isinstance(ds2,DataSet)
-        test_ds(ds,ds2,[1,3,5])
-        del ds2
-
-    #ds[i]
-        ds2=ds[5]
-        assert isinstance(ds2,Example)
-        assert have_raised("ds["+str(len(ds))+"]")  # index not defined
-        assert not have_raised("ds["+str(len(ds)-1)+"]")
-        del ds2
-        
-    #ds[[i1,i2,...in]]# returns a ds with examples i1,i2,...in.
-        ds2=ds[[4,7,2,8]]
-        assert isinstance(ds2,DataSet)
-        test_ds(ds,ds2,[4,7,2,8])
-        del ds2
-
-    #ds[fieldname]# an iterable over the values of the field fieldname across
-      #the ds (the iterable is obtained by default by calling valuesVStack
-      #over the values for individual examples).
-        assert have_raised("ds['h']")  # h is not defined...
-        assert have_raised("ds[['x']]")  # bad syntax
-        assert not have_raised("ds['x']")
-        isinstance(ds['x'],DataSetFields)
-        ds2=ds['x']
-        assert len(ds['x'])==10
-        assert len(ds['y'])==10
-        assert len(ds['z'])==10
-        i=0
-        for example in ds['x']:
-            assert (example==a[i][:3]).all()
-            i+=1
-        i=0
-        for example in ds['y']:
-            assert (example==a[i][3]).all()
-            i+=1
-        i=0
-        for example in ds['z']:
-            assert (example==a[i,0:3:2]).all()
-            i+=1
-        del ds2,i
-
-    #ds.<property># returns the value of a property associated with
-      #the name <property>. The following properties should be supported:
-      #    - 'description': a textual description or name for the ds
-      #    - 'fieldtypes': a list of types (one per field)
-
-    #* ds1 | ds2 | ds3 == ds.hstack([ds1,ds2,ds3])#????
-        #hstack([ds('x','y'),ds('z')]
-        #hstack([ds('z','y'),ds('x')]
-        #assert have_thrown("hstack([ds('x'),ds('x')]")
-        #assert not have_thrown("hstack([ds('x'),ds('x')]")
-        #accept_nonunique_names
-        #assert have_thrown("hstack([ds('y','x'),ds('x')]")
-#        i=0
-#        for example in hstack([ds('x'),ds('y'),ds('z')]):
-#            example==ds[i]
-#            i+=1 
-#        del i,example
-    #* ds1 & ds2 & ds3 == ds.vstack([ds1,ds2,ds3])#????
-
-
     print "test_ArrayDataSet"
-    a = numpy.random.rand(10,4)
-    ds = ArrayDataSet(a,{'x':slice(3),'y':3,'z':[0,2]})###???tuple not tested
-    ds = ArrayDataSet(a,LookupList(['x','y','z'],[slice(3),3,[0,2]]))###???tuple not tested
+    a2 = numpy.random.rand(10,4)
+    ds = ArrayDataSet(a2,{'x':slice(3),'y':3,'z':[0,2]})###???tuple not tested
+    ds = ArrayDataSet(a2,LookupList(['x','y','z'],[slice(3),3,[0,2]]))###???tuple not tested
     assert len(ds)==10
     #assert ds==a? should this work?
 
-    test_iterate_over_examples(a, ds)
-    test_getitem(a, ds)
+    test_iterate_over_examples(a2, ds)
+    test_getitem(a2, ds)
 
 #     - for val1,val2,val3 in dataset(field1, field2,field3):
-    test_ds_iterator(a,ds('x','y'),ds('y','z'),ds('x','y','z'))
+    test_ds_iterator(a2,ds('x','y'),ds('y','z'),ds('x','y','z'))
 
 
     assert len(ds.fields())==3
@@ -380,7 +381,7 @@
     example2 = LookupList(['v','w'], ['a','b'])
     example3 = LookupList(['x','y','z','u','v','w'], [[1, 2, 3],2,3,0,'a','b'])
     assert example+example2==example3
-    assert have_raised("example+example")
+    assert have_raised("var['x']+var['x']",x=example)
 
 def test_ApplyFunctionDataSet():
     print "test_ApplyFunctionDataSet"