pylearn: _test_dataset.py comparison

comparison _test_dataset.py @ 376:c9a89be5cb0a

Redesigning linear_regression

author	Yoshua Bengio <bengioy@iro.umontreal.ca>
date	Mon, 07 Jul 2008 10:08:35 -0400
parents	18702ceb2096
children	82da179d95b2

comparison

equal deleted inserted replaced

-:12ce29abf27d
+:c9a89be5cb0a
 #!/bin/env python
 from dataset import *
 from math import *
 import numpy, unittest, sys
-from misc import *
+#from misc import *
 from lookup_list import LookupList
 def have_raised(to_eval, **var):
 have_thrown = False
 try:
 assert len(minibatch_field)==minibatch_size or ((nb_iter_finished*minibatch_size+len(minibatch_field))==len_ds and len(minibatch_field)<minibatch_size)
 #     - for minibatch in dataset.minibatches([field1, field2, ...],minibatch_size=N):
 i=0
 mi=0
-m=ds.minibatches(['x','z'], minibatch_size=3)
+size=3
-assert isinstance(m,DataSet.MinibatchWrapAroundIterator)
+m=ds.minibatches(['x','z'], minibatch_size=size)
+assert hasattr(m,'__iter__')
 for minibatch in m:
-assert isinstance(minibatch,DataSetFields)
+assert isinstance(minibatch,LookupList)
 assert len(minibatch)==2
-test_minibatch_size(minibatch,m.minibatch_size,len(ds),2,mi)
+test_minibatch_size(minibatch,size,len(ds),2,mi)
 if type(ds)==ArrayDataSet:
 assert (minibatch[0][:,::2]==minibatch[1]).all()
 else:
 for j in xrange(len(minibatch[0])):
 (minibatch[0][j][::2]==minibatch[1][j]).all()
 mi+=1
 i+=len(minibatch[0])
-assert i==len(ds)
+assert i==(len(ds)/size)*size
-assert mi==4
+assert mi==(len(ds)/size)
-del minibatch,i,m,mi
+del minibatch,i,m,mi,size
 i=0
 mi=0
-m=ds.minibatches(['x','y'], minibatch_size=3)
+size=3
-assert isinstance(m,DataSet.MinibatchWrapAroundIterator)
+m=ds.minibatches(['x','y'], minibatch_size=size)
+assert hasattr(m,'__iter__')
 for minibatch in m:
+assert isinstance(minibatch,LookupList)
 assert len(minibatch)==2
-test_minibatch_size(minibatch,m.minibatch_size,len(ds),2,mi)
+test_minibatch_size(minibatch,size,len(ds),2,mi)
 mi+=1
 for id in range(len(minibatch[0])):
 assert (numpy.append(minibatch[0][id],minibatch[1][id])==array[i]).all()
 i+=1
-assert i==len(ds)
+assert i==(len(ds)/size)*size
-assert mi==4
+assert mi==(len(ds)/size)
-del minibatch,i,id,m,mi
+del minibatch,i,id,m,mi,size
 #     - for mini1,mini2,mini3 in dataset.minibatches([field1, field2, field3], minibatch_size=N):
 i=0
 mi=0
-m=ds.minibatches(['x','z'], minibatch_size=3)
+size=3
-assert isinstance(m,DataSet.MinibatchWrapAroundIterator)
+m=ds.minibatches(['x','z'], minibatch_size=size)
+assert hasattr(m,'__iter__')
 for x,z in m:
-test_minibatch_field_size(x,m.minibatch_size,len(ds),mi)
+test_minibatch_field_size(x,size,len(ds),mi)
-test_minibatch_field_size(z,m.minibatch_size,len(ds),mi)
+test_minibatch_field_size(z,size,len(ds),mi)
 for id in range(len(x)):
 assert (x[id][::2]==z[id]).all()
 i+=1
 mi+=1
-assert i==len(ds)
+assert i==(len(ds)/size)*size
-assert mi==4
+assert mi==(len(ds)/size)
-del x,z,i,m,mi
+del x,z,i,m,mi,size
 i=0
 mi=0
+size=3
 m=ds.minibatches(['x','y'], minibatch_size=3)
+assert hasattr(m,'__iter__')
 for x,y in m:
-test_minibatch_field_size(x,m.minibatch_size,len(ds),mi)
+assert len(x)==size
-test_minibatch_field_size(y,m.minibatch_size,len(ds),mi)
+assert len(y)==size
+test_minibatch_field_size(x,size,len(ds),mi)
+test_minibatch_field_size(y,size,len(ds),mi)
 mi+=1
 for id in range(len(x)):
 assert (numpy.append(x[id],y[id])==array[i]).all()
 i+=1
-assert i==len(ds)
+assert i==(len(ds)/size)*size
-assert mi==4
+assert mi==(len(ds)/size)
-del x,y,i,id,m,mi
+del x,y,i,id,m,mi,size
 #not in doc
 i=0
-m=ds.minibatches(['x','y'],n_batches=1,minibatch_size=3,offset=4)
+size=3
-assert isinstance(m,DataSet.MinibatchWrapAroundIterator)
+m=ds.minibatches(['x','y'],n_batches=1,minibatch_size=size,offset=4)
+assert hasattr(m,'__iter__')
 for x,y in m:
-assert len(x)==m.minibatch_size
+assert len(x)==size
-assert len(y)==m.minibatch_size
+assert len(y)==size
-for id in range(m.minibatch_size):
+for id in range(size):
 assert (numpy.append(x[id],y[id])==array[i+4]).all()
 i+=1
-assert i==m.n_batches*m.minibatch_size
+assert i==size
-del x,y,i,id,m
+del x,y,i,id,m,size
 i=0
-m=ds.minibatches(['x','y'],n_batches=2,minibatch_size=3,offset=4)
+size=3
-assert isinstance(m,DataSet.MinibatchWrapAroundIterator)
+m=ds.minibatches(['x','y'],n_batches=2,minibatch_size=size,offset=4)
+assert hasattr(m,'__iter__')
 for x,y in m:
-assert len(x)==m.minibatch_size
+assert len(x)==size
-assert len(y)==m.minibatch_size
+assert len(y)==size
-for id in range(m.minibatch_size):
+for id in range(size):
 assert (numpy.append(x[id],y[id])==array[i+4]).all()
 i+=1
-assert i==m.n_batches*m.minibatch_size
+assert i==2*size
-del x,y,i,id,m
+del x,y,i,id,m,size
 i=0
-m=ds.minibatches(['x','y'],n_batches=20,minibatch_size=3,offset=4)
+size=3
-assert isinstance(m,DataSet.MinibatchWrapAroundIterator)
+m=ds.minibatches(['x','y'],n_batches=20,minibatch_size=size,offset=4)
+assert hasattr(m,'__iter__')
 for x,y in m:
-assert len(x)==m.minibatch_size
+assert len(x)==size
-assert len(y)==m.minibatch_size
+assert len(y)==size
-for id in range(m.minibatch_size):
+for id in range(size):
 assert (numpy.append(x[id],y[id])==array[(i+4)%array.shape[0]]).all()
 i+=1
-assert i==m.n_batches*m.minibatch_size
+assert i==2*size # should not wrap
-del x,y,i,id
+del x,y,i,id,size
-assert not have_raised2(ds.minibatches,['x','y'],n_batches=1,minibatch_size=len(array)+1,offset=0)
+assert have_raised2(ds.minibatches,['x','y'],n_batches=1,minibatch_size=len(array)+1,offset=0)
 assert not have_raised2(ds.minibatches,['x','y'],n_batches=1,minibatch_size=len(array),offset=0)
 def test_ds_iterator(array,iterator1,iterator2,iterator3):
 l=len(iterator1)
 i=0
 assert i==l
 def test_getitem(array,ds):
 def test_ds(orig,ds,index):
 i=0
-assert len(ds)==len(index)
+assert isinstance(ds,LookupList)
-for x,z,y in ds('x','z','y'):
+assert len(ds)==3
-assert (orig[index[i]]['x']==array[index[i]][:3]).all()
+assert len(ds[0])==len(index)
-assert (orig[index[i]]['x']==x).all()
+#        for x,z,y in ds('x','z','y'):
-assert orig[index[i]]['y']==array[index[i]][3]
+for idx in index:
-assert (orig[index[i]]['y']==y).all() # why does it crash sometimes?
+assert (orig[idx]['x']==array[idx][:3]).all()
-assert (orig[index[i]]['z']==array[index[i]][0:3:2]).all()
+assert (orig[idx]['x']==ds['x'][i]).all()
-assert (orig[index[i]]['z']==z).all()
+assert orig[idx]['y']==array[idx][3]
+assert (orig[idx]['y']==ds['y'][i]).all() # why does it crash sometimes?
+assert (orig[idx]['z']==array[idx][0:3:2]).all()
+assert (orig[idx]['z']==ds['z'][i]).all()
 i+=1
 del i
 ds[0]
 if len(ds)>2:
 ds[:1]
 if len(ds)>5:
 ds[[1,2,3]]
 for x in ds:
 pass
-#ds[:n] returns a dataset with the n first examples.
+#ds[:n] returns a LookupList with the n first examples.
 ds2=ds[:3]
-assert isinstance(ds2,LookupList)
 test_ds(ds,ds2,index=[0,1,2])
 del ds2
-#ds[i1:i2:s]# returns a ds with the examples i1,i1+s,...i2-s.
+#ds[i:j] returns a LookupList with examples i,i+1,...,j-1.
-ds2=ds.subset[1:7:2]
+ds2=ds[1:3]
-assert isinstance(ds2,DataSet)
+test_ds(ds,ds2,index=[1,2])
+del ds2
+#ds[i1:i2:s] returns a LookupList with the examples i1,i1+s,...i2-s.
+ds2=ds[1:7:2]
 test_ds(ds,ds2,[1,3,5])
 del ds2
-#ds[i]
+#ds[i] returns the (i+1)-th example of the dataset.
 ds2=ds[5]
 assert isinstance(ds2,Example)
 assert have_raised("var['ds']["+str(len(ds))+"]",ds=ds)  # index not defined
 assert not have_raised("var['ds']["+str(len(ds)-1)+"]",ds=ds)
 del ds2
 #ds[[i1,i2,...in]]# returns a ds with examples i1,i2,...in.
-ds2=ds.subset[[4,7,2,8]]
+ds2=ds[[4,7,2,8]]
-assert isinstance(ds2,DataSet)
+#    assert isinstance(ds2,DataSet)
 test_ds(ds,ds2,[4,7,2,8])
 del ds2
 #ds.<property># returns the value of a property associated with
 #the name <property>. The following properties should be supported:
 #        for example in hstack([ds('x'),ds('y'),ds('z')]):
 #            example==ds[i]
 #            i+=1
 #        del i,example
 #* ds1 & ds2 & ds3 == ds.vstack([ds1,ds2,ds3])#????
+def test_subset(array,ds):
+def test_ds(orig,ds,index):
+i=0
+assert isinstance(ds2,DataSet)
+assert len(ds)==len(index)
+for x,z,y in ds('x','z','y'):
+assert (orig[index[i]]['x']==array[index[i]][:3]).all()
+assert (orig[index[i]]['x']==x).all()
+assert orig[index[i]]['y']==array[index[i]][3]
+assert orig[index[i]]['y']==y
+assert (orig[index[i]]['z']==array[index[i]][0:3:2]).all()
+assert (orig[index[i]]['z']==z).all()
+i+=1
+del i
+ds[0]
+if len(ds)>2:
+ds[:1]
+ds[1:1]
+ds[1:1:1]
+if len(ds)>5:
+ds[[1,2,3]]
+for x in ds:
+pass
+#ds[:n] returns a dataset with the n first examples.
+ds2=ds.subset[:3]
+test_ds(ds,ds2,index=[0,1,2])
+#    del ds2
+#ds[i1:i2:s]# returns a ds with the examples i1,i1+s,...i2-s.
+ds2=ds.subset[1:7:2]
+test_ds(ds,ds2,[1,3,5])
+#     del ds2
+# #ds[i]
+#     ds2=ds.subset[5]
+#     assert isinstance(ds2,Example)
+#     assert have_raised("var['ds']["+str(len(ds))+"]",ds=ds)  # index not defined
+#     assert not have_raised("var['ds']["+str(len(ds)-1)+"]",ds=ds)
+#     del ds2
+#ds[[i1,i2,...in]]# returns a ds with examples i1,i2,...in.
+ds2=ds.subset[[4,7,2,8]]
+test_ds(ds,ds2,[4,7,2,8])
+#     del ds2
+#ds.<property># returns the value of a property associated with
+#the name <property>. The following properties should be supported:
+#    - 'description': a textual description or name for the ds
+#    - 'fieldtypes': a list of types (one per field)
+#* ds1 | ds2 | ds3 == ds.hstack([ds1,ds2,ds3])#????
+#assert hstack([ds('x','y'),ds('z')])==ds
+#hstack([ds('z','y'),ds('x')])==ds
+assert have_raised2(hstack,[ds('x'),ds('x')])
+assert have_raised2(hstack,[ds('y','x'),ds('x')])
+assert not have_raised2(hstack,[ds('x'),ds('y')])
+#        i=0
+#        for example in hstack([ds('x'),ds('y'),ds('z')]):
+#            example==ds[i]
+#            i+=1
+#        del i,example
+#* ds1 & ds2 & ds3 == ds.vstack([ds1,ds2,ds3])#????
 def test_fields_fct(ds):
 #@todo, fill correctly
 assert len(ds.fields())==3
 i=0
 def test_all(array,ds):
 assert len(ds)==10
 test_iterate_over_examples(array, ds)
 test_overrides(ds)
 test_getitem(array, ds)
+test_subset(array, ds)
 test_ds_iterator(array,ds('x','y'),ds('y','z'),ds('x','y','z'))
 test_fields_fct(ds)
 class T_DataSet(unittest.TestCase):
 def test_FieldsSubsetDataSet(self):
 a = numpy.random.rand(10,4)
 ds = ArrayDataSet(a,Example(['x','y','z','w'],[slice(3),3,[0,2],0]))
 ds = FieldsSubsetDataSet(ds,['x','y','z'])
+test_all(a,ds)
+del a, ds
+def test_RenamedFieldsDataSet(self):
+a = numpy.random.rand(10,4)
+ds = ArrayDataSet(a,Example(['x1','y1','z1','w1'],[slice(3),3,[0,2],0]))
+ds = RenamedFieldsDataSet(ds,['x1','y1','z1'],['x','y','z'])
 test_all(a,ds)
 del a, ds
 for k in range(len(dsc)) :
 self.failUnless(numpy.all( dsc[k]('input')[0] == ds[k]('input')[0] ) , (dsc[k],ds[k]) )
 res = dsc[:]
 if __name__=='__main__':
-if len(sys.argv)==2:
+tests = []
-if sys.argv[1]=="--debug":
+debug=False
+if len(sys.argv)==1:
+unittest.main()
+else:
+assert sys.argv[1]=="--debug"
+for arg in sys.argv[2:]:
+tests.append(arg)
+if tests:
+unittest.TestSuite(map(T_DataSet, tests)).debug()
+else:
 module = __import__("_test_dataset")
 tests = unittest.TestLoader().loadTestsFromModule(module)
 tests.debug()
-print "bad argument: only --debug is accepted"
-elif len(sys.argv)==1:
-unittest.main()
-else:
-print "bad argument: only --debug is accepted"

Mercurial > pylearn

comparison _test_dataset.py @ 376:c9a89be5cb0a