# HG changeset patch # User bergstrj@iro.umontreal.ca # Date 1207964527 14400 # Node ID 2d6c49ec57494d40b80efd2c227397578354e2e0 # Parent bf0145fa73e8f807ecff408ddb0962de66d0b845# Parent 46c5c90019c2b8c87c9dd9120da21e1782a318eb merged diff -r bf0145fa73e8 -r 2d6c49ec5749 _test_dataset.py --- a/_test_dataset.py Fri Apr 11 21:41:09 2008 -0400 +++ b/_test_dataset.py Fri Apr 11 21:42:07 2008 -0400 @@ -48,16 +48,6 @@ a_y = a.y self.failUnless(numpy.all( a_y == arr[:,1:4])) - def test_asarray(self): - arr = numpy.random.rand(3,4) - a=ArrayDataSet(data=arr,fields={"x":slice(2),"y":slice(2,4)}) - a_arr = numpy.asarray(a) - self.failUnless(a_arr.shape[1] == 2 + 2) - self.failUnless(numpy.sum(numpy.square(a_arr-a.data))==0) - a=ArrayDataSet(data=arr,fields={"x":slice(2),"y":slice(1,4)}) - a_arr = numpy.asarray(a) - self.failUnless(a_arr.shape[1] == 2 + 3) - def test_minibatch_wraparound_even(self): arr = numpy.random.rand(10,4) arr2 = ArrayDataSet.Iterator.matcat(arr,arr) @@ -90,6 +80,17 @@ b=a.rename({'xx':'x','zz':'z'}) self.failUnless(b.hasFields('xx','zz') and not b.hasFields('x') and not b.hasFields('y')) +class T_applyfunctiondataset(unittest.TestCase): + def setUp(self): + numpy.random.seed(123456) + + def test_function(self): + n = numpy.random.rand(3,8) + a=ArrayDataSet(data=n,fields={"x":slice(2),"y":slice(1,4),"z":slice(4,6)}) + b=a.apply_function(lambda x,y: x+y,x+1, ['x','y'], ['x+y','x+1'], False,False,False) + print b.fieldNames() + print b('x+y') + if __name__ == '__main__': unittest.main() diff -r bf0145fa73e8 -r 2d6c49ec5749 dataset.py --- a/dataset.py Fri Apr 11 21:41:09 2008 -0400 +++ b/dataset.py Fri Apr 11 21:42:07 2008 -0400 @@ -37,8 +37,8 @@ Datasets of finite length should be sub-classes of FiniteLengthDataSet. - Datasets whose elements can be indexed and sub-datasets of consecutive - examples (i.e. slices) can be extracted from should be sub-classes of + Datasets whose elements can be indexed and whose sub-datasets (with a subset + of examples) can be extracted should be sub-classes of SliceableDataSet. Datasets with a finite number of fields should be sub-classes of @@ -150,6 +150,7 @@ of the iterators). """ raise AbstractFunction() + def merge_fields(self,*specifications): """ @@ -182,7 +183,7 @@ def rename(self,rename_dict): """ - Return a new dataset that renames fields, using a dictionnary that maps old field + Changes a dataset into one that renames fields, using a dictionnary that maps old field names to new field names. The only fields visible by the returned dataset are those whose names are keys of the rename_dict. """ @@ -194,9 +195,9 @@ SelfRenamingDataSet.__init__(self,self,rename_dict) return self - def applyFunction(self,function, input_fields, output_fields, copy_inputs=True, accept_minibatches=True, cache=True): + def apply_function(self,function, input_fields, output_fields, copy_inputs=True, accept_minibatches=True, cache=True): """ - Return a dataset that contains as fields the results of applying + Changes a dataset into one that contains as fields the results of applying the given function (example-wise) to the specified input_fields. The function should return a sequence whose elements will be stored in fields whose names are given in the output_fields list. If copy_inputs @@ -209,7 +210,13 @@ are cached (to avoid recomputation if the same examples are again requested). """ - return ApplyFunctionDataSet(function, input_fields, output_fields, copy_inputs, accept_minibatches, cache) + self_class = self.__class__ + class SelfApplyFunctionDataSet(ApplyFunctionDataSet,self_class): + pass + self.__class__ = SelfApplyFunctionDataSet + # set the required additional fields + ApplyFunctionDataSet.__init__(self,self,function, input_fields, output_fields, copy_inputs, accept_minibatches, cache) + return self class FiniteLengthDataSet(DataSet): @@ -223,15 +230,31 @@ def __len__(self): """len(dataset) returns the number of examples in the dataset.""" raise AbstractFunction() - + + def __call__(self,fieldname_or_fieldnames): + """ + Extract one or more fields. This may be an expensive operation when the + dataset is large. It is not the recommanded way to access individual values + (use the iterators instead). If the argument is a string fieldname, then the result + is a sequence (iterable object) of values for that field, for the whole dataset. If the + argument is a list of field names, then the result is a 'batch', i.e., an Example with keys + corresponding to the given field names and values being iterable objects over the + individual example values. + """ + if type(fieldname_or_fieldnames) is string: + minibatch = self.minibatches([fieldname_or_fieldnames],len(self)).next() + return minibatch[fieldname_or_fieldnames] + return self.minibatches(fieldname_or_fieldnames,len(self)).next() class SliceableDataSet(DataSet): """ Virtual interface, a subclass of DataSet for datasets which are sliceable and whose individual elements can be accessed, generally respecting the python semantics for [spec], where spec is either a non-negative integer - (for selecting one example), or a python slice (for selecting a sub-dataset - comprising the specified examples). This is useful for obtaining + (for selecting one example), a python slice(start,stop,step) for selecting a regular + sub-dataset comprising examples start,start+step,start+2*step,...,n (with n= 0 and fieldslice.stop <= cols + assert type(fieldslice) is int or isinstance(fieldslice,slice) or hasattr(fieldslice,"__iter__") + if hasattr(fieldslice,"__iter__"): # is a sequence + for i in fieldslice: + assert type(i) is int + elif isinstance(fieldslice,slice): + # make sure fieldslice.start and fieldslice.step are defined + start=fieldslice.start + step=fieldslice.step + if not start: + start=0 + if not step: + step=1 + if not fieldslice.start or not fieldslice.step: + fields[fieldname] = fieldslice = slice(start,fieldslice.stop,step) + # and coherent with the data array + assert fieldslice.start >= 0 and fieldslice.stop <= cols def minibatches(self, fieldnames = DataSet.minibatches_fieldnames, @@ -457,28 +494,6 @@ if n_batches is None: n_batches = len(self) / minibatch_size return ArrayDataSet.Iterator(self, fieldnames, minibatch_size, n_batches) - def __getattr__(self,fieldname): - """ - Return a numpy array with the content associated with the given field name. - If this is a one-example dataset, then a row, i.e., numpy array (of one less dimension - than the dataset itself) is returned. - """ - if len(self.data)==1: - return self.data[0,self.fields[fieldname]] - return self.data[:,self.fields[fieldname]] - - def __call__(self,*fieldnames): - """Return a sub-dataset containing only the given fieldnames as fields.""" - min_col=self.data.shape[1] - max_col=0 - for field_slice in self.fields.values(): - min_col=min(min_col,field_slice.start) - max_col=max(max_col,field_slice.stop) - new_fields=LookupList() - for fieldname,fieldslice in self.fields.items(): - new_fields[fieldname]=slice(fieldslice.start-min_col,fieldslice.stop-min_col,fieldslice.step) - return ArrayDataSet(self.data[:,min_col:max_col],fields=new_fields) - def fieldNames(self): """Return the list of field names that are supported by getattr and hasField.""" return self.fields.keys() @@ -489,8 +504,11 @@ def __getitem__(self,i): """ - dataset[i] returns the (i+1)-th Example of the dataset. If there are no fields - the result is just a numpy array (for the i-th row of the dataset data matrix). + dataset[i] returns the (i+1)-th Example of the dataset. + If there are no fields the result is just a numpy array (for the i-th row of the dataset data matrix). + dataset[i:j] returns the subdataset with examples i,i+1,...,j-1. + dataset[i:j:s] returns the subdataset with examples i,i+2,i+4...,j-2. + dataset[[i1,i2,..,in]] returns the subdataset with examples i1,i2,...,in. """ if self.fields: fieldnames,fieldslices=zip(*self.fields.items()) @@ -499,36 +517,34 @@ return self.data[i] def __getslice__(self,*args): - """dataset[i:j] returns the subdataset with examples i,i+1,...,j-1.""" + """ + dataset[i:j] returns the subdataset with examples i,i+1,...,j-1. + dataset[i:j:s] returns the subdataset with examples i,i+2,i+4...,j-2. + """ return ArrayDataSet(self.data.__getslice__(*args), fields=self.fields) - def __array__(self): - """Return a view of this dataset which is an numpy.ndarray (i.e. losing - the identity and name of fields within the dataset). - - Numpy uses this special function name to retrieve an ndarray view for - function such as numpy.sum, numpy.dot, numpy.asarray, etc. - - If this dataset has no fields, then we simply return self.data, - otherwise things are complicated. - - why do we want this behaviour when there are fields? (JB) - - for convenience and completeness (but maybe it would make - more sense to implement this through a 'field-merging' - dataset). (YB) + def indices_of_unique_columns_used(self): + """ + Return the unique indices of the columns actually used by the fields, and a boolean + that signals (if True) that used columns overlap. If they do then the + indices are not repeated in the result. """ - if not self.fields: - return self.data - # else, select subsets of columns mapped by the fields columns_used = numpy.zeros((self.data.shape[1]),dtype=bool) - overlapping_fields = False - n_columns = 0 + overlapping_columns = False for field_slice in self.fields.values(): - for c in xrange(field_slice.start,field_slice.stop,field_slice.step): - n_columns += 1 - if columns_used[c]: overlapping_fields=True - columns_used[c]=True - # try to figure out if we can map all the slices into one slice: - mappable_to_one_slice = not overlapping_fields + if sum(columns_used[field_slice])>0: overlapping_columns=True + columns_used[field_slice]=True + return [i for i,used in enumerate(columns_used) if used],overlapping_columns + + def slice_of_unique_columns_used(self): + """ + Return None if the indices_of_unique_columns_used do not form a slice. If they do, + return that slice. It means that the columns used can be extracted + from the data array without making a copy. If the fields overlap + but their unique columns used form a slice, still return that slice. + """ + columns_used,overlapping_columns = self.indices_of_columns_used() + mappable_to_one_slice = True if not overlapping_fields: start=0 while start