pylearn: dataset.py comparison

comparison dataset.py @ 26:672fe4b23032

Fixed dataset errors so that _test_dataset.py works again.

author	bengioy@grenat.iro.umontreal.ca
date	Fri, 11 Apr 2008 11:14:54 -0400
parents	526e192b0699
children	541a273bc89f

comparison

equal deleted inserted replaced

-:526e192b0699
+:672fe4b23032
 from lookup_list import LookupList
 Example = LookupList
+import copy
 class AbstractFunction (Exception): """Derived class must override this function"""
 class DataSet(object):
 """A virtual base class for datasets.
 any other object that supports integer indexing and slicing.
 """
 raise AbstractFunction()
-def hasFields(*fieldnames):
+def hasFields(self,*fieldnames):
 """
 Return true if the given field name (or field names, if multiple arguments are
 given) is recognized by the DataSet (i.e. can be used as a field name in one
 of the iterators).
 """
 raise AbstractFunction()
-def merge_fields(*specifications):
+def merge_fields(self,*specifications):
 """
 Return a new dataset that maps old fields (of self) to new fields (of the returned
 dataset). The minimal syntax that should be supported is the following:
 new_field_specifications = [new_field_spec1, new_field_spec2, ...]
 new_field_spec = ([old_field1, old_field2, ...], new_field)
 support additional indexing schemes within each field (e.g. column slice
 of a matrix-like field).
 """
 raise AbstractFunction()
-def merge_field_values(*field_value_pairs)
+def merge_field_values(self,*field_value_pairs):
 """
 Return the value that corresponds to merging the values of several fields,
 given as arguments (field_name, field_value) pairs with self.hasField(field_name).
 This may be used by implementations of merge_fields.
 Raise a ValueError if the operation is not possible.
 """
 fieldnames,fieldvalues = zip(*field_value_pairs)
 raise ValueError("Unable to merge values of these fields:"+repr(fieldnames))
-def examples2minibatch(examples):
+def examples2minibatch(self,examples):
 """
 Combine a list of Examples into a minibatch. A minibatch is an Example whose fields
 are iterable over the examples of the minibatch.
 """
 raise AbstractFunction()
-def rename(rename_dict):
+def rename(self,rename_dict):
 """
 Return a new dataset that renames fields, using a dictionnary that maps old field
 names to new field names. The only fields visible by the returned dataset are those
 whose names are keys of the rename_dict.
 """
-return RenamingDataSet(self,rename_dict)
+self_class = self.__class__
+class SelfRenamingDataSet(RenamingDataSet,self_class):
-def applyFunction(function, input_fields, output_fields, copy_inputs=True, accept_minibatches=True, cache=True):
+pass
+self.__class__ = SelfRenamingDataSet
+# set the rename_dict and src fields
+SelfRenamingDataSet.__init__(self,self,rename_dict)
+return self
+def applyFunction(self,function, input_fields, output_fields, copy_inputs=True, accept_minibatches=True, cache=True):
 """
 Return a dataset that contains as fields the results of applying
 the given function (example-wise) to the specified input_fields. The
 function should return a sequence whose elements will be stored in
 fields whose names are given in the output_fields list. If copy_inputs
 are cached (to avoid recomputation if the same examples are again
 requested).
 """
 return ApplyFunctionDataSet(function, input_fields, output_fields, copy_inputs, accept_minibatches, cache)
-class RenamingDataSet(DataSet):
+class FiniteLengthDataSet(DataSet):
+"""
+Virtual interface for datasets that have a finite length (number of examples),
+and thus recognize a len(dataset) call.
+"""
+def __init__(self):
+DataSet.__init__(self)
+def __len__(self):
+"""len(dataset) returns the number of examples in the dataset."""
+raise AbstractFunction()
+class SliceableDataSet(DataSet):
+"""
+Virtual interface, a subclass of DataSet for datasets which are sliceable
+and whose individual elements can be accessed, generally respecting the
+python semantics for [spec], where spec is either a non-negative integer
+(for selecting one example), or a python slice (for selecting a sub-dataset
+comprising the specified examples). This is useful for obtaining
+sub-datasets, e.g. for splitting a dataset into training and test sets.
+"""
+def __init__(self):
+DataSet.__init__(self)
+def minibatches(self,
+fieldnames = DataSet.minibatches_fieldnames,
+minibatch_size = DataSet.minibatches_minibatch_size,
+n_batches = DataSet.minibatches_n_batches):
+"""
+If the n_batches is empty, we want to see all the examples possible
+for the given minibatch_size (possibly missing a few at the end of the dataset).
+"""
+# substitute the defaults:
+if n_batches is None: n_batches = len(self) / minibatch_size
+return DataSet.Iterator(self, fieldnames, minibatch_size, n_batches)
+def __getitem__(self,i):
+"""dataset[i] returns the (i+1)-th example of the dataset."""
+raise AbstractFunction()
+def __getslice__(self,*slice_args):
+"""dataset[i:j] returns the subdataset with examples i,i+1,...,j-1."""
+raise AbstractFunction()
+class FiniteWidthDataSet(DataSet):
+"""
+Virtual interface for datasets that have a finite width (number of fields),
+and thus return a list of fieldNames.
+"""
+def __init__(self):
+DataSet.__init__(self)
+def hasFields(self,*fields):
+has_fields=True
+fieldnames = self.fieldNames()
+for name in fields:
+if name not in fieldnames:
+has_fields=False
+return has_fields
+def fieldNames(self):
+"""Return the list of field names that are supported by the iterators,
+and for which hasFields(fieldname) would return True."""
+raise AbstractFunction()
+class RenamingDataSet(FiniteWidthDataSet):
 """A DataSet that wraps another one, and makes it look like the field names
 are different
 Renaming is done by a dictionary that maps new names to the old ones used in
 self.src.
 def __init__(self, src, rename_dct):
 DataSet.__init__(self)
 self.src = src
 self.rename_dct = copy.copy(rename_dct)
+def fieldNames(self):
+return self.rename_dct.keys()
 def minibatches(self,
 fieldnames = DataSet.minibatches_fieldnames,
 minibatch_size = DataSet.minibatches_minibatch_size,
 n_batches = DataSet.minibatches_n_batches):
 dct = self.rename_dct
 new_fieldnames = [dct.get(f, f) for f in fieldnames]
 return self.src.minibatches(new_fieldnames, minibatches_size, n_batches)
-class FiniteLengthDataSet(DataSet):
-"""
-Virtual interface for datasets that have a finite length (number of examples),
-and thus recognize a len(dataset) call.
-"""
-def __init__(self):
-DataSet.__init__(self)
-def __len__(self):
-"""len(dataset) returns the number of examples in the dataset."""
-raise AbstractFunction()
-class SliceableDataSet(DataSet):
-"""
-Virtual interface, a subclass of DataSet for datasets which are sliceable
-and whose individual elements can be accessed, generally respecting the
-python semantics for [spec], where spec is either a non-negative integer
-(for selecting one example), or a python slice (for selecting a sub-dataset
-comprising the specified examples). This is useful for obtaining
-sub-datasets, e.g. for splitting a dataset into training and test sets.
-"""
-def __init__(self):
-DataSet.__init__(self)
-def minibatches(self,
-fieldnames = DataSet.minibatches_fieldnames,
-minibatch_size = DataSet.minibatches_minibatch_size,
-n_batches = DataSet.minibatches_n_batches):
-"""
-If the n_batches is empty, we want to see all the examples possible
-for the given minibatch_size (possibly missing a few at the end of the dataset).
-"""
-# substitute the defaults:
-if n_batches is None: n_batches = len(self) / minibatch_size
-return DataSet.Iterator(self, fieldnames, minibatch_size, n_batches)
-def __getitem__(self,i):
-"""dataset[i] returns the (i+1)-th example of the dataset."""
-raise AbstractFunction()
-def __getslice__(self,*slice_args):
-"""dataset[i:j] returns the subdataset with examples i,i+1,...,j-1."""
-raise AbstractFunction()
-class FiniteWidthDataSet(DataSet):
-"""
-Virtual interface for datasets that have a finite width (number of fields),
-and thus return a list of fieldNames.
-"""
-def __init__(self):
-DataSet.__init__(self)
-def hasFields(*fieldnames):
-has_fields=True
-for fieldname in fieldnames:
-if fieldname not in self.fields.keys():
-has_fields=False
-return has_fields
-def fieldNames(self):
-"""Return the list of field names that are supported by the iterators,
-and for which hasFields(fieldname) would return True."""
-raise AbstractFunction()
 # we may want ArrayDataSet defined in another python file
 import numpy
 # copy the field here
 result[:,slice(c,c+slice_width)]=self.data[:,field_slice]
 c+=slice_width
 return result
-def rename(*new_field_specifications):
-"""
-Return a new dataset that maps old fields (of self) to new fields (of the returned
-dataset). The minimal syntax that should be supported is the following:
-new_field_specifications = [new_field_spec1, new_field_spec2, ...]
-new_field_spec = ([old_field1, old_field2, ...], new_field)
-In general both old_field and new_field should be strings, but some datasets may also
-support additional indexing schemes within each field (e.g. column slice
-of a matrix-like field).
-"""
-# if all old fields of each spec are
-raise NotImplementedError()
 class ApplyFunctionDataSet(DataSet):
 """
 A dataset that contains as fields the results of applying
 a given function (example-wise) to specified input_fields of a source
 dataset. The function should return a sequence whose elements will be stored in
 # and apply the function to it, and transpose into a list of examples (field values, actually)
 self.cached_examples = zip(*Example(output_fields,function(*inputs)))
 else:
 # compute a list with one tuple per example, with the function outputs
 self.cached_examples = [ function(input) for input in src.zip(input_fields) ]
-else if cache:
+elif cache:
 # maybe a fixed-size array kind of structure would be more efficient than a list
 # in the case where src is FiniteDataSet. -YB
 self.cached_examples = []
 def minibatches(self,

Mercurial > pylearn

comparison dataset.py @ 26:672fe4b23032