comparison dataset.py @ 144:ceae4de18981

Automated merge with ssh://p-omega1@lgcm.iro.umontreal.ca/tlearn
author Frederic Bastien <bastienf@iro.umontreal.ca>
date Mon, 12 May 2008 15:08:18 -0400
parents 0d8e721cc63c ad144fa72bf5
children 8173e196e291 9abd19af822e
comparison
equal deleted inserted replaced
135:0d8e721cc63c 144:ceae4de18981
768 self.iterators=iterators 768 self.iterators=iterators
769 def __iter__(self): 769 def __iter__(self):
770 return self 770 return self
771 def next(self): 771 def next(self):
772 # concatenate all the fields of the minibatches 772 # concatenate all the fields of the minibatches
773 return reduce(LookupList.__add__,[iterator.next() for iterator in self.iterators]) 773 l=LookupList()
774 for iter in self.iterators:
775 l.append_lookuplist(iter.next())
776 return l
774 777
775 assert self.hasFields(*fieldnames) 778 assert self.hasFields(*fieldnames)
776 # find out which underlying datasets are necessary to service the required fields 779 # find out which underlying datasets are necessary to service the required fields
777 # and construct corresponding minibatch iterators 780 # and construct corresponding minibatch iterators
778 if fieldnames: 781 if fieldnames and fieldnames!=self.fieldNames():
779 datasets=set([]) 782 datasets=set([])
780 fields_in_dataset=dict([(dataset,[]) for dataset in datasets]) 783 fields_in_dataset=dict([(dataset,[]) for dataset in datasets])
781 for fieldname in fieldnames: 784 for fieldname in fieldnames:
782 dataset=self.datasets[self.fieldnames2dataset[fieldname]] 785 dataset=self.datasets[self.fieldname2dataset[fieldname]]
783 datasets.add(dataset) 786 datasets.add(dataset)
784 fields_in_dataset[dataset].append(fieldname) 787 fields_in_dataset[dataset].append(fieldname)
785 datasets=list(datasets) 788 datasets=list(datasets)
786 iterators=[dataset.minibatches(fields_in_dataset[dataset],minibatch_size,n_batches,offset) 789 iterators=[dataset.minibatches(fields_in_dataset[dataset],minibatch_size,n_batches,offset)
787 for dataset in datasets] 790 for dataset in datasets]