Mercurial > pylearn
comparison dataset.py @ 142:ad144fa72bf5
Automated merge with ssh://p-omega1@lgcm.iro.umontreal.ca/tlearn
| author | Frederic Bastien <bastienf@iro.umontreal.ca> |
|---|---|
| date | Mon, 12 May 2008 14:15:16 -0400 |
| parents | 3f4e5c9bdc5e f5f235bebee4 |
| children | ceae4de18981 |
comparison
equal
deleted
inserted
replaced
| 134:3f4e5c9bdc5e | 142:ad144fa72bf5 |
|---|---|
| 768 self.iterators=iterators | 768 self.iterators=iterators |
| 769 def __iter__(self): | 769 def __iter__(self): |
| 770 return self | 770 return self |
| 771 def next(self): | 771 def next(self): |
| 772 # concatenate all the fields of the minibatches | 772 # concatenate all the fields of the minibatches |
| 773 return reduce(LookupList.__add__,[iterator.next() for iterator in self.iterators]) | 773 l=LookupList() |
| 774 for iter in self.iterators: | |
| 775 l.append_lookuplist(iter.next()) | |
| 776 return l | |
| 774 | 777 |
| 775 assert self.hasFields(*fieldnames) | 778 assert self.hasFields(*fieldnames) |
| 776 # find out which underlying datasets are necessary to service the required fields | 779 # find out which underlying datasets are necessary to service the required fields |
| 777 # and construct corresponding minibatch iterators | 780 # and construct corresponding minibatch iterators |
| 778 if fieldnames: | 781 if fieldnames and fieldnames!=self.fieldNames(): |
| 779 datasets=set([]) | 782 datasets=set([]) |
| 780 fields_in_dataset=dict([(dataset,[]) for dataset in datasets]) | 783 fields_in_dataset=dict([(dataset,[]) for dataset in datasets]) |
| 781 for fieldname in fieldnames: | 784 for fieldname in fieldnames: |
| 782 dataset=self.datasets[self.fieldnames2dataset[fieldname]] | 785 dataset=self.datasets[self.fieldname2dataset[fieldname]] |
| 783 datasets.add(dataset) | 786 datasets.add(dataset) |
| 784 fields_in_dataset[dataset].append(fieldname) | 787 fields_in_dataset[dataset].append(fieldname) |
| 785 datasets=list(datasets) | 788 datasets=list(datasets) |
| 786 iterators=[dataset.minibatches(fields_in_dataset[dataset],minibatch_size,n_batches,offset) | 789 iterators=[dataset.minibatches(fields_in_dataset[dataset],minibatch_size,n_batches,offset) |
| 787 for dataset in datasets] | 790 for dataset in datasets] |