Mercurial > ift6266
diff datasets/gzpklfile.py @ 257:966272e7f14b
Make the datasets lazy-loading and add a maxsize parameter.
| author | Arnaud Bergeron <abergeron@gmail.com> |
|---|---|
| date | Tue, 16 Mar 2010 18:51:27 -0400 |
| parents | 4cfd0eb438af |
| children | c2fae7b96769 |
line wrap: on
line diff
--- a/datasets/gzpklfile.py Tue Mar 16 14:46:25 2010 -0400 +++ b/datasets/gzpklfile.py Tue Mar 16 18:51:27 2010 -0400 @@ -19,8 +19,9 @@ return res class GzpklDataSet(DataSet): - def __init__(self, fname): + def __init__(self, fname, maxsize): self._fname = fname + self.maxsize = maxsize self._train = 0 self._valid = 1 self._test = 2 @@ -35,5 +36,5 @@ def _return_it(self, batchsz, bufsz, id): if not hasattr(self, 'datas'): self._load() - return izip(DataIterator([ArrayFile(self.datas[id][0])], batchsz, bufsz), - DataIterator([ArrayFile(self.datas[id][1])], batchsz, bufsz)) + return izip(DataIterator([ArrayFile(self.datas[id][0][:maxsize])], batchsz, bufsz), + DataIterator([ArrayFile(self.datas[id][1][:maxsize])], batchsz, bufsz))