Mercurial > ift6266

comparison datasets/gzpklfile.py @ 257:966272e7f14b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Make the datasets lazy-loading and add a maxsize parameter.
author Arnaud Bergeron <abergeron@gmail.com>
date Tue, 16 Mar 2010 18:51:27 -0400
parents 4cfd0eb438af
children c2fae7b96769
comparison
equal deleted inserted replaced
248:7e6fecabb656 257:966272e7f14b
17 res = self.ary[self.pos:self.pos+num] 17 res = self.ary[self.pos:self.pos+num]
18 self.pos += num 18 self.pos += num
19 return res 19 return res
20 20
21 class GzpklDataSet(DataSet): 21 class GzpklDataSet(DataSet):
22 def __init__(self, fname): 22 def __init__(self, fname, maxsize):
23 self._fname = fname 23 self._fname = fname
24 self.maxsize = maxsize
24 self._train = 0 25 self._train = 0
25 self._valid = 1 26 self._valid = 1
26 self._test = 2 27 self._test = 2
27 28
28 def _load(self): 29 def _load(self):
33 f.close() 34 f.close()
34 35
35 def _return_it(self, batchsz, bufsz, id): 36 def _return_it(self, batchsz, bufsz, id):
36 if not hasattr(self, 'datas'): 37 if not hasattr(self, 'datas'):
37 self._load() 38 self._load()
38 return izip(DataIterator([ArrayFile(self.datas[id][0])], batchsz, bufsz), 39 return izip(DataIterator([ArrayFile(self.datas[id][0][:maxsize])], batchsz, bufsz),
39 DataIterator([ArrayFile(self.datas[id][1])], batchsz, bufsz)) 40 DataIterator([ArrayFile(self.datas[id][1][:maxsize])], batchsz, bufsz))