Mercurial > pylearn

annotate doc/v2_planning/shared_dataset.py @ 1169:3a1225034751

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
merge
author Pascal Lamblin <lamblinp@iro.umontreal.ca>
date Fri, 17 Sep 2010 14:29:49 -0400
parents c1943feada10
children
rev   line source
1117
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
1 import theano
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
2
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
3 # This is not final and may not even run for now. It is just to give
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
4 # a feeling of what the interface could look like.
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
5
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
6 def shared_dataset(dataset, mem_size):
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
7 if dataset.total_size > mem_size:
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
8 return OnlineDataset(dataset)
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
9 else:
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
10 return MemoryDataset(dataset)
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
11
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
12 class MemoryDataset(theano.Op):
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
13 def __init__(self, dataset):
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
14 self.input = theano.shared(dataset.input)
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
15 self.output = theano.shared(dataset.output)
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
16 self.batch_size = dataset.batch_size
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
17
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
18 def make_node(self, idx):
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
19 idx_ = theano.as_tensor_variable(idx)
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
20 return theano.Apply(self,
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
21 inputs = [idx_],
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
22 outputs = [self.input.type(),
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
23 self.output.type()])
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
24
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
25 def preform(self, node, inputs, output_storage):
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
26 idx, = inputs
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
27 self.output_storage[0][0] = self.input[idx*self.batch_size:(idx+1)*self.batch_size]
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
28 self.output_storage[1][0] = self.output[idx*self.batch_size:(idx+1)*self.batch_size]
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
29
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
30 class OnlineDataset(theano.Op):
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
31 def __init__(self, dataset):
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
32 self.dataset = dataset
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
33
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
34 def make_node(self, idx):
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
35 idx_ = theano.as_tensor_variable(idx)
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
36 return theano.Apply(self,
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
37 inputs = [idx_],
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
38 outputs = [theano.tensor.fmatrix(),
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
39 theano.tensor.fmatrix()])
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
40 # fix this so its not fmatrix(),
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
41 # but whatever the dataset outputs
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
42
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
43 def perform(self, node, inputs, output_storage):
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
44 idx, = inputs
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
45 b = self.dataset.get_batch(idx.value)
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
46 output_storage[0][0] = b.input
c1943feada10 Proposal for theano dataset wrapper. The details still have to be worked out.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
47 output_storage[1][0] = b.output