Mercurial > pylearn

annotate algorithms/logistic_regression.py @ 497:a272f4cbf004

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
'x' => 'input' 'y' => 'output'
author Joseph Turian <turian@gmail.com>
date Tue, 28 Oct 2008 12:25:04 -0400
parents 7560817a07e8
children a419edf4e06c
rev   line source
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
1 import theano
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
2 from theano import tensor as T
495
7560817a07e8 nnet_ops => nnet
Joseph Turian <turian@gmail.com>
parents: 491
diff changeset
3 from theano.tensor import nnet
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
4 from theano.compile import module
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
5 from theano import printing, pprint
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
6 from theano import compile
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
7
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
8 import numpy as N
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
9
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
10
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
11 class Module_Nclass(module.FancyModule):
473
31acd42b2b0b __instance_type__ -> InstanceType
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 472
diff changeset
12 class InstanceType(module.FancyModuleInstance):
476
8fcd0f3d9a17 added a few algorithms
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 473
diff changeset
13 def initialize(self, n_in, n_out, rng=N.random):
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
14 #self.component is the LogisticRegressionTemplate instance that built this guy.
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
15
472
69c800af1370 changed weight initialization for logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 470
diff changeset
16 self.w = N.zeros((n_in, n_out))
69c800af1370 changed weight initialization for logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 470
diff changeset
17 self.b = N.zeros(n_out)
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
18 self.lr = 0.01
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
19 self.__hide__ = ['params']
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
20
497
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
21 def __init__(self, input=None, targ=None, w=None, b=None, lr=None, regularize=False):
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
22 super(Module_Nclass, self).__init__() #boilerplate
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
23
497
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
24 self.input = input if input is not None else T.matrix('input')
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
25 self.targ = targ if targ is not None else T.lvector()
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
26
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
27 self.w = w if w is not None else module.Member(T.dmatrix())
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
28 self.b = b if b is not None else module.Member(T.dvector())
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
29 self.lr = lr if lr is not None else module.Member(T.dscalar())
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
30
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
31 self.params = [p for p in [self.w, self.b] if p.owner is None]
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
32
497
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
33 xent, output = nnet.crossentropy_softmax_1hot(
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
34 T.dot(self.input, self.w) + self.b, self.targ)
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
35 sum_xent = T.sum(xent)
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
36
497
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
37 self.output = output
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
38 self.sum_xent = sum_xent
491
180d125dc7e2 made logistic_regression classes compatible with stacker
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 476
diff changeset
39 self.cost = sum_xent
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
40
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
41 #define the apply method
497
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
42 self.pred = T.argmax(T.dot(self.input, self.w) + self.b, axis=1)
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
43 self.apply = module.Method([self.input], self.pred)
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
44
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
45 if self.params:
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
46 gparams = T.grad(sum_xent, self.params)
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
47
497
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
48 self.update = module.Method([self.input, self.targ], sum_xent,
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
49 updates = dict((p, p - self.lr * g) for p, g in zip(self.params, gparams)))
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
50
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
51 class Module(module.FancyModule):
473
31acd42b2b0b __instance_type__ -> InstanceType
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 472
diff changeset
52 class InstanceType(module.FancyModuleInstance):
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
53 def initialize(self, n_in):
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
54 #self.component is the LogisticRegressionTemplate instance that built this guy.
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
55
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
56 self.w = N.random.randn(n_in,1)
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
57 self.b = N.random.randn(1)
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
58 self.lr = 0.01
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
59 self.__hide__ = ['params']
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
60
497
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
61 def __init__(self, input=None, targ=None, w=None, b=None, lr=None, regularize=False):
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
62 super(Module, self).__init__() #boilerplate
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
63
497
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
64 self.input = input if input is not None else T.matrix('input')
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
65 self.targ = targ if targ is not None else T.lcol()
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
66
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
67 self.w = w if w is not None else module.Member(T.dmatrix())
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
68 self.b = b if b is not None else module.Member(T.dvector())
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
69 self.lr = lr if lr is not None else module.Member(T.dscalar())
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
70
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
71 self.params = [p for p in [self.w, self.b] if p.owner is None]
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
72
497
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
73 output = nnet.sigmoid(T.dot(self.x, self.w))
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
74 xent = -self.targ * T.log(output) - (1.0 - self.targ) * T.log(1.0 - output)
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
75 sum_xent = T.sum(xent)
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
76
497
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
77 self.output = output
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
78 self.xent = xent
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
79 self.sum_xent = sum_xent
491
180d125dc7e2 made logistic_regression classes compatible with stacker
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 476
diff changeset
80 self.cost = sum_xent
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
81
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
82 #define the apply method
497
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
83 self.pred = (T.dot(self.input, self.w) + self.b) > 0.0
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
84 self.apply = module.Method([self.input], self.pred)
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
85
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
86 #if this module has any internal parameters, define an update function for them
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
87 if self.params:
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
88 gparams = T.grad(sum_xent, self.params)
497
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
89 self.update = module.Method([self.input, self.targ], sum_xent,
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
90 updates = dict((p, p - self.lr * g) for p, g in zip(self.params, gparams)))
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
91
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
92
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
93
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
94 class Learner(object):
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
95 """TODO: Encapsulate the algorithm for finding an optimal regularization coefficient"""
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
96 pass
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
97