Mercurial > pylearn
annotate pylearn/algorithms/logistic_regression.py @ 599:bd777e960c7c
misc small changes
author | James Bergstra <bergstrj@iro.umontreal.ca> |
---|---|
date | Tue, 13 Jan 2009 20:08:06 -0500 |
parents | 3f9ec536f2c1 |
children | e242c12eb30d |
rev | line source |
---|---|
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
1 import sys, copy |
470
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
2 import theano |
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
3 from theano import tensor as T |
495 | 4 from theano.tensor import nnet |
470
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
5 from theano.compile import module |
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
6 from theano import printing, pprint |
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
7 from theano import compile |
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
8 |
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
9 import numpy as N |
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
10 |
599
bd777e960c7c
misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
591
diff
changeset
|
11 class LogRegN(module.FancyModule): |
bd777e960c7c
misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
591
diff
changeset
|
12 """ |
bd777e960c7c
misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
591
diff
changeset
|
13 A symbolic module for performing N-class logistic regression. |
bd777e960c7c
misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
591
diff
changeset
|
14 |
bd777e960c7c
misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
591
diff
changeset
|
15 Notable variables |
bd777e960c7c
misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
591
diff
changeset
|
16 ----------------- |
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
17 |
599
bd777e960c7c
misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
591
diff
changeset
|
18 self.input |
bd777e960c7c
misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
591
diff
changeset
|
19 self.target |
bd777e960c7c
misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
591
diff
changeset
|
20 self.softmax |
bd777e960c7c
misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
591
diff
changeset
|
21 self.argmax |
bd777e960c7c
misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
591
diff
changeset
|
22 self.regularized_cost |
bd777e960c7c
misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
591
diff
changeset
|
23 self.unregularized_cost |
bd777e960c7c
misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
591
diff
changeset
|
24 """ |
499
a419edf4e06c
removed unpicklable nested classes in logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
497
diff
changeset
|
25 |
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
26 def __init__(self, |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
27 n_in=None, n_out=None, |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
28 input=None, target=None, |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
29 w=None, b=None, |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
30 l2=None, l1=None): |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
31 super(LogRegN, self).__init__() #boilerplate |
470
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
32 |
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
33 self.n_in = n_in |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
34 self.n_out = n_out |
470
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
35 |
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
36 self.input = input if input is not None else T.matrix() |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
37 self.target = target if target is not None else T.lvector() |
470
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
38 |
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
39 self.w = w if w is not None else module.Member(T.dmatrix()) |
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
40 self.b = b if b is not None else module.Member(T.dvector()) |
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
41 |
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
42 #the params of the model are the ones we fit to the data |
470
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
43 self.params = [p for p in [self.w, self.b] if p.owner is None] |
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
44 |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
45 #the hyper-parameters of the model are not fit to the data |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
46 self.l2 = l2 if l2 is not None else module.Member(T.dscalar()) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
47 self.l1 = l1 if l1 is not None else module.Member(T.dscalar()) |
470
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
48 |
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
49 #here we actually build the model |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
50 self.linear_output = T.dot(self.input, self.w) + self.b |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
51 if 0: |
563
16f91ca016b1
* added NStages as a stopper (moved from hpu/conv)
desjagui@atchoum.iro.umontreal.ca
parents:
540
diff
changeset
|
52 # TODO: pending support for target being a sparse matrix |
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
53 self.softmax = nnet.softmax(self.linear_output) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
54 |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
55 self._max_pr, self.argmax = T.max_and_argmax(self.linear_output) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
56 self._xent = self.target * T.log(self.softmax) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
57 else: |
563
16f91ca016b1
* added NStages as a stopper (moved from hpu/conv)
desjagui@atchoum.iro.umontreal.ca
parents:
540
diff
changeset
|
58 # TODO: when above is fixed, remove this hack (need an argmax |
16f91ca016b1
* added NStages as a stopper (moved from hpu/conv)
desjagui@atchoum.iro.umontreal.ca
parents:
540
diff
changeset
|
59 # which is independent of targets) |
591 | 60 self.argmax_standalone = T.argmax(self.linear_output) |
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
61 (self._xent, self.softmax, self._max_pr, self.argmax) =\ |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
62 nnet.crossentropy_softmax_max_and_argmax_1hot( |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
63 self.linear_output, self.target) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
64 |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
65 self.unregularized_cost = T.sum(self._xent) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
66 self.l1_cost = self.l1 * T.sum(abs(self.w)) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
67 self.l2_cost = self.l2 * T.sum(self.w**2) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
68 self.regularized_cost = self.unregularized_cost + self.l1_cost + self.l2_cost |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
69 self._loss_zero_one = T.mean(T.neq(self.argmax, self.target)) |
503
c7ce66b4e8f4
Extensions to algorithms, and some cleanup (by defining linear_output result).
Joseph Turian <turian@gmail.com>
parents:
502
diff
changeset
|
70 |
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
71 # METHODS |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
72 if 0: #TODO: PENDING THE BETTER IMPLEMENTATION ABOVE |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
73 self.predict = module.Method([self.input], self.argmax) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
74 self.label_probs = module.Method([self.input], self.softmax) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
75 self.validate = module.Method([self.input, self.target], |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
76 [self._loss_zero_one, self.regularized_cost, self.unregularized_cost]) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
77 |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
78 def _instance_initialize(self, obj): |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
79 obj.w = N.zeros((self.n_in, self.n_out)) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
80 obj.b = N.zeros(self.n_out) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
81 obj.__pp_hide__ = ['params'] |
470
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
82 |
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
83 def logistic_regression(n_in, n_out, l1, l2, minimizer=None): |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
84 if n_out == 2: |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
85 raise NotImplementedError() |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
86 else: |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
87 rval = LogRegN(n_in=n_in, n_out=n_out, l1=l1, l2=l2) |
540 | 88 print 'RVAL input target', rval.input, rval.target |
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
89 rval.minimizer = minimizer([rval.input, rval.target], rval.regularized_cost, |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
90 rval.params) |
540 | 91 return rval.make(mode='FAST_RUN') |
499
a419edf4e06c
removed unpicklable nested classes in logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
497
diff
changeset
|
92 |
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
93 #TODO: grouping parameters by prefix does not play well with providing defaults. Think... |
540 | 94 #FIX : Guillaume suggested a convention: plugin handlers (dataset_factory, minimizer_factory, |
95 # etc.) should never provide default arguments for parameters, and accept **kwargs to catch | |
96 # irrelevant parameters. | |
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
97 class _fit_logreg_defaults(object): |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
98 minimizer_algo = 'dummy' |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
99 #minimizer_lr = 0.001 |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
100 dataset = 'MNIST_1k' |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
101 l1 = 0.0 |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
102 l2 = 0.0 |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
103 batchsize = 8 |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
104 verbose = 1 |
503
c7ce66b4e8f4
Extensions to algorithms, and some cleanup (by defining linear_output result).
Joseph Turian <turian@gmail.com>
parents:
502
diff
changeset
|
105 |
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
106 # consider pre-importing each file in algorithms, datasets (possibly with try/catch around each |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
107 # import so that this import failure is ignored) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
108 |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
109 def fit_logistic_regression_online(state, channel=lambda *args, **kwargs:None): |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
110 #use stochastic gradient descent |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
111 state.use_defaults(_fit_logreg_defaults) |
470
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
112 |
599
bd777e960c7c
misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
591
diff
changeset
|
113 dataset = make(state.dataset) |
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
114 train = dataset.train |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
115 valid = dataset.valid |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
116 test = dataset.test |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
117 |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
118 logreg = logistic_regression( |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
119 n_in=train.x.shape[1], |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
120 n_out=dataset.n_classes, |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
121 l2=state.l2, |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
122 l1=state.l1, |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
123 minimizer=make_minimizer(**state.subdict(prefix='minimizer_'))) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
124 |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
125 batchsize = state.batchsize |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
126 verbose = state.verbose |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
127 iter = [0] |
470
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
128 |
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
129 def step(): |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
130 # step by making a pass through the training set |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
131 for j in xrange(0,len(train.x)-batchsize+1,batchsize): |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
132 cost_j = logreg.minimizer.step_cost(train.x[j:j+batchsize], train.y[j:j+batchsize]) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
133 if verbose > 1: |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
134 print 'estimated train cost', cost_j |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
135 #TODO: consult iter[0] for periodic saving to cwd (model, minimizer, and stopper) |
503
c7ce66b4e8f4
Extensions to algorithms, and some cleanup (by defining linear_output result).
Joseph Turian <turian@gmail.com>
parents:
502
diff
changeset
|
136 |
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
137 def check(): |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
138 validate = logreg.validate(valid.x, valid.y) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
139 if verbose > 0: |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
140 print 'iter', iter[0], 'validate', validate |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
141 sys.stdout.flush() |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
142 iter[0] += 1 |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
143 return validate[0] |
470
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
144 |
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
145 def save(): |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
146 return copy.deepcopy(logreg) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
147 |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
148 stopper = make_stopper(**state.subdict(prefix='stopper_')) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
149 stopper.find_min(step, check, save) |
470
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
150 |
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
151 state.train_01, state.train_rcost, state.train_cost = logreg.validate(train.x, train.y) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
152 state.valid_01, state.valid_rcost, state.valid_cost = logreg.validate(valid.x, valid.y) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
153 state.test_01, state.test_rcost, state.test_cost = logreg.validate(test.x, test.y) |
470
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
154 |
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
155 state.n_train = len(train.y) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
156 state.n_valid = len(valid.y) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
157 state.n_test = len(test.y) |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
158 |
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
159 class LogReg2(module.FancyModule): |
497 | 160 def __init__(self, input=None, targ=None, w=None, b=None, lr=None, regularize=False): |
537
b054271b2504
new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
503
diff
changeset
|
161 super(LogReg2, self).__init__() #boilerplate |
470
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
162 |
519
b267a8000f92
Added theano.Member to Member variables which were passed in during initialization.
Joseph Turian <turian@iro.umontreal.ca>
parents:
503
diff
changeset
|
163 self.input = module.Member(input) if input is not None else T.matrix('input') |
b267a8000f92
Added theano.Member to Member variables which were passed in during initialization.
Joseph Turian <turian@iro.umontreal.ca>
parents:
503
diff
changeset
|
164 self.targ = module.Member(targ) if targ is not None else T.lcol() |
470
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
165 |
519
b267a8000f92
Added theano.Member to Member variables which were passed in during initialization.
Joseph Turian <turian@iro.umontreal.ca>
parents:
503
diff
changeset
|
166 self.w = module.Member(w) if w is not None else module.Member(T.dmatrix()) |
b267a8000f92
Added theano.Member to Member variables which were passed in during initialization.
Joseph Turian <turian@iro.umontreal.ca>
parents:
503
diff
changeset
|
167 self.b = module.Member(b) if b is not None else module.Member(T.dvector()) |
b267a8000f92
Added theano.Member to Member variables which were passed in during initialization.
Joseph Turian <turian@iro.umontreal.ca>
parents:
503
diff
changeset
|
168 self.lr = module.Member(lr) if lr is not None else module.Member(T.dscalar()) |
470
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
169 |
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
170 self.params = [p for p in [self.w, self.b] if p.owner is None] |
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
171 |
502 | 172 output = nnet.sigmoid(T.dot(self.x, self.w) + self.b) |
497 | 173 xent = -self.targ * T.log(output) - (1.0 - self.targ) * T.log(1.0 - output) |
470
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
174 sum_xent = T.sum(xent) |
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
175 |
497 | 176 self.output = output |
470
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
177 self.xent = xent |
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
178 self.sum_xent = sum_xent |
491
180d125dc7e2
made logistic_regression classes compatible with stacker
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
476
diff
changeset
|
179 self.cost = sum_xent |
470
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
180 |
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
181 #define the apply method |
497 | 182 self.pred = (T.dot(self.input, self.w) + self.b) > 0.0 |
183 self.apply = module.Method([self.input], self.pred) | |
470
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
184 |
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
185 #if this module has any internal parameters, define an update function for them |
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
186 if self.params: |
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
187 gparams = T.grad(sum_xent, self.params) |
497 | 188 self.update = module.Method([self.input, self.targ], sum_xent, |
470
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
189 updates = dict((p, p - self.lr * g) for p, g in zip(self.params, gparams))) |
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
190 |
bd937e845bbb
new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
191 |