Mercurial > pylearn

annotate pylearn/algorithms/logistic_regression.py @ 1499:f82b80c841b2

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Remove deprecation warning.
author Frederic Bastien <nouiz@nouiz.org>
date Fri, 09 Sep 2011 10:49:54 -0400
parents bd7d540db70d
children
rev   line source
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
1 import sys, copy
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
2 import theano
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
3 from theano import tensor as T
495
7560817a07e8 nnet_ops => nnet
Joseph Turian <turian@gmail.com>
parents: 491
diff changeset
4 from theano.tensor import nnet
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
5 from theano.compile import module
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
6 from theano import printing, pprint
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
7 from theano import compile
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
8
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
9 import numpy as N
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
10
599
bd777e960c7c misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 591
diff changeset
11 class LogRegN(module.FancyModule):
bd777e960c7c misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 591
diff changeset
12 """
bd777e960c7c misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 591
diff changeset
13 A symbolic module for performing N-class logistic regression.
bd777e960c7c misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 591
diff changeset
14
bd777e960c7c misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 591
diff changeset
15 Notable variables
bd777e960c7c misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 591
diff changeset
16 -----------------
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
17
599
bd777e960c7c misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 591
diff changeset
18 self.input
bd777e960c7c misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 591
diff changeset
19 self.target
bd777e960c7c misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 591
diff changeset
20 self.softmax
bd777e960c7c misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 591
diff changeset
21 self.argmax
bd777e960c7c misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 591
diff changeset
22 self.regularized_cost
bd777e960c7c misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 591
diff changeset
23 self.unregularized_cost
bd777e960c7c misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 591
diff changeset
24 """
499
a419edf4e06c removed unpicklable nested classes in logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 497
diff changeset
25
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
26 def __init__(self,
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
27 n_in=None, n_out=None,
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
28 input=None, target=None,
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
29 w=None, b=None,
635
89bc88affef0 Reverting out of changeset 633,634. Hopefully i did this write !
projects@lgcm
parents: 633
diff changeset
30 l2=None, l1=None):
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
31 super(LogRegN, self).__init__() #boilerplate
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
32
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
33 self.n_in = n_in
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
34 self.n_out = n_out
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
35
818
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
36 if input is not None:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
37 self.input = input
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
38 else:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
39 self.input = T.matrix()
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
40
818
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
41 if target is not None:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
42 self.target = target
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
43 else:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
44 self.target = T.lvector()
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
45
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
46 #backport
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
47 #self.input = input if input is not None else T.matrix()
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
48 #self.target = target if target is not None else T.lvector()
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
49
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
50 if w is not None:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
51 self.w = w
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
52 else:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
53 self.w = (T.dmatrix())
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
54
818
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
55 if b is not None:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
56 self.b = b
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
57 else:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
58 self.b = (T.dvector())
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
59
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
60 #backport
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
61 #self.w = w if w is not None else (T.dmatrix())
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
62 #self.b = b if b is not None else (T.dvector())
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
63
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
64 self.params = []
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
65 for p in [self.w, self.b]:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
66 if p.owner is None:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
67 self.params += [p]
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
68
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
69 #backport
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
70 #the params of the model are the ones we fit to the data
818
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
71 #self.params = [p for p in [self.w, self.b] if p.owner is None]
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
72
818
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
73 if l2 is not None:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
74 self.l2 = l2
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
75 else:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
76 self.l2 = (T.dscalar())
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
77
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
78 if l1 is not None:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
79 self.l1 = l1
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
80 else:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
81 self.l1 = (T.dscalar())
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
82
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
83 #backport
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
84 #the hyper-parameters of the model are not fit to the data
818
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
85 #self.l2 = l2 if l2 is not None else (T.dscalar())
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
86 #self.l1 = l1 if l1 is not None else (T.dscalar())
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
87
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
88 #here we actually build the model
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
89 self.linear_output = T.dot(self.input, self.w) + self.b
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
90 if 0:
563
16f91ca016b1 * added NStages as a stopper (moved from hpu/conv)
desjagui@atchoum.iro.umontreal.ca
parents: 540
diff changeset
91 # TODO: pending support for target being a sparse matrix
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
92 self.softmax = nnet.softmax(self.linear_output)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
93
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
94 self._max_pr, self.argmax = T.max_and_argmax(self.linear_output)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
95 self._xent = self.target * T.log(self.softmax)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
96 else:
563
16f91ca016b1 * added NStages as a stopper (moved from hpu/conv)
desjagui@atchoum.iro.umontreal.ca
parents: 540
diff changeset
97 # TODO: when above is fixed, remove this hack (need an argmax
16f91ca016b1 * added NStages as a stopper (moved from hpu/conv)
desjagui@atchoum.iro.umontreal.ca
parents: 540
diff changeset
98 # which is independent of targets)
591
3f9ec536f2c1 Two minor fixes.
Joseph Turian <turian@iro.umontreal.ca>
parents: 564
diff changeset
99 self.argmax_standalone = T.argmax(self.linear_output)
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
100 (self._xent, self.softmax, self._max_pr, self.argmax) =\
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
101 nnet.crossentropy_softmax_max_and_argmax_1hot(
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
102 self.linear_output, self.target)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
103
856
0cfbaf0c598d sum change to mean for logistic regression cost over mini-batches
Xavier Glorot <glorotxa@iro.umontreal.ca>
parents: 818
diff changeset
104 self.unregularized_cost = T.mean(self._xent)
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
105 self.l1_cost = self.l1 * T.sum(abs(self.w))
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
106 self.l2_cost = self.l2 * T.sum(self.w**2)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
107 self.regularized_cost = self.unregularized_cost + self.l1_cost + self.l2_cost
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
108 self._loss_zero_one = T.mean(T.neq(self.argmax, self.target))
503
c7ce66b4e8f4 Extensions to algorithms, and some cleanup (by defining linear_output result).
Joseph Turian <turian@gmail.com>
parents: 502
diff changeset
109
643
5f9ffefa9ca8 pylearn/algorithms/logistic_regression.py:
Joseph Turian <turian@iro.umontreal.ca>
parents: 635
diff changeset
110 # Softmax being computed directly.
5f9ffefa9ca8 pylearn/algorithms/logistic_regression.py:
Joseph Turian <turian@iro.umontreal.ca>
parents: 635
diff changeset
111 # TODO: Move somewhere else, more clean.
5f9ffefa9ca8 pylearn/algorithms/logistic_regression.py:
Joseph Turian <turian@iro.umontreal.ca>
parents: 635
diff changeset
112 self.softmax_unsupervised = nnet.softmax(self.linear_output)
5f9ffefa9ca8 pylearn/algorithms/logistic_regression.py:
Joseph Turian <turian@iro.umontreal.ca>
parents: 635
diff changeset
113
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
114 # METHODS
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
115 if 0: #TODO: PENDING THE BETTER IMPLEMENTATION ABOVE
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
116 self.predict = module.Method([self.input], self.argmax)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
117 self.label_probs = module.Method([self.input], self.softmax)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
118 self.validate = module.Method([self.input, self.target],
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
119 [self._loss_zero_one, self.regularized_cost, self.unregularized_cost])
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
120
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
121 def _instance_initialize(self, obj):
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
122 obj.w = N.zeros((self.n_in, self.n_out))
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
123 obj.b = N.zeros(self.n_out)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
124 obj.__pp_hide__ = ['params']
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
125
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
126 def logistic_regression(n_in, n_out, l1, l2, minimizer=None):
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
127 if n_out == 2:
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
128 raise NotImplementedError()
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
129 else:
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
130 rval = LogRegN(n_in=n_in, n_out=n_out, l1=l1, l2=l2)
540
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 537
diff changeset
131 print 'RVAL input target', rval.input, rval.target
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
132 rval.minimizer = minimizer([rval.input, rval.target], rval.regularized_cost,
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
133 rval.params)
540
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 537
diff changeset
134 return rval.make(mode='FAST_RUN')
499
a419edf4e06c removed unpicklable nested classes in logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 497
diff changeset
135
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
136 #TODO: grouping parameters by prefix does not play well with providing defaults. Think...
540
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 537
diff changeset
137 #FIX : Guillaume suggested a convention: plugin handlers (dataset_factory, minimizer_factory,
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 537
diff changeset
138 # etc.) should never provide default arguments for parameters, and accept **kwargs to catch
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 537
diff changeset
139 # irrelevant parameters.
647
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
140 #SOLUTION: the jobman deals in nested dictionaries. This means that there is no [dumb] reason that
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
141 # irrelevant arguments should be passed at all.
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
142 class _fit_logreg_defaults(object):
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
143 minimizer_algo = 'dummy'
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
144 #minimizer_lr = 0.001
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
145 dataset = 'MNIST_1k'
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
146 l1 = 0.0
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
147 l2 = 0.0
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
148 batchsize = 8
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
149 verbose = 1
503
c7ce66b4e8f4 Extensions to algorithms, and some cleanup (by defining linear_output result).
Joseph Turian <turian@gmail.com>
parents: 502
diff changeset
150
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
151 def fit_logistic_regression_online(state, channel=lambda *args, **kwargs:None):
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
152 #use stochastic gradient descent
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
153 state.use_defaults(_fit_logreg_defaults)
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
154
599
bd777e960c7c misc small changes
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 591
diff changeset
155 dataset = make(state.dataset)
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
156 train = dataset.train
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
157 valid = dataset.valid
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
158 test = dataset.test
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
159
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
160 logreg = logistic_regression(
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
161 n_in=train.x.shape[1],
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
162 n_out=dataset.n_classes,
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
163 l2=state.l2,
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
164 l1=state.l1,
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
165 minimizer=make_minimizer(**state.subdict(prefix='minimizer_')))
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
166
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
167 batchsize = state.batchsize
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
168 verbose = state.verbose
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
169 iter = [0]
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
170
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
171 def step():
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
172 # step by making a pass through the training set
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
173 for j in xrange(0,len(train.x)-batchsize+1,batchsize):
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
174 cost_j = logreg.minimizer.step_cost(train.x[j:j+batchsize], train.y[j:j+batchsize])
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
175 if verbose > 1:
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
176 print 'estimated train cost', cost_j
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
177 #TODO: consult iter[0] for periodic saving to cwd (model, minimizer, and stopper)
503
c7ce66b4e8f4 Extensions to algorithms, and some cleanup (by defining linear_output result).
Joseph Turian <turian@gmail.com>
parents: 502
diff changeset
178
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
179 def check():
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
180 validate = logreg.validate(valid.x, valid.y)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
181 if verbose > 0:
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
182 print 'iter', iter[0], 'validate', validate
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
183 sys.stdout.flush()
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
184 iter[0] += 1
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
185 return validate[0]
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
186
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
187 def save():
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
188 return copy.deepcopy(logreg)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
189
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
190 stopper = make_stopper(**state.subdict(prefix='stopper_'))
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
191 stopper.find_min(step, check, save)
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
192
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
193 state.train_01, state.train_rcost, state.train_cost = logreg.validate(train.x, train.y)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
194 state.valid_01, state.valid_rcost, state.valid_cost = logreg.validate(valid.x, valid.y)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
195 state.test_01, state.test_rcost, state.test_cost = logreg.validate(test.x, test.y)
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
196
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
197 state.n_train = len(train.y)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
198 state.n_valid = len(valid.y)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
199 state.n_test = len(test.y)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
200
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
201 class LogReg2(module.FancyModule):
497
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
202 def __init__(self, input=None, targ=None, w=None, b=None, lr=None, regularize=False):
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
203 super(LogReg2, self).__init__() #boilerplate
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
204
818
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
205 if input is not None:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
206 self.input = (input)
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
207 else:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
208 self.input = T.matrix('input')
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
209
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
210 if targ is not None:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
211 self.targ = (targ)
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
212 else:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
213 self.targ = T.lcol()
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
214
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
215 #self.input = (input) if input is not None else T.matrix('input')
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
216 #self.targ = (targ) if targ is not None else T.lcol()
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
217
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
218 if w is not None:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
219 self.w = (w)
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
220 else:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
221 self.w = (T.dmatrix())
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
222
818
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
223 if b is not None:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
224 self.b = (b)
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
225 else:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
226 self.b = (T.dvector())
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
227
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
228 if lr is not None:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
229 self.lr = (lr)
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
230 else:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
231 self.lr = (T.scalar())
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
232
818
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
233 #backport
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
234 #self.w = (w) if w is not None else (T.dmatrix())
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
235 #self.b = (b) if b is not None else (T.dvector())
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
236 #self.lr = (lr) if lr is not None else (T.dscalar())
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
237
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
238 self.params = []
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
239 for p in [self.w, self.b]:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
240 if p.owner is None:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
241 self.params += [p]
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
242
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
243 #backport
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
244 #self.params = [p for p in [self.w, self.b] if p.owner is None]
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
245
502
17945defd813 Bug fix
Joseph Turian <turian@gmail.com>
parents: 501
diff changeset
246 output = nnet.sigmoid(T.dot(self.x, self.w) + self.b)
497
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
247 xent = -self.targ * T.log(output) - (1.0 - self.targ) * T.log(1.0 - output)
857
bd7d540db70d sum change to mean for logistic regression cost over mini-batches for LogReg2
Xavier Glorot <glorotxa@iro.umontreal.ca>
parents: 856
diff changeset
248 mean_xent = T.mean(xent)
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
249
497
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
250 self.output = output
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
251 self.xent = xent
857
bd7d540db70d sum change to mean for logistic regression cost over mini-batches for LogReg2
Xavier Glorot <glorotxa@iro.umontreal.ca>
parents: 856
diff changeset
252 self.mean_xent = mean_xent
bd7d540db70d sum change to mean for logistic regression cost over mini-batches for LogReg2
Xavier Glorot <glorotxa@iro.umontreal.ca>
parents: 856
diff changeset
253 self.cost = mean_xent
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
254
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
255 #define the apply method
497
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
256 self.pred = (T.dot(self.input, self.w) + self.b) > 0.0
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
257 self.apply = module.Method([self.input], self.pred)
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
258
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
259 #if this module has any internal parameters, define an update function for them
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
260 if self.params:
857
bd7d540db70d sum change to mean for logistic regression cost over mini-batches for LogReg2
Xavier Glorot <glorotxa@iro.umontreal.ca>
parents: 856
diff changeset
261 gparams = T.grad(mean_xent, self.params)
bd7d540db70d sum change to mean for logistic regression cost over mini-batches for LogReg2
Xavier Glorot <glorotxa@iro.umontreal.ca>
parents: 856
diff changeset
262 self.update = module.Method([self.input, self.targ], mean_xent,
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
263 updates = dict((p, p - self.lr * g) for p, g in zip(self.params, gparams)))
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
264
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
265
650
83e8fe9b1c82 factoring out classification from LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 649
diff changeset
266 class classification: #this would go to a file called pylearn/algorithms/classification.py
83e8fe9b1c82 factoring out classification from LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 649
diff changeset
267
83e8fe9b1c82 factoring out classification from LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 649
diff changeset
268 @staticmethod
83e8fe9b1c82 factoring out classification from LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 649
diff changeset
269 def xent(p, q):
651
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
270 """cross-entropy (row-wise)
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
271
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
272 :type p: M x N symbolic matrix (sparse or dense)
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
273
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
274 :param p: each row is a true distribution over N things
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
275
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
276 :type q: M x N symbolic matrix (sparse or dense)
650
83e8fe9b1c82 factoring out classification from LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 649
diff changeset
277
651
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
278 :param q: each row is an approximating distribution over N things
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
279
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
280 :rtype: symbolic vector of length M
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
281
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
282 :returns: the cross entropy between each row of p and the corresponding row of q.
650
83e8fe9b1c82 factoring out classification from LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 649
diff changeset
283
651
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
284
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
285 Hint: To sum row-wise costs into a scalar value, use "xent(p, q).sum()"
650
83e8fe9b1c82 factoring out classification from LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 649
diff changeset
286 """
651
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
287 return (p * tensor.log(q)).sum(axis=1)
650
83e8fe9b1c82 factoring out classification from LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 649
diff changeset
288
83e8fe9b1c82 factoring out classification from LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 649
diff changeset
289 @staticmethod
651
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
290 def errors(target, prediction):
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
291 """classification error (row-wise)
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
292
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
293 :type p: M x N symbolic matrix (sparse or dense)
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
294
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
295 :param p: each row is a true distribution over N things
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
296
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
297 :type q: M x N symbolic matrix (sparse or dense)
650
83e8fe9b1c82 factoring out classification from LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 649
diff changeset
298
651
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
299 :param q: each row is an approximating distribution over N things
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
300
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
301 :rtype: symbolic vector of length M
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
302
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
303 :returns: a vector with 0 for every row pair that has a maximum in the same position,
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
304 and 1 for every other row pair.
650
83e8fe9b1c82 factoring out classification from LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 649
diff changeset
305
651
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
306
650
83e8fe9b1c82 factoring out classification from LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 649
diff changeset
307 Hint: Count errors with "errors(prediction, target).sum()", and get the error-rate with
83e8fe9b1c82 factoring out classification from LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 649
diff changeset
308 "errors(prediction, target).mean()"
83e8fe9b1c82 factoring out classification from LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 649
diff changeset
309 """
651
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
310 return tensor.neq(
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
311 tensor.argmax(prediction, axis=1),
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
312 tensor.argmax(target, axis=1))
650
83e8fe9b1c82 factoring out classification from LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 649
diff changeset
313
647
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
314 class LogReg_New(module.FancyModule):
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
315 """A symbolic module for performing multi-class logistic regression."""
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
316
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
317 params = property(
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
318 lambda self: [p for p in [self.w, self.b] if p.owner is None],
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
319 doc="WRITEME"
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
320 )
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
321
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
322 def __init__(self, n_in=None, n_out=None, w=None, b=None):
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
323 super(LogRegNew, self).__init__() #boilerplate
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
324
818
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
325 if w is not None:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
326 self.w = w
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
327 else:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
328 self.w = (T.dmatrix())
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
329
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
330 if b is not None:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
331 self.b = b
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
332 else:
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
333 self.b = (T.dvector())
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
334
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
335
647
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
336 self.n_in = n_in
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
337 self.n_out = n_out
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
338
818
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
339 #backport
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
340 #self.w = w if w is not None else (T.dmatrix())
f4729745bb58 backporting to 2.4
dumitru@deepnets.mtv.corp.google.com
parents: 670
diff changeset
341 #self.b = b if b is not None else (T.dvector())
651
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
342
650
83e8fe9b1c82 factoring out classification from LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 649
diff changeset
343 def _instance_initialize(self, obj):
83e8fe9b1c82 factoring out classification from LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 649
diff changeset
344 obj.w = N.zeros((self.n_in, self.n_out))
83e8fe9b1c82 factoring out classification from LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 649
diff changeset
345 obj.b = N.zeros(self.n_out)
83e8fe9b1c82 factoring out classification from LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 649
diff changeset
346 obj.__pp_hide__ = ['params']
83e8fe9b1c82 factoring out classification from LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 649
diff changeset
347
647
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
348
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
349 def l1(self):
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
350 return abs(self.w).sum()
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
351
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
352 def l2(self):
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
353 return (self.w**2).sum()
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
354
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
355 def activation(self, input):
650
83e8fe9b1c82 factoring out classification from LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 649
diff changeset
356 return theano.dot(input, self.w) + self.b
647
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
357
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
358 def softmax(self, input):
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
359 return nnet.softmax(self.activation(input))
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
360
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
361 def argmax(self, input):
650
83e8fe9b1c82 factoring out classification from LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 649
diff changeset
362 return tensor.argmax(self.activation(input))
647
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
363
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
364 def xent(self, input, target):
651
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
365 return classification.xent(target, self.softmax(input))
647
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
366
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
367 def errors(self, input, target):
651
d03b5d8e4bf6 revised classification and LogReg_New
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 650
diff changeset
368 return classification.errors(target, self.softmax(input))
647
546795d7cbaf added new way of writing logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 635
diff changeset
369