Mercurial > pylearn

annotate pylearn/algorithms/logistic_regression.py @ 591:3f9ec536f2c1

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Two minor fixes.
author Joseph Turian <turian@iro.umontreal.ca>
date Fri, 19 Dec 2008 16:46:51 -0500
parents e878003c3009
children bd777e960c7c
rev   line source
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
1 import sys, copy
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
2 import theano
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
3 from theano import tensor as T
495
7560817a07e8 nnet_ops => nnet
Joseph Turian <turian@gmail.com>
parents: 491
diff changeset
4 from theano.tensor import nnet
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
5 from theano.compile import module
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
6 from theano import printing, pprint
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
7 from theano import compile
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
8
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
9 import numpy as N
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
10
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
11 from ..datasets import make_dataset
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
12 from .minimizer import make_minimizer
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
13 from .stopper import make_stopper
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
14
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
15 class LogRegN(module.FancyModule):
499
a419edf4e06c removed unpicklable nested classes in logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 497
diff changeset
16
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
17 def __init__(self,
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
18 n_in=None, n_out=None,
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
19 input=None, target=None,
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
20 w=None, b=None,
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
21 l2=None, l1=None):
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
22 super(LogRegN, self).__init__() #boilerplate
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
23
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
24 self.n_in = n_in
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
25 self.n_out = n_out
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
26
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
27 self.input = input if input is not None else T.matrix()
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
28 self.target = target if target is not None else T.lvector()
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
29
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
30 self.w = w if w is not None else module.Member(T.dmatrix())
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
31 self.b = b if b is not None else module.Member(T.dvector())
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
32
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
33 #the params of the model are the ones we fit to the data
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
34 self.params = [p for p in [self.w, self.b] if p.owner is None]
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
35
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
36 #the hyper-parameters of the model are not fit to the data
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
37 self.l2 = l2 if l2 is not None else module.Member(T.dscalar())
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
38 self.l1 = l1 if l1 is not None else module.Member(T.dscalar())
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
39
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
40 #here we actually build the model
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
41 self.linear_output = T.dot(self.input, self.w) + self.b
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
42 if 0:
563
16f91ca016b1 * added NStages as a stopper (moved from hpu/conv)
desjagui@atchoum.iro.umontreal.ca
parents: 540
diff changeset
43 # TODO: pending support for target being a sparse matrix
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
44 self.softmax = nnet.softmax(self.linear_output)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
45
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
46 self._max_pr, self.argmax = T.max_and_argmax(self.linear_output)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
47 self._xent = self.target * T.log(self.softmax)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
48 else:
563
16f91ca016b1 * added NStages as a stopper (moved from hpu/conv)
desjagui@atchoum.iro.umontreal.ca
parents: 540
diff changeset
49 # TODO: when above is fixed, remove this hack (need an argmax
16f91ca016b1 * added NStages as a stopper (moved from hpu/conv)
desjagui@atchoum.iro.umontreal.ca
parents: 540
diff changeset
50 # which is independent of targets)
591
3f9ec536f2c1 Two minor fixes.
Joseph Turian <turian@iro.umontreal.ca>
parents: 564
diff changeset
51 self.argmax_standalone = T.argmax(self.linear_output)
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
52 (self._xent, self.softmax, self._max_pr, self.argmax) =\
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
53 nnet.crossentropy_softmax_max_and_argmax_1hot(
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
54 self.linear_output, self.target)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
55
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
56 self.unregularized_cost = T.sum(self._xent)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
57 self.l1_cost = self.l1 * T.sum(abs(self.w))
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
58 self.l2_cost = self.l2 * T.sum(self.w**2)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
59 self.regularized_cost = self.unregularized_cost + self.l1_cost + self.l2_cost
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
60 self._loss_zero_one = T.mean(T.neq(self.argmax, self.target))
503
c7ce66b4e8f4 Extensions to algorithms, and some cleanup (by defining linear_output result).
Joseph Turian <turian@gmail.com>
parents: 502
diff changeset
61
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
62 # METHODS
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
63 if 0: #TODO: PENDING THE BETTER IMPLEMENTATION ABOVE
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
64 self.predict = module.Method([self.input], self.argmax)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
65 self.label_probs = module.Method([self.input], self.softmax)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
66 self.validate = module.Method([self.input, self.target],
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
67 [self._loss_zero_one, self.regularized_cost, self.unregularized_cost])
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
68
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
69 def _instance_initialize(self, obj):
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
70 obj.w = N.zeros((self.n_in, self.n_out))
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
71 obj.b = N.zeros(self.n_out)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
72 obj.__pp_hide__ = ['params']
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
73
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
74 def logistic_regression(n_in, n_out, l1, l2, minimizer=None):
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
75 if n_out == 2:
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
76 raise NotImplementedError()
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
77 else:
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
78 rval = LogRegN(n_in=n_in, n_out=n_out, l1=l1, l2=l2)
540
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 537
diff changeset
79 print 'RVAL input target', rval.input, rval.target
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
80 rval.minimizer = minimizer([rval.input, rval.target], rval.regularized_cost,
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
81 rval.params)
540
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 537
diff changeset
82 return rval.make(mode='FAST_RUN')
499
a419edf4e06c removed unpicklable nested classes in logistic regression
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 497
diff changeset
83
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
84 #TODO: grouping parameters by prefix does not play well with providing defaults. Think...
540
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 537
diff changeset
85 #FIX : Guillaume suggested a convention: plugin handlers (dataset_factory, minimizer_factory,
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 537
diff changeset
86 # etc.) should never provide default arguments for parameters, and accept **kwargs to catch
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 537
diff changeset
87 # irrelevant parameters.
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
88 class _fit_logreg_defaults(object):
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
89 minimizer_algo = 'dummy'
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
90 #minimizer_lr = 0.001
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
91 dataset = 'MNIST_1k'
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
92 l1 = 0.0
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
93 l2 = 0.0
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
94 batchsize = 8
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
95 verbose = 1
503
c7ce66b4e8f4 Extensions to algorithms, and some cleanup (by defining linear_output result).
Joseph Turian <turian@gmail.com>
parents: 502
diff changeset
96
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
97 from ..datasets import MNIST
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
98 import sgd #TODO: necessary to add it to factory list
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
99 # consider pre-importing each file in algorithms, datasets (possibly with try/catch around each
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
100 # import so that this import failure is ignored)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
101
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
102 def fit_logistic_regression_online(state, channel=lambda *args, **kwargs:None):
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
103 #use stochastic gradient descent
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
104 state.use_defaults(_fit_logreg_defaults)
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
105
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
106 dataset = make_dataset(**state.subdict(prefix='dataset_'))
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
107 train = dataset.train
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
108 valid = dataset.valid
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
109 test = dataset.test
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
110
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
111 logreg = logistic_regression(
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
112 n_in=train.x.shape[1],
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
113 n_out=dataset.n_classes,
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
114 l2=state.l2,
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
115 l1=state.l1,
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
116 minimizer=make_minimizer(**state.subdict(prefix='minimizer_')))
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
117
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
118 batchsize = state.batchsize
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
119 verbose = state.verbose
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
120 iter = [0]
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
121
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
122 def step():
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
123 # step by making a pass through the training set
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
124 for j in xrange(0,len(train.x)-batchsize+1,batchsize):
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
125 cost_j = logreg.minimizer.step_cost(train.x[j:j+batchsize], train.y[j:j+batchsize])
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
126 if verbose > 1:
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
127 print 'estimated train cost', cost_j
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
128 #TODO: consult iter[0] for periodic saving to cwd (model, minimizer, and stopper)
503
c7ce66b4e8f4 Extensions to algorithms, and some cleanup (by defining linear_output result).
Joseph Turian <turian@gmail.com>
parents: 502
diff changeset
129
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
130 def check():
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
131 validate = logreg.validate(valid.x, valid.y)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
132 if verbose > 0:
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
133 print 'iter', iter[0], 'validate', validate
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
134 sys.stdout.flush()
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
135 iter[0] += 1
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
136 return validate[0]
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
137
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
138 def save():
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
139 return copy.deepcopy(logreg)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
140
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
141 stopper = make_stopper(**state.subdict(prefix='stopper_'))
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
142 stopper.find_min(step, check, save)
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
143
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
144 state.train_01, state.train_rcost, state.train_cost = logreg.validate(train.x, train.y)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
145 state.valid_01, state.valid_rcost, state.valid_cost = logreg.validate(valid.x, valid.y)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
146 state.test_01, state.test_rcost, state.test_cost = logreg.validate(test.x, test.y)
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
147
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
148 state.n_train = len(train.y)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
149 state.n_valid = len(valid.y)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
150 state.n_test = len(test.y)
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
151
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
152 class LogReg2(module.FancyModule):
497
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
153 def __init__(self, input=None, targ=None, w=None, b=None, lr=None, regularize=False):
537
b054271b2504 new file structure layout, factories, etc.
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 503
diff changeset
154 super(LogReg2, self).__init__() #boilerplate
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
155
519
b267a8000f92 Added theano.Member to Member variables which were passed in during initialization.
Joseph Turian <turian@iro.umontreal.ca>
parents: 503
diff changeset
156 self.input = module.Member(input) if input is not None else T.matrix('input')
b267a8000f92 Added theano.Member to Member variables which were passed in during initialization.
Joseph Turian <turian@iro.umontreal.ca>
parents: 503
diff changeset
157 self.targ = module.Member(targ) if targ is not None else T.lcol()
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
158
519
b267a8000f92 Added theano.Member to Member variables which were passed in during initialization.
Joseph Turian <turian@iro.umontreal.ca>
parents: 503
diff changeset
159 self.w = module.Member(w) if w is not None else module.Member(T.dmatrix())
b267a8000f92 Added theano.Member to Member variables which were passed in during initialization.
Joseph Turian <turian@iro.umontreal.ca>
parents: 503
diff changeset
160 self.b = module.Member(b) if b is not None else module.Member(T.dvector())
b267a8000f92 Added theano.Member to Member variables which were passed in during initialization.
Joseph Turian <turian@iro.umontreal.ca>
parents: 503
diff changeset
161 self.lr = module.Member(lr) if lr is not None else module.Member(T.dscalar())
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
162
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
163 self.params = [p for p in [self.w, self.b] if p.owner is None]
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
164
502
17945defd813 Bug fix
Joseph Turian <turian@gmail.com>
parents: 501
diff changeset
165 output = nnet.sigmoid(T.dot(self.x, self.w) + self.b)
497
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
166 xent = -self.targ * T.log(output) - (1.0 - self.targ) * T.log(1.0 - output)
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
167 sum_xent = T.sum(xent)
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
168
497
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
169 self.output = output
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
170 self.xent = xent
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
171 self.sum_xent = sum_xent
491
180d125dc7e2 made logistic_regression classes compatible with stacker
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents: 476
diff changeset
172 self.cost = sum_xent
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
173
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
174 #define the apply method
497
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
175 self.pred = (T.dot(self.input, self.w) + self.b) > 0.0
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
176 self.apply = module.Method([self.input], self.pred)
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
177
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
178 #if this module has any internal parameters, define an update function for them
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
179 if self.params:
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
180 gparams = T.grad(sum_xent, self.params)
497
a272f4cbf004 'x' => 'input'
Joseph Turian <turian@gmail.com>
parents: 495
diff changeset
181 self.update = module.Method([self.input, self.targ], sum_xent,
470
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
182 updates = dict((p, p - self.lr * g) for p, g in zip(self.params, gparams)))
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
183
bd937e845bbb new stuff: algorithms/logistic_regression, datasets/MNIST
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
184