annotate stopper.py @ 474:40c8a46b3da7

added Stopper.find_min
author James Bergstra <bergstrj@iro.umontreal.ca>
date Thu, 23 Oct 2008 18:05:46 -0400
parents cc96f93a7810
children 0ea793361d85
rev   line source
178
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
1 """Early stopping iterators
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
2
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
3 The idea here is to supply early-stopping heuristics that can be used in the
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
4 form:
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
5
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
6 stopper = SomeEarlyStopper()
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
7
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
8 for i in stopper():
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
9 # train from data
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
10 if i.set_score:
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
11 i.score = validation_score
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
12
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
13
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
14 So far I only have one heuristic, so maybe this won't scale.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
15 """
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
16
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
17 class Stopper(object):
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
18
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
19 def train(self, data, update_rows_fn, update, validate, save=None):
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
20 """Return the best model trained on data
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
21
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
22 Parameters:
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
23 data - a thing that accepts getitem(<list of int64>), or a tuple of such things
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
24 update_rows_fn - fn : int --> <list or tensor of int>
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
25 update - fn: update an internal model from elements of data
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
26 validate - fn: evaluate an internal model based on elements of data
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
27 save - fn: return a copy of the internal model
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
28
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
29 The body of this function exhausts the <self> iterator, and trains a
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
30 model using early stopping in the process.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
31 """
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
32
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
33 best = None
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
34 for stp in self:
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
35 i = stp.iter
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
36
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
37 # call update on some training set rows
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
38 t_rows = update_rows_fn(i)
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
39 if isinstance(data, (tuple, list)):
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
40 update(*[d[t_rows] for d in data])
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
41 else:
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
42 update(data[t_rows])
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
43
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
44 if stp.set_score:
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
45 stp.score = validate()
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
46 if (stp.score < stp.best_score) and save:
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
47 best = save()
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
48 return best
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
49
474
40c8a46b3da7 added Stopper.find_min
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 213
diff changeset
50 def find_min(self, step, check, save):
40c8a46b3da7 added Stopper.find_min
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 213
diff changeset
51 best = None
40c8a46b3da7 added Stopper.find_min
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 213
diff changeset
52 for stp in self:
40c8a46b3da7 added Stopper.find_min
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 213
diff changeset
53 step()
40c8a46b3da7 added Stopper.find_min
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 213
diff changeset
54 if stp.set_score:
40c8a46b3da7 added Stopper.find_min
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 213
diff changeset
55 stp.score = check()
40c8a46b3da7 added Stopper.find_min
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 213
diff changeset
56 if (stp.score < stp.best_score) and save:
40c8a46b3da7 added Stopper.find_min
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 213
diff changeset
57 best = save()
40c8a46b3da7 added Stopper.find_min
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 213
diff changeset
58 return best
40c8a46b3da7 added Stopper.find_min
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 213
diff changeset
59
40c8a46b3da7 added Stopper.find_min
James Bergstra <bergstrj@iro.umontreal.ca>
parents: 213
diff changeset
60
178
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
61
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
62 class ICML08Stopper(Stopper):
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
63 @staticmethod
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
64 def icml08(ntrain, batchsize):
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
65 """Some setting similar to what I used for ICML08 submission"""
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
66 #TODO: what did I actually use? put that in here.
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
67 return ICML08Stopper(30*ntrain/batchsize,
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
68 ntrain/batchsize, 0.96, 2.0, 100000000)
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
69
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
70 def __init__(self, i_wait, v_int, min_improvement, patience, hard_limit):
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
71 self.initial_wait = i_wait
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
72 self.set_score_interval = v_int
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
73 self.min_improvement = min_improvement
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
74 self.patience = patience
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
75 self.hard_limit = hard_limit
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
76
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
77 self.best_score = float('inf')
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
78 self.best_iter = -1
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
79 self.iter = -1
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
80
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
81 self.set_score = False
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
82 self.score = None
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
83
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
84 def __iter__(self):
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
85 return self
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
86
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
87 E_set_score = 'when iter.set_score is True, caller must assign a score to iter.score'
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
88 def next(self):
211
bd728c83faff in __get__, problem if the i.stop was None, i being the slice, added one line replacing None by the len(self)
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents: 178
diff changeset
89
213
cc96f93a7810 previous commit was supposed to concern only one file, dataset.py, try to undo my other changes with this commit (nothing was broken though, just useless debugging prints)
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents: 211
diff changeset
90 #print 'ICML08 stopper, were doing a next'
211
bd728c83faff in __get__, problem if the i.stop was None, i being the slice, added one line replacing None by the len(self)
Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca>
parents: 178
diff changeset
91
178
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
92 if self.set_score: #left over from last time
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
93 if self.score is None:
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
94 raise Exception(ICML08Stopper.E_set_score)
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
95 if self.score < (self.best_score * self.min_improvement):
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
96 (self.best_score, self.best_iter) = (self.score, self.iter)
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
97 self.score = None #un-set it
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
98
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
99
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
100 starting = self.iter < self.initial_wait
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
101 waiting = self.iter < (self.patience * self.best_iter)
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
102 if starting or waiting:
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
103 # continue to iterate
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
104 self.iter += 1
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
105 if self.iter == self.hard_limit:
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
106 raise StopIteration
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
107 self.set_score = (self.iter % self.set_score_interval == 0)
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
108 return self
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
109
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
110 raise StopIteration
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
111
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
112
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
113 class NStages(ICML08Stopper):
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
114 """Run for a fixed number of steps, checking validation set every so
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
115 often."""
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
116 def __init__(self, hard_limit, v_int):
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
117 ICML08Stopper.__init__(self, hard_limit, v_int, 1.0, 1.0, hard_limit)
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
118
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
119 #TODO: could optimize next() function. Most of what's in ICML08Stopper.next()
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
120 #is not necessary
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
121
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
122