Mercurial > pylearn
annotate pylearn/dbdict/newstuff.py @ 579:a83f62555c57
doc fix
| author | Olivier Breuleux <breuleuo@iro.umontreal.ca> |
|---|---|
| date | Thu, 04 Dec 2008 17:07:29 -0500 |
| parents | a027c4cedf98 |
| children | 9f5691d5587d |
| rev | line source |
|---|---|
| 565 | 1 |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
2 from __future__ import with_statement |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
3 |
| 565 | 4 from collections import defaultdict |
|
572
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
5 import re, sys, inspect, os, signal, tempfile, shutil, socket |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
6 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
7 import sql |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
8 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
9 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
10 ################################################################################ |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
11 ### misc |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
12 ################################################################################ |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
13 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
14 class DD(defaultdict): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
15 def __getattr__(self, attr): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
16 return self[attr] |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
17 def __setattr__(self, attr, value): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
18 self[attr] = value |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
19 def __str__(self): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
20 return 'DD%s' % dict(self) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
21 def __repr__(self): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
22 return str(self) |
| 565 | 23 |
| 24 ################################################################################ | |
| 25 ### resolve | |
| 26 ################################################################################ | |
| 27 | |
| 28 def resolve(name): | |
| 29 symbols = name.split('.') | |
| 30 builder = __import__(symbols[0]) | |
| 31 for sym in symbols[1:]: | |
| 32 builder = getattr(builder, sym) | |
| 33 return builder | |
| 34 | |
| 35 ################################################################################ | |
| 36 ### dictionary | |
| 37 ################################################################################ | |
| 38 | |
| 39 def convert(obj): | |
|
572
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
40 try: |
|
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
41 return eval(obj, {}, {}) |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
42 except (NameError, SyntaxError): |
|
572
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
43 return obj |
| 565 | 44 |
| 45 def flatten(obj): | |
| 46 d = {} | |
| 47 def helper(d, prefix, obj): | |
| 48 if isinstance(obj, (str, int, float)): | |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
49 d[prefix] = obj #convert(obj) |
| 565 | 50 else: |
| 51 if isinstance(obj, dict): | |
| 52 subd = obj | |
| 53 else: | |
| 54 subd = obj.state() | |
| 55 subd['__builder__'] = '%s.%s' % (obj.__module__, obj.__class__.__name__) | |
| 56 for k, v in subd.iteritems(): | |
| 57 pfx = '.'.join([prefix, k]) if prefix else k | |
| 58 helper(d, pfx, v) | |
| 59 helper(d, '', obj) | |
| 60 return d | |
| 61 | |
| 62 def expand(d): | |
| 63 def dd(): | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
64 return DD(dd) |
| 565 | 65 struct = dd() |
| 66 for k, v in d.iteritems(): | |
| 67 if k == '': | |
| 68 raise NotImplementedError() | |
| 69 else: | |
| 70 keys = k.split('.') | |
| 71 current = struct | |
| 72 for k2 in keys[:-1]: | |
| 73 current = current[k2] | |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
74 current[keys[-1]] = v #convert(v) |
| 565 | 75 return struct |
| 76 | |
| 77 def realize(d): | |
| 78 if not isinstance(d, dict): | |
| 79 return d | |
| 80 d = dict((k, realize(v)) for k, v in d.iteritems()) | |
| 81 if '__builder__' in d: | |
| 82 builder = resolve(d.pop('__builder__')) | |
| 83 return builder(**d) | |
| 84 return d | |
| 85 | |
| 86 def make(d): | |
| 87 return realize(expand(d)) | |
| 88 | |
| 89 ################################################################################ | |
| 90 ### errors | |
| 91 ################################################################################ | |
| 92 | |
| 93 class UsageError(Exception): | |
| 94 pass | |
| 95 | |
| 96 ################################################################################ | |
| 97 ### parsing and formatting | |
| 98 ################################################################################ | |
| 99 | |
| 100 def parse(*strings): | |
| 101 d = {} | |
| 102 for string in strings: | |
| 103 s1 = re.split(' *= *', string, 1) | |
| 104 s2 = re.split(' *:: *', string, 1) | |
| 105 if len(s1) == 1 and len(s2) == 1: | |
| 106 raise UsageError('Expected a keyword argument in place of "%s"' % s1[0]) | |
| 107 elif len(s1) == 2: | |
| 108 k, v = s1 | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
109 v = convert(v) |
| 565 | 110 elif len(s2) == 2: |
| 111 k, v = s2 | |
| 112 k += '.__builder__' | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
113 d[k] = v |
| 565 | 114 return d |
| 115 | |
| 566 | 116 def format_d(d, sep = '\n', space = True): |
| 117 d = flatten(d) | |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
118 pattern = "%s = %r" if space else "%s=%r" |
| 566 | 119 return sep.join(pattern % (k, v) for k, v in d.iteritems()) |
| 120 | |
| 565 | 121 def format_help(topic): |
| 122 if topic is None: | |
| 123 return 'No help.' | |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
124 elif isinstance(topic, str): |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
125 help = topic |
| 565 | 126 elif hasattr(topic, 'help'): |
| 127 help = topic.help() | |
| 128 else: | |
| 129 help = topic.__doc__ | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
130 if not help: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
131 return 'No help.' |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
132 |
| 565 | 133 ss = map(str.rstrip, help.split('\n')) |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
134 try: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
135 baseline = min([len(line) - len(line.lstrip()) for line in ss if line]) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
136 except: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
137 return 'No help.' |
| 565 | 138 s = '\n'.join([line[baseline:] for line in ss]) |
| 139 s = re.sub(string = s, pattern = '\n{2,}', repl = '\n\n') | |
| 140 s = re.sub(string = s, pattern = '(^\n*)|(\n*$)', repl = '') | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
141 |
| 565 | 142 return s |
| 143 | |
| 144 ################################################################################ | |
| 566 | 145 ### single channels |
| 565 | 146 ################################################################################ |
| 147 | |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
148 # try: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
149 # import greenlet |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
150 # except: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
151 # try: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
152 # from py import greenlet |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
153 # except: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
154 # print >>sys.stderr, 'the greenlet module is unavailable' |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
155 # greenlet = None |
| 566 | 156 |
| 157 | |
| 565 | 158 class Channel(object): |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
159 |
| 566 | 160 COMPLETE = None |
| 161 INCOMPLETE = True | |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
162 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
163 START = 0 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
164 """dbdict.status == START means a experiment is ready to run""" |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
165 RUNNING = 1 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
166 """dbdict.status == RUNNING means a experiment is running on dbdict_hostname""" |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
167 DONE = 2 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
168 """dbdict.status == DONE means a experiment has completed (not necessarily successfully)""" |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
169 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
170 # Methods to be used by the experiment to communicate with the channel |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
171 |
| 566 | 172 def save(self): |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
173 """ |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
174 Save the experiment's state to the various media supported by |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
175 the Channel. |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
176 """ |
| 565 | 177 raise NotImplementedError() |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
178 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
179 def switch(self, message = None): |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
180 """ |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
181 Called from the experiment to give the control back to the channel. |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
182 The following return values are meaningful: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
183 * 'stop' -> the experiment must stop as soon as possible. It may save what |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
184 it needs to save. This occurs when SIGTERM or SIGINT are sent (or in |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
185 user-defined circumstances). |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
186 switch() may give the control to the user. In this case, the user may |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
187 resume the experiment by calling switch() again. If an argument is given |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
188 by the user, it will be relayed to the experiment. |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
189 """ |
| 566 | 190 pass |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
191 |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
192 def __call__(self, message = None): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
193 return self.switch(message) |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
194 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
195 def save_and_switch(self): |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
196 self.save() |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
197 self.switch() |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
198 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
199 # Methods to run the experiment |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
200 |
| 566 | 201 def setup(self): |
| 202 pass | |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
203 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
204 def __enter__(self): |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
205 pass |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
206 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
207 def __exit__(self): |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
208 pass |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
209 |
| 566 | 210 def run(self): |
| 565 | 211 pass |
| 212 | |
| 213 | |
| 214 | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
215 class JobError(Exception): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
216 RUNNING = 0 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
217 DONE = 1 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
218 NOJOB = 2 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
219 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
220 |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
221 class SingleChannel(Channel): |
| 566 | 222 |
| 223 def __init__(self, experiment, state): | |
| 224 self.experiment = experiment | |
| 225 self.state = state | |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
226 self.feedback = None |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
227 |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
228 def switch(self, message = None): |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
229 feedback = self.feedback |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
230 self.feedback = None |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
231 return feedback |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
232 |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
233 def run(self, force = False): |
| 566 | 234 self.setup() |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
235 |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
236 status = self.state.dbdict.get('status', self.START) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
237 if status is self.DONE and not force: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
238 # If you want to disregard this, use the --force flag (not yet implemented) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
239 raise JobError(JobError.RUNNING, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
240 'The job has already completed.') |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
241 elif status is self.RUNNING and not force: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
242 raise JobError(JobError.DONE, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
243 'The job is already running.') |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
244 self.state.dbdict.status = self.RUNNING |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
245 |
|
572
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
246 v = self.COMPLETE |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
247 with self: |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
248 try: |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
249 v = self.experiment(self.state, self) |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
250 finally: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
251 self.state.dbdict.status = self.DONE if v is self.COMPLETE else self.START |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
252 |
| 566 | 253 return v |
| 254 | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
255 def on_sigterm(self, signo, frame): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
256 # SIGTERM handler. It is the experiment function's responsibility to |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
257 # call switch() often enough to get this feedback. |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
258 self.feedback = 'stop' |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
259 |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
260 def __enter__(self): |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
261 # install a SIGTERM handler that asks the experiment function to return |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
262 # the next time it will call switch() |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
263 self.prev_sigterm = signal.getsignal(signal.SIGTERM) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
264 self.prev_sigint = signal.getsignal(signal.SIGINT) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
265 signal.signal(signal.SIGTERM, self.on_sigterm) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
266 signal.signal(signal.SIGINT, self.on_sigterm) |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
267 return self |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
268 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
269 def __exit__(self, type, value, traceback): |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
270 signal.signal(signal.SIGTERM, self.prev_sigterm) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
271 signal.signal(signal.SIGINT, self.prev_sigint) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
272 self.prev_sigterm = None |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
273 self.prev_sigint = None |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
274 self.save() |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
275 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
276 |
| 566 | 277 class StandardChannel(SingleChannel): |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
278 |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
279 def __init__(self, path, experiment, state, redirect_stdout = False, redirect_stderr = False): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
280 super(StandardChannel, self).__init__(experiment, state) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
281 self.path = os.path.realpath(path) |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
282 self.redirect_stdout = redirect_stdout |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
283 self.redirect_stderr = redirect_stderr |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
284 |
| 566 | 285 def save(self): |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
286 with open(os.path.join(self.path, 'current.conf'), 'w') as current: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
287 current.write(format_d(self.state)) |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
288 current.write('\n') |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
289 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
290 def __enter__(self): |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
291 self.old_cwd = os.getcwd() |
| 566 | 292 os.chdir(self.path) |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
293 if self.redirect_stdout: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
294 self.old_stdout = sys.stdout |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
295 sys.stdout = open('stdout', 'a') |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
296 if self.redirect_stderr: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
297 self.old_stderr = sys.stderr |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
298 sys.stderr = open('stderr', 'a') |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
299 return super(StandardChannel, self).__enter__() |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
300 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
301 def __exit__(self, type, value, traceback): |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
302 if self.redirect_stdout: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
303 sys.stdout.close() |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
304 sys.stdout = self.old_stdout |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
305 if self.redirect_stderr: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
306 sys.stderr.close() |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
307 sys.stderr = self.old_stderr |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
308 os.chdir(self.old_cwd) |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
309 return super(StandardChannel, self).__exit__(type, value, traceback) |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
310 |
| 566 | 311 def setup(self): |
| 312 if not os.path.isdir(self.path): | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
313 os.makedirs(self.path) |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
314 with self: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
315 origf = os.path.join(self.path, 'orig.conf') |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
316 if not os.path.isfile(origf): |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
317 with open(origf, 'w') as orig: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
318 orig.write(format_d(self.state)) |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
319 orig.write('\n') |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
320 currentf = os.path.join(self.path, 'current.conf') |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
321 if os.path.isfile(currentf): |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
322 with open(currentf, 'r') as current: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
323 self.state = expand(parse(*map(str.strip, current.readlines()))) |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
324 |
| 566 | 325 |
| 326 class RSyncException(Exception): | |
| 327 pass | |
| 328 | |
| 329 class RSyncChannel(StandardChannel): | |
| 330 | |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
331 def __init__(self, path, remote_path, experiment, state, redirect_stdout = False, redirect_stderr = False): |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
332 super(RSyncChannel, self).__init__(path, experiment, state, redirect_stdout, redirect_stderr) |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
333 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
334 ssh_prefix='ssh://' |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
335 if remote_path.startswith(ssh_prefix): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
336 remote_path = remote_path[len(ssh_prefix):] |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
337 colon_pos = remote_path.find(':') |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
338 self.host = remote_path[:colon_pos] |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
339 self.remote_path = remote_path[colon_pos+1:] |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
340 else: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
341 self.host = '' |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
342 self.remote_path = os.path.realpath(remote_path) |
| 566 | 343 |
| 344 def rsync(self, direction): | |
| 345 """The directory at which experiment-related files are stored. | |
| 346 """ | |
| 347 | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
348 path = self.path |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
349 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
350 remote_path = self.remote_path |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
351 if self.host: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
352 remote_path = ':'.join([self.host, remote_path]) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
353 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
354 # TODO: use something more portable than os.system |
| 566 | 355 if direction == 'push': |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
356 rsync_cmd = 'rsync -ac "%s/" "%s/"' % (path, remote_path) |
| 566 | 357 elif direction == 'pull': |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
358 rsync_cmd = 'rsync -ac "%s/" "%s/"' % (remote_path, path) |
| 566 | 359 else: |
| 360 raise RSyncException('invalid direction', direction) | |
| 361 | |
| 362 rsync_rval = os.system(rsync_cmd) | |
| 363 if rsync_rval != 0: | |
| 364 raise RSyncException('rsync failure', (rsync_rval, rsync_cmd)) | |
| 365 | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
366 def touch(self): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
367 if self.host: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
368 host = self.host |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
369 touch_cmd = ('ssh %(host)s "mkdir -p \'%(path)s\'"' % dict(host = self.host, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
370 path = self.remote_path)) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
371 else: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
372 touch_cmd = ("mkdir -p '%(path)s'" % dict(path = self.remote_path)) |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
373 # print "ECHO", touch_cmd |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
374 touch_rval = os.system(touch_cmd) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
375 if 0 != touch_rval: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
376 raise Exception('touch failure', (touch_rval, touch_cmd)) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
377 |
| 566 | 378 def pull(self): |
| 379 return self.rsync('pull') | |
| 380 | |
| 381 def push(self): | |
| 382 return self.rsync('push') | |
| 383 | |
| 384 def save(self): | |
| 385 super(RSyncChannel, self).save() | |
| 386 self.push() | |
| 387 | |
| 388 def setup(self): | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
389 self.touch() |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
390 self.pull() |
| 566 | 391 super(RSyncChannel, self).setup() |
| 392 | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
393 |
| 566 | 394 class DBRSyncChannel(RSyncChannel): |
| 395 | |
|
572
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
396 RESTART_PRIORITY = 2.0 |
|
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
397 |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
398 def __init__(self, username, password, hostname, dbname, tablename, path, remote_root, redirect_stdout = False, redirect_stderr = False): |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
399 self.username, self.password, self.hostname, self.dbname, self.tablename \ |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
400 = username, password, hostname, dbname, tablename |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
401 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
402 self.db = sql.postgres_serial( |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
403 user = self.username, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
404 password = self.password, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
405 host = self.hostname, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
406 database = self.dbname, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
407 table_prefix = self.tablename) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
408 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
409 self.dbstate = sql.book_dct_postgres_serial(self.db) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
410 if self.dbstate is None: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
411 raise JobError(JobError.NOJOB, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
412 'No job was found to run.') |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
413 |
|
572
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
414 try: |
|
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
415 state = expand(self.dbstate) |
|
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
416 experiment = resolve(state.dbdict.experiment) |
|
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
417 remote_path = os.path.join(remote_root, self.dbname, self.tablename, str(self.dbstate.id)) |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
418 super(DBRSyncChannel, self).__init__(path, remote_path, experiment, state, redirect_stdout, redirect_stderr) |
|
572
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
419 except: |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
420 self.dbstate['dbdict.status'] = self.DONE |
|
572
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
421 raise |
| 566 | 422 |
| 423 def save(self): | |
| 424 super(DBRSyncChannel, self).save() | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
425 self.dbstate.update(flatten(self.state)) |
| 566 | 426 |
| 427 def setup(self): | |
| 428 # Extract a single experiment from the table that is not already running. | |
| 429 # set self.experiment and self.state | |
| 430 super(DBRSyncChannel, self).setup() | |
|
572
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
431 self.state.dbdict.sql.host_name = socket.gethostname() |
|
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
432 self.state.dbdict.sql.host_workdir = self.path |
|
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
433 self.dbstate.update(flatten(self.state)) |
| 566 | 434 |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
435 def run(self): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
436 # We pass the force flag as True because the status flag is |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
437 # already set to RUNNING by book_dct in __init__ |
|
572
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
438 v = super(DBRSyncChannel, self).run(force = True) |
|
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
439 if v is self.INCOMPLETE and self.state.dbdict.sql.priority != self.RESTART_PRIORITY: |
|
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
440 self.state.dbdict.sql.priority = self.RESTART_PRIORITY |
|
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
441 self.save() |
|
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
442 return v |
| 565 | 443 |
| 444 | |
| 445 | |
| 566 | 446 ################################################################################ |
| 447 ### running | |
| 448 ################################################################################ | |
| 565 | 449 |
| 566 | 450 def run(type, arguments): |
| 451 runner = runner_registry.get(type, None) | |
| 452 if not runner: | |
| 453 raise UsageError('Unknown runner: "%s"' % type) | |
| 454 argspec = inspect.getargspec(runner) | |
| 455 minargs = len(argspec[0])-(len(argspec[3]) if argspec[3] else 0) | |
| 456 maxargs = len(argspec[0]) | |
| 457 if minargs > len(arguments) or maxargs < len(arguments) and not argspec[1]: | |
| 458 s = format_help(runner) | |
| 459 raise UsageError(s) | |
| 460 runner(*arguments) | |
| 565 | 461 |
| 566 | 462 runner_registry = dict() |
| 565 | 463 |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
464 def runner_cmdline(experiment, *strings): |
| 566 | 465 """ |
| 466 Start an experiment with parameters given on the command line. | |
| 467 | |
| 579 | 468 Usage: cmdline <experiment> <parameters> |
| 565 | 469 |
| 566 | 470 Run an experiment with parameters provided on the command |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
471 line. See the help topics for experiment and parameters for |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
472 syntax information. |
| 565 | 473 |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
474 Example use: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
475 dbdict-run cmdline mymodule.my_experiment \\ |
| 566 | 476 stopper::pylearn.stopper.nsteps \\ # use pylearn.stopper.nsteps |
| 477 stopper.n=10000 \\ # the argument "n" of nsteps is 10000 | |
| 478 lr=0.03 | |
| 479 """ | |
| 480 state = expand(parse(*strings)) | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
481 state.dbdict.experiment = experiment |
| 566 | 482 experiment = resolve(experiment) |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
483 #channel = RSyncChannel('.', 'yaddayadda', experiment, state) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
484 channel = StandardChannel(format_d(state, sep=',', space = False), |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
485 experiment, state) |
| 566 | 486 channel.run() |
| 565 | 487 |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
488 runner_registry['cmdline'] = runner_cmdline |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
489 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
490 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
491 def runner_sqlschedule(dbdescr, experiment, *strings): |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
492 """ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
493 Schedule a job to run using the sql command. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
494 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
495 Usage: sqlschedule <tablepath> <experiment> <parameters> |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
496 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
497 See the experiment and parameters topics for more information about |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
498 these parameters. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
499 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
500 Assuming that a postgres database is running on `host`, contains a |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
501 database called `dbname` and that `user` has the permissions to |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
502 create, read and modify tables on that database, tablepath should |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
503 be of the following form: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
504 postgres://user:pass@host/dbname/tablename |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
505 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
506 If no table is named `tablename`, one will be created |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
507 automatically. The state corresponding to the experiment and |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
508 parameters specified in the command will be saved in the database, |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
509 but no experiment will be run. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
510 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
511 To run an experiment scheduled using sqlschedule, see the sql |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
512 command. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
513 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
514 Example use: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
515 dbdict-run sqlschedule postgres://user:pass@host/dbname/tablename \\ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
516 mymodule.my_experiment \\ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
517 stopper::pylearn.stopper.nsteps \\ # use pylearn.stopper.nsteps |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
518 stopper.n=10000 \\ # the argument "n" of nsteps is 10000 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
519 lr=0.03 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
520 """ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
521 |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
522 try: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
523 username, password, hostname, dbname, tablename \ |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
524 = sql.parse_dbstring(dbdescr) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
525 except: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
526 raise UsageError('Wrong syntax for dbdescr') |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
527 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
528 db = sql.postgres_serial( |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
529 user = username, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
530 password = password, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
531 host = hostname, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
532 database = dbname, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
533 table_prefix = tablename) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
534 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
535 state = parse(*strings) |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
536 try: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
537 resolve(experiment) |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
538 except: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
539 raise UsageError('The first parameter to sqlschedule must be a valid, importable symbol.') |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
540 state['dbdict.experiment'] = experiment |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
541 sql.add_experiments_to_db([state], db, verbose = 1) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
542 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
543 runner_registry['sqlschedule'] = runner_sqlschedule |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
544 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
545 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
546 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
547 def runner_sql(dbdescr, exproot): |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
548 """ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
549 Run jobs from a sql table. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
550 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
551 Usage: sql <tablepath> <exproot> |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
552 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
553 The jobs should be scheduled first with the sqlschedule command. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
554 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
555 Assuming that a postgres database is running on `host`, contains a |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
556 database called `dbname` and that `user` has the permissions to |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
557 create, read and modify tables on that database, tablepath should |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
558 be of the following form: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
559 postgres://user:pass@host/dbname/tablename |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
560 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
561 exproot can be a local path or a remote path. Examples of exproots: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
562 /some/local/path |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
563 ssh://some_host:/some/remote/path # relative to the filesystem root |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
564 ssh://some_host:other/remote/path # relative to the HOME on some_host |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
565 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
566 The exproot will contain a subdirectory hierarchy corresponding to |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
567 the dbname, tablename and job id which is a unique integer. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
568 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
569 The sql runner will pick any job in the table which is not running |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
570 and is not done and will terminate when that job ends. You may call |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
571 the same command multiple times, sequentially or in parallel, to |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
572 run as many unfinished jobs as have been scheduled in that table |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
573 with sqlschedule. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
574 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
575 Example use: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
576 dbdict-run sql \\ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
577 postgres://user:pass@host/dbname/tablename \\ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
578 ssh://central_host:myexperiments |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
579 """ |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
580 try: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
581 username, password, hostname, dbname, tablename \ |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
582 = sql.parse_dbstring(dbdescr) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
583 except: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
584 raise UsageError('Wrong syntax for dbdescr') |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
585 workdir = tempfile.mkdtemp() |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
586 #print 'wdir', workdir |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
587 channel = DBRSyncChannel(username, password, hostname, dbname, tablename, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
588 workdir, |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
589 exproot, |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
590 redirect_stdout = True, |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
591 redirect_stderr = True) |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
592 channel.run() |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
593 shutil.rmtree(workdir, ignore_errors=True) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
594 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
595 runner_registry['sql'] = runner_sql |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
596 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
597 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
598 |
| 565 | 599 |
| 600 | |
| 566 | 601 def help(topic = None): |
| 602 """ | |
| 603 Get help for a topic. | |
| 565 | 604 |
| 566 | 605 Usage: help <topic> |
| 606 """ | |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
607 def bold(x): |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
608 return '\033[1m%s\033[0m' % x |
| 566 | 609 if topic is None: |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
610 print bold('Topics: (use help <topic> for more info)') |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
611 print 'example Example of defining and running an experiment.' |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
612 print 'experiment How to define an experiment.' |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
613 print 'parameters How to list the parameters for an experiment.' |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
614 print |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
615 print bold('Available commands: (use help <command> for more info)') |
| 566 | 616 for name, command in sorted(runner_registry.iteritems()): |
| 617 print name.ljust(20), format_help(command).split('\n')[0] | |
| 618 return | |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
619 elif topic == 'experiment': |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
620 helptext = """ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
621 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
622 dbdict-run serves to run experiments. To define an experiment, you |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
623 only have to define a function respecting the following protocol in |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
624 a python file or module: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
625 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
626 def my_experiment(state, channel): |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
627 # experiment code goes here |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
628 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
629 The return value of my_experiment may be channel.COMPLETE or |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
630 channel.INCOMPLETE. If the latter is returned, the experiment may |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
631 be resumed at a later point. Note that the return value `None` |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
632 is interpreted as channel.COMPLETE. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
633 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
634 If a command defined by dbdict-run has an <experiment> parameter, |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
635 that parameter must be a string such that it could be used in a |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
636 python import statement to import the my_experiment function. For |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
637 example if you defined my_experiment in my_module.py, you can pass |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
638 'my_module.my_experiment' as the experiment parameter. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
639 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
640 When entering my_experiment, the current working directory will be |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
641 set for you to a directory specially created for the experiment. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
642 The location and name of that directory vary depending on which |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
643 dbdict-run command you run. You may create logs, save files, pictures, |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
644 results, etc. in it. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
645 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
646 state is an object containing the parameters given to the experiment. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
647 For example, if you run the followinc command: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
648 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
649 dbdict-run cmdline my_module.my_experiment a.x=6 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
650 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
651 `state.a.x` will contain the integer 6, and so will `state['a']['x']`. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
652 If the state is changed, it will be saved when the experiment ends |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
653 or when channel.save() is called. The next time the experiment is run |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
654 with the same working directory, the modified state will be provided. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
655 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
656 It is not recommended to store large amounts of data in the state. It |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
657 should be limited to scalar or string parameters. Results such as |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
658 weight matrices should be stored in files in the working directory. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
659 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
660 channel is an object with the following important methods: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
661 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
662 - channel.switch() (or channel()) will give the control back to the |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
663 user, if it is appropriate to do so. If a call to channel.switch() |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
664 returns the string 'stop', it typically means that the signal |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
665 SIGTERM (or SIGINT) was received. Therefore, the experiment may be |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
666 killed soon, so it should save and return True or |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
667 channel.INCOMPLETE so it can be resumed later. This should be |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
668 checked periodically or data loss may be incurred. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
669 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
670 - channel.save() will save the current state. It is automatically |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
671 called when the function returns, but it is a good idea to do it |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
672 periodically. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
673 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
674 - channel.save_and_switch() is an useful shortcut to do both operations |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
675 described above. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
676 """ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
677 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
678 elif topic == 'parameters': |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
679 helptext = """ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
680 If a command takes <parameters> arguments, the arguments should each |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
681 take one of the following forms: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
682 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
683 key=value |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
684 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
685 Set a parameter with name `key` to `value`. The value will be casted |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
686 to an appropriate type automatically and it will be accessible to |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
687 the experiment using `state.key`. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
688 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
689 If `key` is a dotted name, the value will be set in nested |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
690 dictionaries corresponding to each part. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
691 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
692 Examples: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
693 a=1 state.a <- 1 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
694 b=2.3 state.b <- 2.3 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
695 c.d="hello" state.c.d <- "hello" |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
696 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
697 key::builder |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
698 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
699 This is equivalent to key.__builder__=builder. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
700 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
701 The builder should be a symbol that can be used with import or |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
702 __import__ and it should be callable. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
703 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
704 If a key has a builder defined, the experiment code may easily make |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
705 an object out of it using the `make` function. `obj = make(state.key)`. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
706 This will call the builder on the substate corresponding to state.key, |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
707 as will be made clear in the example: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
708 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
709 Example: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
710 regexp::re.compile |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
711 regexp.pattern='a.*c' |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
712 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
713 from pylearn.dbdict.newstuff import make |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
714 def experiment(state, channel): |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
715 regexp = make(state.regexp) # regexp is now re.compile(pattern = 'a.*c') |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
716 print regexp.sub('blahblah', 'hello abbbbc there') |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
717 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
718 If the above experiment was called with the state produced by the |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
719 parameters in the example, it would print 'hello blahblah there'. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
720 """ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
721 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
722 elif topic == 'example': |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
723 helptext = """ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
724 Example of an experiment that trains some model for 100000 iterations: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
725 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
726 # defined in: my_experiments.py |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
727 def experiment(state, channel): |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
728 try: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
729 model = cPickle.load(open('model', 'r')) |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
730 except: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
731 model = my_model(state.some_param, state.other_param) |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
732 state.n = 0 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
733 dataset = my_dataset(skipto = state.n) |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
734 for i in xrange(100000 - state.n): |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
735 model.update(dataset.next()) |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
736 if i and i % 1000 == 0: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
737 if channel.save_and_switch() == 'stop': |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
738 state.n += i + 1 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
739 rval = channel.INCOMPLETE |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
740 break |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
741 else: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
742 state.result = model.cost(some_test_set) |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
743 rval = channel.COMPLETE |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
744 cPickle.dump(model, open('model', 'w')) |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
745 return rval |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
746 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
747 And then you could run it this way: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
748 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
749 dbdict-run cmdline my_experiments.experiment \\ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
750 some_param=1 \\ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
751 other_param=0.4 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
752 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
753 Or this way: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
754 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
755 dbdict-run sqlschedule postgres://user:pass@host/dbname/tablename \\ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
756 my_experiments.experiment \\ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
757 some_param=1 \\ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
758 other_param=0.4 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
759 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
760 dbdict-run sql postgres://user:pass@host/dbname/tablename workdir |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
761 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
762 You need to make sure that the module `my_experiments` is accessible |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
763 from python. You can check with the command |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
764 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
765 $ python -m my_experiments |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
766 """ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
767 else: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
768 helptext = runner_registry.get(topic, None) |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
769 print format_help(helptext) |
| 566 | 770 |
| 771 runner_registry['help'] = help | |
| 565 | 772 |
| 566 | 773 ################################################################################ |
| 774 ### main | |
| 775 ################################################################################ | |
| 565 | 776 |
| 566 | 777 def run_cmdline(): |
| 778 try: | |
| 779 if len(sys.argv) <= 1: | |
| 780 raise UsageError('Usage: %s <run_type> [<arguments>*]' % sys.argv[0]) | |
| 781 run(sys.argv[1], sys.argv[2:]) | |
| 782 except UsageError, e: | |
| 783 print 'Usage error:' | |
| 784 print e | |
| 565 | 785 |
| 566 | 786 if __name__ == '__main__': |
| 787 run_cmdline() | |
| 565 | 788 |
| 789 |