Mercurial > pylearn
annotate pylearn/dbdict/newstuff.py @ 589:6a7f3d83c72b
dbdict stuff
| author | James Bergstra <bergstrj@iro.umontreal.ca> |
|---|---|
| date | Wed, 17 Dec 2008 15:39:30 -0500 |
| parents | 5020b12e87ee |
| children | f8d29730f146 |
| rev | line source |
|---|---|
| 565 | 1 |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
2 from __future__ import with_statement |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
3 |
| 565 | 4 from collections import defaultdict |
|
572
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
5 import re, sys, inspect, os, signal, tempfile, shutil, socket |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
6 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
7 import sql |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
8 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
9 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
10 ################################################################################ |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
11 ### misc |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
12 ################################################################################ |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
13 |
| 589 | 14 class DD(dict): |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
15 def __getattr__(self, attr): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
16 return self[attr] |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
17 def __setattr__(self, attr, value): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
18 self[attr] = value |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
19 def __str__(self): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
20 return 'DD%s' % dict(self) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
21 def __repr__(self): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
22 return str(self) |
| 565 | 23 |
| 24 ################################################################################ | |
| 25 ### resolve | |
| 26 ################################################################################ | |
| 27 | |
| 28 def resolve(name): | |
| 29 symbols = name.split('.') | |
| 30 builder = __import__(symbols[0]) | |
| 31 for sym in symbols[1:]: | |
| 32 builder = getattr(builder, sym) | |
| 33 return builder | |
| 34 | |
| 35 ################################################################################ | |
| 36 ### dictionary | |
| 37 ################################################################################ | |
| 38 | |
| 39 def convert(obj): | |
|
572
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
40 try: |
|
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
41 return eval(obj, {}, {}) |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
42 except (NameError, SyntaxError): |
|
572
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
43 return obj |
| 565 | 44 |
| 45 def flatten(obj): | |
| 589 | 46 """nested dictionary -> flat dictionary with '.' notation """ |
| 565 | 47 d = {} |
| 48 def helper(d, prefix, obj): | |
| 49 if isinstance(obj, (str, int, float)): | |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
50 d[prefix] = obj #convert(obj) |
| 565 | 51 else: |
| 52 if isinstance(obj, dict): | |
| 53 subd = obj | |
| 54 else: | |
| 55 subd = obj.state() | |
| 56 subd['__builder__'] = '%s.%s' % (obj.__module__, obj.__class__.__name__) | |
| 57 for k, v in subd.iteritems(): | |
| 58 pfx = '.'.join([prefix, k]) if prefix else k | |
| 59 helper(d, pfx, v) | |
| 60 helper(d, '', obj) | |
| 61 return d | |
| 62 | |
| 63 def expand(d): | |
| 589 | 64 """inverse of flatten()""" |
| 65 #def dd(): | |
| 66 #return DD(dd) | |
| 67 struct = DD() | |
| 565 | 68 for k, v in d.iteritems(): |
| 69 if k == '': | |
| 70 raise NotImplementedError() | |
| 71 else: | |
| 72 keys = k.split('.') | |
| 73 current = struct | |
| 74 for k2 in keys[:-1]: | |
| 589 | 75 current = current.setdefault(k2, DD()) |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
76 current[keys[-1]] = v #convert(v) |
| 565 | 77 return struct |
| 78 | |
| 79 def realize(d): | |
| 80 if not isinstance(d, dict): | |
| 81 return d | |
| 82 d = dict((k, realize(v)) for k, v in d.iteritems()) | |
| 83 if '__builder__' in d: | |
| 84 builder = resolve(d.pop('__builder__')) | |
| 85 return builder(**d) | |
| 86 return d | |
| 87 | |
| 88 def make(d): | |
| 89 return realize(expand(d)) | |
| 90 | |
| 91 ################################################################################ | |
| 92 ### errors | |
| 93 ################################################################################ | |
| 94 | |
| 95 class UsageError(Exception): | |
| 96 pass | |
| 97 | |
| 98 ################################################################################ | |
| 99 ### parsing and formatting | |
| 100 ################################################################################ | |
| 101 | |
| 102 def parse(*strings): | |
| 103 d = {} | |
| 104 for string in strings: | |
| 105 s1 = re.split(' *= *', string, 1) | |
| 106 s2 = re.split(' *:: *', string, 1) | |
| 107 if len(s1) == 1 and len(s2) == 1: | |
| 108 raise UsageError('Expected a keyword argument in place of "%s"' % s1[0]) | |
| 109 elif len(s1) == 2: | |
| 110 k, v = s1 | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
111 v = convert(v) |
| 565 | 112 elif len(s2) == 2: |
| 113 k, v = s2 | |
| 114 k += '.__builder__' | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
115 d[k] = v |
| 565 | 116 return d |
| 117 | |
| 566 | 118 def format_d(d, sep = '\n', space = True): |
| 119 d = flatten(d) | |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
120 pattern = "%s = %r" if space else "%s=%r" |
| 566 | 121 return sep.join(pattern % (k, v) for k, v in d.iteritems()) |
| 122 | |
| 565 | 123 def format_help(topic): |
| 124 if topic is None: | |
| 125 return 'No help.' | |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
126 elif isinstance(topic, str): |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
127 help = topic |
| 565 | 128 elif hasattr(topic, 'help'): |
| 129 help = topic.help() | |
| 130 else: | |
| 131 help = topic.__doc__ | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
132 if not help: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
133 return 'No help.' |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
134 |
| 565 | 135 ss = map(str.rstrip, help.split('\n')) |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
136 try: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
137 baseline = min([len(line) - len(line.lstrip()) for line in ss if line]) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
138 except: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
139 return 'No help.' |
| 565 | 140 s = '\n'.join([line[baseline:] for line in ss]) |
| 141 s = re.sub(string = s, pattern = '\n{2,}', repl = '\n\n') | |
| 142 s = re.sub(string = s, pattern = '(^\n*)|(\n*$)', repl = '') | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
143 |
| 565 | 144 return s |
| 145 | |
| 146 ################################################################################ | |
| 566 | 147 ### single channels |
| 565 | 148 ################################################################################ |
| 149 | |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
150 # try: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
151 # import greenlet |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
152 # except: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
153 # try: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
154 # from py import greenlet |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
155 # except: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
156 # print >>sys.stderr, 'the greenlet module is unavailable' |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
157 # greenlet = None |
| 566 | 158 |
| 159 | |
| 565 | 160 class Channel(object): |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
161 |
| 589 | 162 COMPLETE = property(lambda s:None, |
| 163 doc=("Experiments should return this value to " | |
| 164 "indicate that they are done (if not done, return `Incomplete`")) | |
| 165 INCOMPLETE = property(lambda s:True, | |
| 166 doc=("Experiments should return this value to indicate that " | |
| 167 "they are not done (if done return `COMPLETE`)")) | |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
168 |
| 589 | 169 START = property(lambda s: 0, |
| 170 doc="dbdict.status == START means a experiment is ready to run") | |
| 171 RUNNING = property(lambda s: 1, | |
| 172 doc="dbdict.status == RUNNING means a experiment is running on dbdict_hostname") | |
| 173 DONE = property(lambda s: 2, | |
| 174 doc="dbdict.status == DONE means a experiment has completed (not necessarily successfully)") | |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
175 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
176 # Methods to be used by the experiment to communicate with the channel |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
177 |
| 566 | 178 def save(self): |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
179 """ |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
180 Save the experiment's state to the various media supported by |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
181 the Channel. |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
182 """ |
| 565 | 183 raise NotImplementedError() |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
184 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
185 def switch(self, message = None): |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
186 """ |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
187 Called from the experiment to give the control back to the channel. |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
188 The following return values are meaningful: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
189 * 'stop' -> the experiment must stop as soon as possible. It may save what |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
190 it needs to save. This occurs when SIGTERM or SIGINT are sent (or in |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
191 user-defined circumstances). |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
192 switch() may give the control to the user. In this case, the user may |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
193 resume the experiment by calling switch() again. If an argument is given |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
194 by the user, it will be relayed to the experiment. |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
195 """ |
| 566 | 196 pass |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
197 |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
198 def __call__(self, message = None): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
199 return self.switch(message) |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
200 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
201 def save_and_switch(self): |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
202 self.save() |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
203 self.switch() |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
204 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
205 # Methods to run the experiment |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
206 |
| 566 | 207 def setup(self): |
| 208 pass | |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
209 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
210 def __enter__(self): |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
211 pass |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
212 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
213 def __exit__(self): |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
214 pass |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
215 |
| 566 | 216 def run(self): |
| 565 | 217 pass |
| 218 | |
| 219 | |
| 220 | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
221 class JobError(Exception): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
222 RUNNING = 0 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
223 DONE = 1 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
224 NOJOB = 2 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
225 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
226 |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
227 class SingleChannel(Channel): |
| 566 | 228 |
| 229 def __init__(self, experiment, state): | |
| 230 self.experiment = experiment | |
| 231 self.state = state | |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
232 self.feedback = None |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
233 |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
234 def switch(self, message = None): |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
235 feedback = self.feedback |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
236 self.feedback = None |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
237 return feedback |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
238 |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
239 def run(self, force = False): |
| 566 | 240 self.setup() |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
241 |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
242 status = self.state.dbdict.get('status', self.START) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
243 if status is self.DONE and not force: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
244 # If you want to disregard this, use the --force flag (not yet implemented) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
245 raise JobError(JobError.RUNNING, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
246 'The job has already completed.') |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
247 elif status is self.RUNNING and not force: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
248 raise JobError(JobError.DONE, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
249 'The job is already running.') |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
250 self.state.dbdict.status = self.RUNNING |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
251 |
|
572
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
252 v = self.COMPLETE |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
253 with self: |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
254 try: |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
255 v = self.experiment(self.state, self) |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
256 finally: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
257 self.state.dbdict.status = self.DONE if v is self.COMPLETE else self.START |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
258 |
| 566 | 259 return v |
| 260 | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
261 def on_sigterm(self, signo, frame): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
262 # SIGTERM handler. It is the experiment function's responsibility to |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
263 # call switch() often enough to get this feedback. |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
264 self.feedback = 'stop' |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
265 |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
266 def __enter__(self): |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
267 # install a SIGTERM handler that asks the experiment function to return |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
268 # the next time it will call switch() |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
269 self.prev_sigterm = signal.getsignal(signal.SIGTERM) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
270 self.prev_sigint = signal.getsignal(signal.SIGINT) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
271 signal.signal(signal.SIGTERM, self.on_sigterm) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
272 signal.signal(signal.SIGINT, self.on_sigterm) |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
273 return self |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
274 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
275 def __exit__(self, type, value, traceback): |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
276 signal.signal(signal.SIGTERM, self.prev_sigterm) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
277 signal.signal(signal.SIGINT, self.prev_sigint) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
278 self.prev_sigterm = None |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
279 self.prev_sigint = None |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
280 self.save() |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
281 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
282 |
| 566 | 283 class StandardChannel(SingleChannel): |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
284 |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
285 def __init__(self, path, experiment, state, redirect_stdout = False, redirect_stderr = False): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
286 super(StandardChannel, self).__init__(experiment, state) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
287 self.path = os.path.realpath(path) |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
288 self.redirect_stdout = redirect_stdout |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
289 self.redirect_stderr = redirect_stderr |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
290 |
| 566 | 291 def save(self): |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
292 with open(os.path.join(self.path, 'current.conf'), 'w') as current: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
293 current.write(format_d(self.state)) |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
294 current.write('\n') |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
295 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
296 def __enter__(self): |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
297 self.old_cwd = os.getcwd() |
| 566 | 298 os.chdir(self.path) |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
299 if self.redirect_stdout: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
300 self.old_stdout = sys.stdout |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
301 sys.stdout = open('stdout', 'a') |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
302 if self.redirect_stderr: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
303 self.old_stderr = sys.stderr |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
304 sys.stderr = open('stderr', 'a') |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
305 return super(StandardChannel, self).__enter__() |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
306 |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
307 def __exit__(self, type, value, traceback): |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
308 if self.redirect_stdout: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
309 sys.stdout.close() |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
310 sys.stdout = self.old_stdout |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
311 if self.redirect_stderr: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
312 sys.stderr.close() |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
313 sys.stderr = self.old_stderr |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
314 os.chdir(self.old_cwd) |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
315 return super(StandardChannel, self).__exit__(type, value, traceback) |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
316 |
| 566 | 317 def setup(self): |
| 318 if not os.path.isdir(self.path): | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
319 os.makedirs(self.path) |
|
567
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
320 with self: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
321 origf = os.path.join(self.path, 'orig.conf') |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
322 if not os.path.isfile(origf): |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
323 with open(origf, 'w') as orig: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
324 orig.write(format_d(self.state)) |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
325 orig.write('\n') |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
326 currentf = os.path.join(self.path, 'current.conf') |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
327 if os.path.isfile(currentf): |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
328 with open(currentf, 'r') as current: |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
329 self.state = expand(parse(*map(str.strip, current.readlines()))) |
|
d88c35e8f83a
another checkpoint
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
566
diff
changeset
|
330 |
| 566 | 331 |
| 332 class RSyncException(Exception): | |
| 333 pass | |
| 334 | |
| 335 class RSyncChannel(StandardChannel): | |
| 336 | |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
337 def __init__(self, path, remote_path, experiment, state, redirect_stdout = False, redirect_stderr = False): |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
338 super(RSyncChannel, self).__init__(path, experiment, state, redirect_stdout, redirect_stderr) |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
339 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
340 ssh_prefix='ssh://' |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
341 if remote_path.startswith(ssh_prefix): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
342 remote_path = remote_path[len(ssh_prefix):] |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
343 colon_pos = remote_path.find(':') |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
344 self.host = remote_path[:colon_pos] |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
345 self.remote_path = remote_path[colon_pos+1:] |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
346 else: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
347 self.host = '' |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
348 self.remote_path = os.path.realpath(remote_path) |
| 566 | 349 |
| 350 def rsync(self, direction): | |
| 351 """The directory at which experiment-related files are stored. | |
| 352 """ | |
| 353 | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
354 path = self.path |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
355 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
356 remote_path = self.remote_path |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
357 if self.host: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
358 remote_path = ':'.join([self.host, remote_path]) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
359 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
360 # TODO: use something more portable than os.system |
| 566 | 361 if direction == 'push': |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
362 rsync_cmd = 'rsync -ac "%s/" "%s/"' % (path, remote_path) |
| 566 | 363 elif direction == 'pull': |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
364 rsync_cmd = 'rsync -ac "%s/" "%s/"' % (remote_path, path) |
| 566 | 365 else: |
| 366 raise RSyncException('invalid direction', direction) | |
| 367 | |
| 368 rsync_rval = os.system(rsync_cmd) | |
| 369 if rsync_rval != 0: | |
| 370 raise RSyncException('rsync failure', (rsync_rval, rsync_cmd)) | |
| 371 | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
372 def touch(self): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
373 if self.host: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
374 host = self.host |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
375 touch_cmd = ('ssh %(host)s "mkdir -p \'%(path)s\'"' % dict(host = self.host, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
376 path = self.remote_path)) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
377 else: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
378 touch_cmd = ("mkdir -p '%(path)s'" % dict(path = self.remote_path)) |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
379 # print "ECHO", touch_cmd |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
380 touch_rval = os.system(touch_cmd) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
381 if 0 != touch_rval: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
382 raise Exception('touch failure', (touch_rval, touch_cmd)) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
383 |
| 566 | 384 def pull(self): |
| 385 return self.rsync('pull') | |
| 386 | |
| 387 def push(self): | |
| 388 return self.rsync('push') | |
| 389 | |
| 390 def save(self): | |
| 391 super(RSyncChannel, self).save() | |
| 392 self.push() | |
| 393 | |
| 394 def setup(self): | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
395 self.touch() |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
396 self.pull() |
| 566 | 397 super(RSyncChannel, self).setup() |
| 398 | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
399 |
| 566 | 400 class DBRSyncChannel(RSyncChannel): |
| 401 | |
|
572
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
402 RESTART_PRIORITY = 2.0 |
|
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
403 |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
404 def __init__(self, username, password, hostname, dbname, tablename, path, remote_root, redirect_stdout = False, redirect_stderr = False): |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
405 self.username, self.password, self.hostname, self.dbname, self.tablename \ |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
406 = username, password, hostname, dbname, tablename |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
407 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
408 self.db = sql.postgres_serial( |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
409 user = self.username, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
410 password = self.password, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
411 host = self.hostname, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
412 database = self.dbname, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
413 table_prefix = self.tablename) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
414 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
415 self.dbstate = sql.book_dct_postgres_serial(self.db) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
416 if self.dbstate is None: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
417 raise JobError(JobError.NOJOB, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
418 'No job was found to run.') |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
419 |
|
572
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
420 try: |
|
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
421 state = expand(self.dbstate) |
|
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
422 experiment = resolve(state.dbdict.experiment) |
|
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
423 remote_path = os.path.join(remote_root, self.dbname, self.tablename, str(self.dbstate.id)) |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
424 super(DBRSyncChannel, self).__init__(path, remote_path, experiment, state, redirect_stdout, redirect_stderr) |
|
572
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
425 except: |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
426 self.dbstate['dbdict.status'] = self.DONE |
|
572
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
427 raise |
| 566 | 428 |
| 429 def save(self): | |
| 430 super(DBRSyncChannel, self).save() | |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
431 self.dbstate.update(flatten(self.state)) |
| 566 | 432 |
| 433 def setup(self): | |
| 434 # Extract a single experiment from the table that is not already running. | |
| 435 # set self.experiment and self.state | |
| 436 super(DBRSyncChannel, self).setup() | |
|
572
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
437 self.state.dbdict.sql.host_name = socket.gethostname() |
|
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
438 self.state.dbdict.sql.host_workdir = self.path |
|
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
439 self.dbstate.update(flatten(self.state)) |
| 566 | 440 |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
441 def run(self): |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
442 # We pass the force flag as True because the status flag is |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
443 # already set to RUNNING by book_dct in __init__ |
|
572
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
444 v = super(DBRSyncChannel, self).run(force = True) |
|
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
445 if v is self.INCOMPLETE and self.state.dbdict.sql.priority != self.RESTART_PRIORITY: |
|
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
446 self.state.dbdict.sql.priority = self.RESTART_PRIORITY |
|
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
447 self.save() |
|
9f5891cd4048
added host name and workdir to the db
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
571
diff
changeset
|
448 return v |
| 565 | 449 |
| 450 | |
| 451 | |
| 566 | 452 ################################################################################ |
| 453 ### running | |
| 454 ################################################################################ | |
| 565 | 455 |
|
584
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
456 import optparse |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
457 OptionParser = optparse.OptionParser |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
458 # class OptionParser(optparse.OptionParser): |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
459 # def error(self, message): |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
460 # pass |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
461 |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
462 def parse_and_run(command, arguments): |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
463 parser, runner = runner_registry.get(command, (None, None)) |
| 566 | 464 if not runner: |
|
584
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
465 raise UsageError('Unknown runner: "%s"' % command) |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
466 if parser: |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
467 options, arguments = parser.parse_args(arguments) |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
468 else: |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
469 options = optparse.Values() |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
470 run(runner, [options] + arguments) |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
471 |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
472 def run(runner, arguments): |
| 566 | 473 argspec = inspect.getargspec(runner) |
| 474 minargs = len(argspec[0])-(len(argspec[3]) if argspec[3] else 0) | |
| 475 maxargs = len(argspec[0]) | |
| 476 if minargs > len(arguments) or maxargs < len(arguments) and not argspec[1]: | |
| 477 s = format_help(runner) | |
| 478 raise UsageError(s) | |
| 479 runner(*arguments) | |
| 565 | 480 |
| 566 | 481 runner_registry = dict() |
| 565 | 482 |
|
584
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
483 |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
484 parser_cmdline = OptionParser(usage = '%prog cmdline [options] <experiment> <parameters>') |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
485 parser_cmdline.add_option('-f', '--force', action = 'store_true', dest = 'force', default = False, |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
486 help = 'force running the experiment even if it is already running or completed') |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
487 parser_cmdline.add_option('--redirect-stdout', action = 'store_true', dest = 'redirect_stdout', default = False, |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
488 help = 'redirect stdout to the workdir/stdout file') |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
489 parser_cmdline.add_option('--redirect-stderr', action = 'store_true', dest = 'redirect_stderr', default = False, |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
490 help = 'redirect stderr to the workdir/stdout file') |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
491 parser_cmdline.add_option('-r', '--redirect', action = 'store_true', dest = 'redirect', default = False, |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
492 help = 'redirect stdout and stderr to the workdir/stdout and workdir/stderr files') |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
493 parser_cmdline.add_option('-w', '--workdir', action = 'store', dest = 'workdir', default = None, |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
494 help = 'the working directory in which to run the experiment') |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
495 |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
496 def runner_cmdline(options, experiment, *strings): |
| 566 | 497 """ |
| 498 Start an experiment with parameters given on the command line. | |
| 499 | |
|
584
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
500 Usage: cmdline [options] <experiment> <parameters> |
| 565 | 501 |
| 566 | 502 Run an experiment with parameters provided on the command |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
503 line. See the help topics for experiment and parameters for |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
504 syntax information. |
| 565 | 505 |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
506 Example use: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
507 dbdict-run cmdline mymodule.my_experiment \\ |
| 566 | 508 stopper::pylearn.stopper.nsteps \\ # use pylearn.stopper.nsteps |
| 509 stopper.n=10000 \\ # the argument "n" of nsteps is 10000 | |
| 510 lr=0.03 | |
| 511 """ | |
| 512 state = expand(parse(*strings)) | |
| 589 | 513 state.setdefault('dbdict', DD()).experiment = experiment |
| 566 | 514 experiment = resolve(experiment) |
|
584
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
515 workdir = options.workdir or format_d(state, sep=',', space = False) |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
516 channel = StandardChannel(workdir, |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
517 experiment, state, |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
518 redirect_stdout = options.redirect or options.redirect_stdout, |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
519 redirect_stderr = options.redirect or options.redirect_stderr) |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
520 channel.run(force = options.force) |
| 565 | 521 |
|
584
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
522 runner_registry['cmdline'] = (parser_cmdline, runner_cmdline) |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
523 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
524 |
|
584
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
525 |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
526 parser_sqlschedule = OptionParser(usage = '%prog sqlschedule [options] <tablepath> <experiment> <parameters>') |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
527 |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
528 def runner_sqlschedule(options, dbdescr, experiment, *strings): |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
529 """ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
530 Schedule a job to run using the sql command. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
531 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
532 Usage: sqlschedule <tablepath> <experiment> <parameters> |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
533 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
534 See the experiment and parameters topics for more information about |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
535 these parameters. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
536 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
537 Assuming that a postgres database is running on `host`, contains a |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
538 database called `dbname` and that `user` has the permissions to |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
539 create, read and modify tables on that database, tablepath should |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
540 be of the following form: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
541 postgres://user:pass@host/dbname/tablename |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
542 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
543 If no table is named `tablename`, one will be created |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
544 automatically. The state corresponding to the experiment and |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
545 parameters specified in the command will be saved in the database, |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
546 but no experiment will be run. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
547 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
548 To run an experiment scheduled using sqlschedule, see the sql |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
549 command. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
550 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
551 Example use: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
552 dbdict-run sqlschedule postgres://user:pass@host/dbname/tablename \\ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
553 mymodule.my_experiment \\ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
554 stopper::pylearn.stopper.nsteps \\ # use pylearn.stopper.nsteps |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
555 stopper.n=10000 \\ # the argument "n" of nsteps is 10000 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
556 lr=0.03 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
557 """ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
558 |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
559 try: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
560 username, password, hostname, dbname, tablename \ |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
561 = sql.parse_dbstring(dbdescr) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
562 except: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
563 raise UsageError('Wrong syntax for dbdescr') |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
564 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
565 db = sql.postgres_serial( |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
566 user = username, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
567 password = password, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
568 host = hostname, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
569 database = dbname, |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
570 table_prefix = tablename) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
571 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
572 state = parse(*strings) |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
573 try: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
574 resolve(experiment) |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
575 except: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
576 raise UsageError('The first parameter to sqlschedule must be a valid, importable symbol.') |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
577 state['dbdict.experiment'] = experiment |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
578 sql.add_experiments_to_db([state], db, verbose = 1) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
579 |
|
584
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
580 runner_registry['sqlschedule'] = (parser_sqlschedule, runner_sqlschedule) |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
581 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
582 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
583 |
|
584
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
584 parser_sql = OptionParser(usage = '%prog sql [options] <tablepath> <exproot>') |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
585 parser_sql.add_option('-n', dest = 'n', type = 'int', default = 1, |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
586 help = 'Run N experiments sequentially (default 1) ' |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
587 '(if N is <= 0, runs as many experiments as possible).') |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
588 |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
589 def runner_sql(options, dbdescr, exproot): |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
590 """ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
591 Run jobs from a sql table. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
592 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
593 Usage: sql <tablepath> <exproot> |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
594 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
595 The jobs should be scheduled first with the sqlschedule command. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
596 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
597 Assuming that a postgres database is running on `host`, contains a |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
598 database called `dbname` and that `user` has the permissions to |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
599 create, read and modify tables on that database, tablepath should |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
600 be of the following form: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
601 postgres://user:pass@host/dbname/tablename |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
602 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
603 exproot can be a local path or a remote path. Examples of exproots: |
|
584
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
604 /some/local/path |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
605 ssh://some_host:/some/remote/path # relative to the filesystem root |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
606 ssh://some_host:other/remote/path # relative to the HOME on some_host |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
607 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
608 The exproot will contain a subdirectory hierarchy corresponding to |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
609 the dbname, tablename and job id which is a unique integer. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
610 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
611 The sql runner will pick any job in the table which is not running |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
612 and is not done and will terminate when that job ends. You may call |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
613 the same command multiple times, sequentially or in parallel, to |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
614 run as many unfinished jobs as have been scheduled in that table |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
615 with sqlschedule. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
616 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
617 Example use: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
618 dbdict-run sql \\ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
619 postgres://user:pass@host/dbname/tablename \\ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
620 ssh://central_host:myexperiments |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
621 """ |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
622 try: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
623 username, password, hostname, dbname, tablename \ |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
624 = sql.parse_dbstring(dbdescr) |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
625 except: |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
626 raise UsageError('Wrong syntax for dbdescr') |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
627 |
|
584
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
628 n = options.n if options.n else -1 |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
629 nrun = 0 |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
630 try: |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
631 while n != 0: |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
632 workdir = tempfile.mkdtemp() |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
633 #print 'wdir', workdir |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
634 channel = DBRSyncChannel(username, password, hostname, dbname, tablename, |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
635 workdir, |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
636 exproot, |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
637 redirect_stdout = True, |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
638 redirect_stderr = True) |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
639 channel.run() |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
640 shutil.rmtree(workdir, ignore_errors=True) |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
641 n -= 1 |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
642 nrun += 1 |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
643 except JobError, e: |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
644 if e.args[0] == JobError.NOJOB: |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
645 print 'No more jobs to run (run %i jobs)' % nrun |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
646 |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
647 runner_registry['sql'] = (parser_sql, runner_sql) |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
648 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
649 |
|
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
650 |
| 565 | 651 |
| 652 | |
|
584
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
653 def runner_help(options, topic = None): |
| 566 | 654 """ |
| 655 Get help for a topic. | |
| 565 | 656 |
| 566 | 657 Usage: help <topic> |
| 658 """ | |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
659 def bold(x): |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
660 return '\033[1m%s\033[0m' % x |
| 566 | 661 if topic is None: |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
662 print bold('Topics: (use help <topic> for more info)') |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
663 print 'example Example of defining and running an experiment.' |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
664 print 'experiment How to define an experiment.' |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
665 print 'parameters How to list the parameters for an experiment.' |
|
568
1f036d934ad9
improvements to dbdict interface
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
567
diff
changeset
|
666 print |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
667 print bold('Available commands: (use help <command> for more info)') |
| 566 | 668 for name, command in sorted(runner_registry.iteritems()): |
| 669 print name.ljust(20), format_help(command).split('\n')[0] | |
| 670 return | |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
671 elif topic == 'experiment': |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
672 helptext = """ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
673 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
674 dbdict-run serves to run experiments. To define an experiment, you |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
675 only have to define a function respecting the following protocol in |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
676 a python file or module: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
677 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
678 def my_experiment(state, channel): |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
679 # experiment code goes here |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
680 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
681 The return value of my_experiment may be channel.COMPLETE or |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
682 channel.INCOMPLETE. If the latter is returned, the experiment may |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
683 be resumed at a later point. Note that the return value `None` |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
684 is interpreted as channel.COMPLETE. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
685 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
686 If a command defined by dbdict-run has an <experiment> parameter, |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
687 that parameter must be a string such that it could be used in a |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
688 python import statement to import the my_experiment function. For |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
689 example if you defined my_experiment in my_module.py, you can pass |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
690 'my_module.my_experiment' as the experiment parameter. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
691 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
692 When entering my_experiment, the current working directory will be |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
693 set for you to a directory specially created for the experiment. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
694 The location and name of that directory vary depending on which |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
695 dbdict-run command you run. You may create logs, save files, pictures, |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
696 results, etc. in it. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
697 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
698 state is an object containing the parameters given to the experiment. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
699 For example, if you run the followinc command: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
700 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
701 dbdict-run cmdline my_module.my_experiment a.x=6 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
702 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
703 `state.a.x` will contain the integer 6, and so will `state['a']['x']`. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
704 If the state is changed, it will be saved when the experiment ends |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
705 or when channel.save() is called. The next time the experiment is run |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
706 with the same working directory, the modified state will be provided. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
707 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
708 It is not recommended to store large amounts of data in the state. It |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
709 should be limited to scalar or string parameters. Results such as |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
710 weight matrices should be stored in files in the working directory. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
711 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
712 channel is an object with the following important methods: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
713 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
714 - channel.switch() (or channel()) will give the control back to the |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
715 user, if it is appropriate to do so. If a call to channel.switch() |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
716 returns the string 'stop', it typically means that the signal |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
717 SIGTERM (or SIGINT) was received. Therefore, the experiment may be |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
718 killed soon, so it should save and return True or |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
719 channel.INCOMPLETE so it can be resumed later. This should be |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
720 checked periodically or data loss may be incurred. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
721 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
722 - channel.save() will save the current state. It is automatically |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
723 called when the function returns, but it is a good idea to do it |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
724 periodically. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
725 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
726 - channel.save_and_switch() is an useful shortcut to do both operations |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
727 described above. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
728 """ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
729 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
730 elif topic == 'parameters': |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
731 helptext = """ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
732 If a command takes <parameters> arguments, the arguments should each |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
733 take one of the following forms: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
734 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
735 key=value |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
736 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
737 Set a parameter with name `key` to `value`. The value will be casted |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
738 to an appropriate type automatically and it will be accessible to |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
739 the experiment using `state.key`. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
740 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
741 If `key` is a dotted name, the value will be set in nested |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
742 dictionaries corresponding to each part. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
743 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
744 Examples: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
745 a=1 state.a <- 1 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
746 b=2.3 state.b <- 2.3 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
747 c.d="hello" state.c.d <- "hello" |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
748 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
749 key::builder |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
750 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
751 This is equivalent to key.__builder__=builder. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
752 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
753 The builder should be a symbol that can be used with import or |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
754 __import__ and it should be callable. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
755 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
756 If a key has a builder defined, the experiment code may easily make |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
757 an object out of it using the `make` function. `obj = make(state.key)`. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
758 This will call the builder on the substate corresponding to state.key, |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
759 as will be made clear in the example: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
760 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
761 Example: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
762 regexp::re.compile |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
763 regexp.pattern='a.*c' |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
764 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
765 from pylearn.dbdict.newstuff import make |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
766 def experiment(state, channel): |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
767 regexp = make(state.regexp) # regexp is now re.compile(pattern = 'a.*c') |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
768 print regexp.sub('blahblah', 'hello abbbbc there') |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
769 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
770 If the above experiment was called with the state produced by the |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
771 parameters in the example, it would print 'hello blahblah there'. |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
772 """ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
773 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
774 elif topic == 'example': |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
775 helptext = """ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
776 Example of an experiment that trains some model for 100000 iterations: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
777 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
778 # defined in: my_experiments.py |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
779 def experiment(state, channel): |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
780 try: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
781 model = cPickle.load(open('model', 'r')) |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
782 except: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
783 model = my_model(state.some_param, state.other_param) |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
784 state.n = 0 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
785 dataset = my_dataset(skipto = state.n) |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
786 for i in xrange(100000 - state.n): |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
787 model.update(dataset.next()) |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
788 if i and i % 1000 == 0: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
789 if channel.save_and_switch() == 'stop': |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
790 state.n += i + 1 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
791 rval = channel.INCOMPLETE |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
792 break |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
793 else: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
794 state.result = model.cost(some_test_set) |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
795 rval = channel.COMPLETE |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
796 cPickle.dump(model, open('model', 'w')) |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
797 return rval |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
798 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
799 And then you could run it this way: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
800 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
801 dbdict-run cmdline my_experiments.experiment \\ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
802 some_param=1 \\ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
803 other_param=0.4 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
804 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
805 Or this way: |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
806 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
807 dbdict-run sqlschedule postgres://user:pass@host/dbname/tablename \\ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
808 my_experiments.experiment \\ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
809 some_param=1 \\ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
810 other_param=0.4 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
811 |
|
584
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
812 dbdict-run sql postgres://user:pass@host/dbname/tablename exproot |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
813 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
814 You need to make sure that the module `my_experiments` is accessible |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
815 from python. You can check with the command |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
816 |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
817 $ python -m my_experiments |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
818 """ |
|
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
819 else: |
|
586
5020b12e87ee
fixed broken help
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
584
diff
changeset
|
820 helptext = runner_registry.get(topic, None)[1] |
|
576
ef424abb7458
bugfixes and a lot of documentation
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
572
diff
changeset
|
821 print format_help(helptext) |
| 566 | 822 |
|
584
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
823 runner_registry['help'] = (None, runner_help) |
| 565 | 824 |
| 566 | 825 ################################################################################ |
| 826 ### main | |
| 827 ################################################################################ | |
| 565 | 828 |
| 566 | 829 def run_cmdline(): |
| 830 try: | |
| 831 if len(sys.argv) <= 1: | |
| 832 raise UsageError('Usage: %s <run_type> [<arguments>*]' % sys.argv[0]) | |
|
584
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
833 cmd = None |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
834 args = [] |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
835 for arg in sys.argv[1:]: |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
836 if cmd is not None or arg.startswith('-'): |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
837 args.append(arg) |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
838 else: |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
839 cmd = arg |
|
9f5691d5587d
added command line options
Olivier Breuleux <breuleuo@iro.umontreal.ca>
parents:
579
diff
changeset
|
840 parse_and_run(cmd, args) |
| 566 | 841 except UsageError, e: |
| 842 print 'Usage error:' | |
| 843 print e | |
| 565 | 844 |
| 566 | 845 if __name__ == '__main__': |
| 846 run_cmdline() | |
| 565 | 847 |
| 848 |