|
371
|
1 #!/usr/bin/python
|
|
|
2 # coding: utf-8
|
|
|
3
|
|
|
4 from __future__ import with_statement
|
|
|
5
|
|
|
6 from jobman import DD
|
|
|
7
|
|
|
8 # from pylearn codebase
|
|
|
9 # useful in __init__(param1, param2, etc.) to save
|
|
|
10 # values in self.param1, self.param2... just call
|
|
|
11 # update_locals(self, locals())
|
|
|
12 def update_locals(obj, dct):
|
|
|
13 if 'self' in dct:
|
|
|
14 del dct['self']
|
|
|
15 obj.__dict__.update(dct)
|
|
|
16
|
|
|
17 # from a dictionary of possible values for hyperparameters, e.g.
|
|
|
18 # hp_values = {'learning_rate':[0.1, 0.01], 'num_layers': [1,2]}
|
|
|
19 # create a list of other dictionaries representing all the possible
|
|
|
20 # combinations, thus in this example creating:
|
|
|
21 # [{'learning_rate': 0.1, 'num_layers': 1}, ...]
|
|
|
22 # (similarly for combinations (0.1, 2), (0.01, 1), (0.01, 2))
|
|
|
23 def produit_cartesien_jobs(val_dict):
|
|
|
24 job_list = [DD()]
|
|
|
25 all_keys = val_dict.keys()
|
|
|
26
|
|
|
27 for key in all_keys:
|
|
|
28 possible_values = val_dict[key]
|
|
|
29 new_job_list = []
|
|
|
30 for val in possible_values:
|
|
|
31 for job in job_list:
|
|
|
32 to_insert = job.copy()
|
|
|
33 to_insert.update({key: val})
|
|
|
34 new_job_list.append(to_insert)
|
|
|
35 job_list = new_job_list
|
|
|
36
|
|
|
37 return job_list
|
|
|
38
|
|
|
39 def test_produit_cartesien_jobs():
|
|
|
40 vals = {'a': [1,2], 'b': [3,4,5]}
|
|
|
41 print produit_cartesien_jobs(vals)
|
|
|
42
|
|
|
43
|
|
|
44 # taken from http://stackoverflow.com/questions/276052/how-to-get-current-cpu-and-ram-usage-in-python
|
|
|
45 """Simple module for getting amount of memory used by a specified user's
|
|
|
46 processes on a UNIX system.
|
|
|
47 It uses UNIX ps utility to get the memory usage for a specified username and
|
|
|
48 pipe it to awk for summing up per application memory usage and return the total.
|
|
|
49 Python's Popen() from subprocess module is used for spawning ps and awk.
|
|
|
50
|
|
|
51 """
|
|
|
52
|
|
|
53 import subprocess
|
|
|
54
|
|
|
55 class MemoryMonitor(object):
|
|
|
56
|
|
|
57 def __init__(self, username):
|
|
|
58 """Create new MemoryMonitor instance."""
|
|
|
59 self.username = username
|
|
|
60
|
|
|
61 def usage(self):
|
|
|
62 """Return int containing memory used by user's processes."""
|
|
|
63 self.process = subprocess.Popen("ps -u %s -o rss | awk '{sum+=$1} END {print sum}'" % self.username,
|
|
|
64 shell=True,
|
|
|
65 stdout=subprocess.PIPE,
|
|
|
66 )
|
|
|
67 self.stdout_list = self.process.communicate()[0].split('\n')
|
|
|
68 return int(self.stdout_list[0])
|
|
|
69
|
