annotate deep/stacked_dae/v_youssouf/utils.py @ 507:b8e33d3d7f65

merge
author Yoshua Bengio <bengioy@iro.umontreal.ca>
date Tue, 01 Jun 2010 13:57:16 -0400
parents 8cf52a1c8055
children
rev   line source
371
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
1 #!/usr/bin/python
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
2 # coding: utf-8
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
3
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
4 from __future__ import with_statement
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
5
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
6 from jobman import DD
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
7
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
8 # from pylearn codebase
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
9 # useful in __init__(param1, param2, etc.) to save
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
10 # values in self.param1, self.param2... just call
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
11 # update_locals(self, locals())
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
12 def update_locals(obj, dct):
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
13 if 'self' in dct:
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
14 del dct['self']
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
15 obj.__dict__.update(dct)
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
16
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
17 # from a dictionary of possible values for hyperparameters, e.g.
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
18 # hp_values = {'learning_rate':[0.1, 0.01], 'num_layers': [1,2]}
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
19 # create a list of other dictionaries representing all the possible
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
20 # combinations, thus in this example creating:
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
21 # [{'learning_rate': 0.1, 'num_layers': 1}, ...]
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
22 # (similarly for combinations (0.1, 2), (0.01, 1), (0.01, 2))
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
23 def produit_cartesien_jobs(val_dict):
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
24 job_list = [DD()]
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
25 all_keys = val_dict.keys()
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
26
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
27 for key in all_keys:
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
28 possible_values = val_dict[key]
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
29 new_job_list = []
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
30 for val in possible_values:
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
31 for job in job_list:
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
32 to_insert = job.copy()
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
33 to_insert.update({key: val})
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
34 new_job_list.append(to_insert)
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
35 job_list = new_job_list
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
36
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
37 return job_list
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
38
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
39 def test_produit_cartesien_jobs():
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
40 vals = {'a': [1,2], 'b': [3,4,5]}
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
41 print produit_cartesien_jobs(vals)
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
42
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
43
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
44 # taken from http://stackoverflow.com/questions/276052/how-to-get-current-cpu-and-ram-usage-in-python
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
45 """Simple module for getting amount of memory used by a specified user's
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
46 processes on a UNIX system.
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
47 It uses UNIX ps utility to get the memory usage for a specified username and
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
48 pipe it to awk for summing up per application memory usage and return the total.
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
49 Python's Popen() from subprocess module is used for spawning ps and awk.
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
50
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
51 """
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
52
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
53 import subprocess
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
54
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
55 class MemoryMonitor(object):
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
56
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
57 def __init__(self, username):
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
58 """Create new MemoryMonitor instance."""
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
59 self.username = username
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
60
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
61 def usage(self):
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
62 """Return int containing memory used by user's processes."""
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
63 self.process = subprocess.Popen("ps -u %s -o rss | awk '{sum+=$1} END {print sum}'" % self.username,
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
64 shell=True,
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
65 stdout=subprocess.PIPE,
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
66 )
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
67 self.stdout_list = self.process.communicate()[0].split('\n')
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
68 return int(self.stdout_list[0])
8cf52a1c8055 initial commit of sda with 36 classes
youssouf
parents:
diff changeset
69