Mercurial > ift6266

annotate deep/convolutional_dae/salah_exp/utils.py @ 456:66b05c6077c7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Ajout d'option pour choisir l'ensemble de donnees (train/valid/test) ainsi que mettre NIST,P07 et PNIST pour tous les modeles testes
author SylvainPL <sylvain.pannetier.lebeuf@umontreal.ca>
date Wed, 26 May 2010 20:24:16 -0400
parents 31641a84e0ae
children
rev   line source
358
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
1 #!/usr/bin/python
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
2 # coding: utf-8
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
3
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
4 from __future__ import with_statement
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
5
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
6 from jobman import DD
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
7
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
8 # from pylearn codebase
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
9 # useful in __init__(param1, param2, etc.) to save
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
10 # values in self.param1, self.param2... just call
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
11 # update_locals(self, locals())
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
12 def update_locals(obj, dct):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
13 if 'self' in dct:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
14 del dct['self']
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
15 obj.__dict__.update(dct)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
16
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
17 # from a dictionary of possible values for hyperparameters, e.g.
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
18 # hp_values = {'learning_rate':[0.1, 0.01], 'num_layers': [1,2]}
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
19 # create a list of other dictionaries representing all the possible
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
20 # combinations, thus in this example creating:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
21 # [{'learning_rate': 0.1, 'num_layers': 1}, ...]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
22 # (similarly for combinations (0.1, 2), (0.01, 1), (0.01, 2))
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
23 def produit_cartesien_jobs(val_dict):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
24 job_list = [DD()]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
25 all_keys = val_dict.keys()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
26
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
27 for key in all_keys:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
28 possible_values = val_dict[key]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
29 new_job_list = []
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
30 for val in possible_values:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
31 for job in job_list:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
32 to_insert = job.copy()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
33 to_insert.update({key: val})
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
34 new_job_list.append(to_insert)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
35 job_list = new_job_list
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
36
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
37 return job_list
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
38
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
39 def test_produit_cartesien_jobs():
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
40 vals = {'a': [1,2], 'b': [3,4,5]}
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
41 print produit_cartesien_jobs(vals)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
42
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
43
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
44 # taken from http://stackoverflow.com/questions/276052/how-to-get-current-cpu-and-ram-usage-in-python
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
45 """Simple module for getting amount of memory used by a specified user's
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
46 processes on a UNIX system.
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
47 It uses UNIX ps utility to get the memory usage for a specified username and
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
48 pipe it to awk for summing up per application memory usage and return the total.
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
49 Python's Popen() from subprocess module is used for spawning ps and awk.
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
50
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
51 """
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
52
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
53 import subprocess
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
54
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
55 class MemoryMonitor(object):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
56
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
57 def __init__(self, username):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
58 """Create new MemoryMonitor instance."""
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
59 self.username = username
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
60
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
61 def usage(self):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
62 """Return int containing memory used by user's processes."""
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
63 self.process = subprocess.Popen("ps -u %s -o rss | awk '{sum+=$1} END {print sum}'" % self.username,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
64 shell=True,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
65 stdout=subprocess.PIPE,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
66 )
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
67 self.stdout_list = self.process.communicate()[0].split('\n')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
68 return int(self.stdout_list[0])
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
69