Mercurial > pylearn

annotate bin/pkldu.py @ 1433:14ba52c38f07

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
removed import to file that don't exist in this repo.
author Frederic Bastien <nouiz@nouiz.org>
date Tue, 22 Feb 2011 10:04:32 -0500
parents ea5d27727869
children f88508a42a38
rev   line source
1423
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
1 #!/bin/env python
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
2 import sys
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
3 import cPickle
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
4 import time
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
5
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
6 """ Usage:
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
7 first argument is a cPickle file to load
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
8 if no more arguments are supplied, will analyze the disk usage of each element of the root-level object stored in the file
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
9 subsequent arguments let you index into fields / dictionary entries of the object
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
10 For example,
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
11 pkldu.py foo.pkl .my_field [my_key] 3
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
12 will load an object obj from foo.pkl and analyze obj.my_field["my_key"][3]
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
13 """
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
14
1433
14ba52c38f07 removed import to file that don't exist in this repo.
Frederic Bastien <nouiz@nouiz.org>
parents: 1423
diff changeset
15 def load(filepath):
14ba52c38f07 removed import to file that don't exist in this repo.
Frederic Bastien <nouiz@nouiz.org>
parents: 1423
diff changeset
16 f = open(filepath,'rb')
14ba52c38f07 removed import to file that don't exist in this repo.
Frederic Bastien <nouiz@nouiz.org>
parents: 1423
diff changeset
17 obj = cPickle.load(f)
14ba52c38f07 removed import to file that don't exist in this repo.
Frederic Bastien <nouiz@nouiz.org>
parents: 1423
diff changeset
18 f.close()
14ba52c38f07 removed import to file that don't exist in this repo.
Frederic Bastien <nouiz@nouiz.org>
parents: 1423
diff changeset
19 return obj
14ba52c38f07 removed import to file that don't exist in this repo.
Frederic Bastien <nouiz@nouiz.org>
parents: 1423
diff changeset
20
1423
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
21 filepath = sys.argv[1]
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
22
1433
14ba52c38f07 removed import to file that don't exist in this repo.
Frederic Bastien <nouiz@nouiz.org>
parents: 1423
diff changeset
23 orig_obj = load(filepath)
1423
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
24
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
25 cycle_check = {}
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
26
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
27 obj_name = 'root_obj'
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
28 cycle_check[id(orig_obj)] = obj_name
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
29
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
30 for field in sys.argv[2:]:
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
31 if field.startswith('['):
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
32 assert field.endswith(']')
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
33 obj_name += '[' + field[1:-1] + ']'
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
34 orig_obj = orig_obj[field[1:-1]]
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
35 elif field.startswith('.'):
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
36 obj_name += '.' + field
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
37 orig_obj = getattr(orig_obj,field[1:])
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
38 else:
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
39 obj_name + '[' + field + ']'
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
40 orig_obj = orig_obj[eval(field)]
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
41 if id(orig_obj) in cycle_check:
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
42 print "You're going in circles, "+obj_name+" is the same as "+cycle_check[id(orig_obj)]
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
43 quit()
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
44 cycle_check[id(orig_obj)] = obj_name
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
45
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
46 s = cPickle.dumps(orig_obj)
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
47 prev_bytes = len(s)
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
48 print 'orig_obj bytes: \t\t\t\t'+str(prev_bytes)
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
49 t1 = time.time()
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
50 x = cPickle.loads(s)
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
51 t2 = time.time()
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
52 prev_t = t2 - t1
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
53 print 'orig load time: '+str(prev_t)
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
54
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
55
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
56 idx = 0
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
57
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
58 while len(dir(orig_obj)) > idx:
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
59 stop = False
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
60
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
61 while True:
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
62 fields = dir(orig_obj)
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
63 if idx >= len(fields):
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
64 stop = True
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
65 break
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
66 field = fields[idx]
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
67
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
68 success = True
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
69 try:
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
70 delattr(orig_obj,field)
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
71
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
72 except:
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
73 print "got error trying to delete "+field
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
74 idx += 1
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
75 success = False
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
76 if success and field in dir(orig_obj):
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
77 print field + ' reappears after being deleted'
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
78 idx += 1
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
79 if success:
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
80 break
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
81
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
82 if stop:
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
83 break
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
84
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
85 s = cPickle.dumps(orig_obj)
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
86 new_bytes = len(s)
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
87 diff_bytes = prev_bytes - new_bytes
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
88 prev_bytes = new_bytes
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
89 t1 = time.time()
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
90 x = cPickle.loads(s)
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
91 t2 = time.time()
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
92 new_t = t2 - t1
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
93 diff_t = prev_t - new_t
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
94 prev_t = new_t
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
95 print field+': \t\t\t\t'+str(diff_bytes)+'\t\t\t'+str(diff_t)
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
96
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
97 if type(orig_obj) == type({}):
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
98 print 'orig_obj is a dictionary'
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
99
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
100 keys = [ key for key in orig_obj.keys() ]
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
101
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
102 for key in keys:
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
103 del orig_obj[key]
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
104
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
105 s = cPickle.dumps(orig_obj)
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
106 new_bytes = len(s)
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
107 t1 = time.time()
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
108 x = cPickle.loads(s)
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
109 t2 = time.time()
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
110 new_t = t2 - t1
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
111 diff_t = prev_t - new_t
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
112 prev_t = new_t
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
113 print field+': \t\t\t\t'+str(diff_bytes)+'\t\t\t'+str(diff_t)
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
114
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
115
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
116 if type(orig_obj) == type([]):
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
117 print 'orig_obj is a list'
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
118
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
119 i = 0
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
120 while len(orig_obj) > 0:
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
121 stringrep = str(orig_obj[0])
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
122 if len(stringrep) > 15:
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
123 stringrep = stringrep[0:12] + "..."
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
124 del orig_obj[0]
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
125
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
126 s = cPickle.dumps(orig_obj)
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
127 new_bytes = len(s)
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
128 diff_bytes = prev_bytes - new_bytes
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
129 prev_bytes = new_bytes
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
130
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
131 t1 = time.time()
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
132 x = cPickle.loads(s)
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
133 t2 = time.time()
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
134 new_t = t2 - t1
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
135 diff_t = prev_t - new_t
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
136 prev_t = new_t
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
137 print field+': \t\t\t\t'+str(diff_bytes)+'\t\t\t'+str(diff_t)
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
138
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
139
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
140 i+= 1
ea5d27727869 added pickle disk usage inspection utility 'pkldu'
Ian Goodfellow
parents:
diff changeset
141