Mercurial > ift6266

annotate deep/amt/amt.py @ 402:83413ac10913

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added more stats printing. Now you dont need to parameters which dataset you are testing, it will detect it automatically
author humel
date Wed, 28 Apr 2010 11:28:28 -0400
parents 86d5e583e278
children a11692910312
rev   line source
399
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
1 # Script usage : python amt.py filname.cvs
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
2
402
83413ac10913 Added more stats printing. Now you dont need to parameters which dataset you are testing, it will detect it automatically
humel
parents: 401
diff changeset
3 import csv,numpy,re,decimal
399
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
4
402
83413ac10913 Added more stats printing. Now you dont need to parameters which dataset you are testing, it will detect it automatically
humel
parents: 401
diff changeset
5 DATASET_PATH = { 'nist' : '/data/lisa/data/ift6266h10/amt_data/nist/',
83413ac10913 Added more stats printing. Now you dont need to parameters which dataset you are testing, it will detect it automatically
humel
parents: 401
diff changeset
6 'p07' : '/data/lisa/data/ift6266h10/amt_data/p07/',
83413ac10913 Added more stats printing. Now you dont need to parameters which dataset you are testing, it will detect it automatically
humel
parents: 401
diff changeset
7 'pnist' : '/data/lisa/data/ift6266h10/amt_data/pnist/' }
399
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
8
402
83413ac10913 Added more stats printing. Now you dont need to parameters which dataset you are testing, it will detect it automatically
humel
parents: 401
diff changeset
9 CVSFILE = None
83413ac10913 Added more stats printing. Now you dont need to parameters which dataset you are testing, it will detect it automatically
humel
parents: 401
diff changeset
10 #PATH = None
83413ac10913 Added more stats printing. Now you dont need to parameters which dataset you are testing, it will detect it automatically
humel
parents: 401
diff changeset
11 answer_labels = [ 'answer.c'+str(i+1) for i in range(10) ]
83413ac10913 Added more stats printing. Now you dont need to parameters which dataset you are testing, it will detect it automatically
humel
parents: 401
diff changeset
12 img_url = 'input.image_url'
399
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
13 turks_per_batch = 3
402
83413ac10913 Added more stats printing. Now you dont need to parameters which dataset you are testing, it will detect it automatically
humel
parents: 401
diff changeset
14 image_per_batch = 10
399
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
15
401
86d5e583e278 Fixed class number bug
humel
parents: 399
diff changeset
16 def setup_association():
86d5e583e278 Fixed class number bug
humel
parents: 399
diff changeset
17 answer_assoc = {}
86d5e583e278 Fixed class number bug
humel
parents: 399
diff changeset
18 for i in range(0,10):
86d5e583e278 Fixed class number bug
humel
parents: 399
diff changeset
19 answer_assoc[str(i)]=i
86d5e583e278 Fixed class number bug
humel
parents: 399
diff changeset
20 for i in range(10,36):
86d5e583e278 Fixed class number bug
humel
parents: 399
diff changeset
21 answer_assoc[chr(i+55)]=i
86d5e583e278 Fixed class number bug
humel
parents: 399
diff changeset
22 for i in range(36,62):
86d5e583e278 Fixed class number bug
humel
parents: 399
diff changeset
23 answer_assoc[chr(i+61)]=i
86d5e583e278 Fixed class number bug
humel
parents: 399
diff changeset
24 return answer_assoc
86d5e583e278 Fixed class number bug
humel
parents: 399
diff changeset
25
86d5e583e278 Fixed class number bug
humel
parents: 399
diff changeset
26 answer_assoc = setup_association()
86d5e583e278 Fixed class number bug
humel
parents: 399
diff changeset
27
399
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
28 def test_error():
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
29 turks = []
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
30 reader = csv.DictReader(open(CVSFILE), delimiter=',')
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
31 entries = [ turk for turk in reader ]
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
32
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
33 errors = numpy.zeros((len(entries),))
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
34 if len(entries) % turks_per_batch != 0 :
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
35 raise Exception('Wrong number of entries or turks_per_batch')
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
36
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
37 total_uniq_entries = len(entries) / turks_per_batch
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
38
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
39 errors = numpy.zeros((len(entries),))
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
40 error_means = numpy.zeros((total_uniq_entries,))
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
41 error_variances = numpy.zeros((total_uniq_entries,))
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
42
402
83413ac10913 Added more stats printing. Now you dont need to parameters which dataset you are testing, it will detect it automatically
humel
parents: 401
diff changeset
43 PATH = DATASET_PATH[find_dataset(entries[0])]
399
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
44 for i in range(total_uniq_entries):
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
45 for t in range(turks_per_batch):
402
83413ac10913 Added more stats printing. Now you dont need to parameters which dataset you are testing, it will detect it automatically
humel
parents: 401
diff changeset
46 errors[i*turks_per_batch+t] = get_error(entries[i*turks_per_batch+t],PATH)
399
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
47 error_means[i] = errors[i*turks_per_batch:(i+1)*turks_per_batch].mean()
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
48 error_variances[i] = errors[i*turks_per_batch:(i+1)*turks_per_batch].var()
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
49
402
83413ac10913 Added more stats printing. Now you dont need to parameters which dataset you are testing, it will detect it automatically
humel
parents: 401
diff changeset
50 decimal.getcontext().prec = 3
83413ac10913 Added more stats printing. Now you dont need to parameters which dataset you are testing, it will detect it automatically
humel
parents: 401
diff changeset
51 print 'Total entries : ' + str(len(entries))
83413ac10913 Added more stats printing. Now you dont need to parameters which dataset you are testing, it will detect it automatically
humel
parents: 401
diff changeset
52 print 'Turks per batch : ' + str(turks_per_batch)
83413ac10913 Added more stats printing. Now you dont need to parameters which dataset you are testing, it will detect it automatically
humel
parents: 401
diff changeset
53 print 'Average test error : ' + str(error_means.mean()*image_per_batch) +'%'
83413ac10913 Added more stats printing. Now you dont need to parameters which dataset you are testing, it will detect it automatically
humel
parents: 401
diff changeset
54 print 'Error variance : ' + str(error_variances.mean()*image_per_batch) +'%'
399
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
55
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
56
402
83413ac10913 Added more stats printing. Now you dont need to parameters which dataset you are testing, it will detect it automatically
humel
parents: 401
diff changeset
57 def find_dataset(entry):
83413ac10913 Added more stats printing. Now you dont need to parameters which dataset you are testing, it will detect it automatically
humel
parents: 401
diff changeset
58 file = parse_filename(entry[img_url])
83413ac10913 Added more stats printing. Now you dont need to parameters which dataset you are testing, it will detect it automatically
humel
parents: 401
diff changeset
59 return file.split('_')[0]
83413ac10913 Added more stats printing. Now you dont need to parameters which dataset you are testing, it will detect it automatically
humel
parents: 401
diff changeset
60
83413ac10913 Added more stats printing. Now you dont need to parameters which dataset you are testing, it will detect it automatically
humel
parents: 401
diff changeset
61 def get_error(entry,PATH):
399
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
62 file = parse_filename(entry[img_url])
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
63 f = open(PATH+file,'r')
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
64 labels = re.sub("\s+", "",f.readline()).strip()[1:-2].split('.')
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
65 f.close()
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
66 test_error = 0
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
67 for i in range(len(answer_labels)):
401
86d5e583e278 Fixed class number bug
humel
parents: 399
diff changeset
68 answer = entry[answer_labels[i]]
86d5e583e278 Fixed class number bug
humel
parents: 399
diff changeset
69 if len(answer) != 0:
86d5e583e278 Fixed class number bug
humel
parents: 399
diff changeset
70 try:
86d5e583e278 Fixed class number bug
humel
parents: 399
diff changeset
71 if answer_assoc[answer] != int(labels[i]):
86d5e583e278 Fixed class number bug
humel
parents: 399
diff changeset
72 test_error+=1
399
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
73 except:
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
74 test_error+=1
401
86d5e583e278 Fixed class number bug
humel
parents: 399
diff changeset
75 else:
399
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
76 test_error+=1
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
77 return test_error
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
78
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
79
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
80 def parse_filename(string):
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
81 filename = string.split('/')[-1]
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
82 return filename.split('.')[0]+'.txt'
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
83
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
84 if __name__ =='__main__':
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
85 import sys
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
86 CVSFILE = sys.argv[1]
99905d9bc9dd Initial commit for calculating the test error of the AMT classifier
humel
parents:
diff changeset
87 test_error()