Mercurial > ift6266

annotate deep/convolutional_dae/salah_exp/nist_csda.py @ 358:31641a84e0ae

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Initial commit for the experimental setup of the denoising convolutional network
author humel
date Thu, 22 Apr 2010 00:49:42 -0400
parents
children c05680f8c92f
rev   line source
358
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
1 #!/usr/bin/python
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
2 # coding: utf-8
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
3
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
4 import ift6266
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
5 import pylearn
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
6
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
7 import numpy
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
8 import theano
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
9 import time
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
10
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
11 import pylearn.version
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
12 import theano.tensor as T
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
13 from theano.tensor.shared_randomstreams import RandomStreams
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
14
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
15 import copy
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
16 import sys
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
17 import os
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
18 import os.path
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
19
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
20 from jobman import DD
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
21 import jobman, jobman.sql
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
22 from pylearn.io import filetensor
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
23
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
24 from utils import produit_cartesien_jobs
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
25 from copy import copy
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
26
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
27 from sgd_optimization_new import CSdASgdOptimizer
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
28
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
29 #from ift6266.utils.scalar_series import *
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
30 from ift6266.utils.seriestables import *
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
31 import tables
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
32
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
33 from ift6266 import datasets
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
34 from config import *
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
35
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
36 '''
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
37 Function called by jobman upon launching each job
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
38 Its path is the one given when inserting jobs: see EXPERIMENT_PATH
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
39 '''
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
40 def jobman_entrypoint(state, channel):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
41 # record mercurial versions of each package
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
42 pylearn.version.record_versions(state,[theano,ift6266,pylearn])
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
43 # TODO: remove this, bad for number of simultaneous requests on DB
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
44 channel.save()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
45
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
46 # For test runs, we don't want to use the whole dataset so
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
47 # reduce it to fewer elements if asked to.
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
48 rtt = None
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
49 #REDUCE_TRAIN_TO = 40000
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
50 if state.has_key('reduce_train_to'):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
51 rtt = state['reduce_train_to']
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
52 elif REDUCE_TRAIN_TO:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
53 rtt = REDUCE_TRAIN_TO
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
54
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
55 if state.has_key('decrease_lr'):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
56 decrease_lr = state['decrease_lr']
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
57 else :
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
58 decrease_lr = 0
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
59
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
60 n_ins = 32*32
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
61 n_outs = 62 # 10 digits, 26*2 (lower, capitals)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
62
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
63 examples_per_epoch = 100000#NIST_ALL_TRAIN_SIZE
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
64
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
65 #To be sure variables will not be only in the if statement
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
66 PATH = ''
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
67 nom_reptrain = ''
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
68 nom_serie = ""
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
69 if state['pretrain_choice'] == 0:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
70 nom_serie="series_NIST.h5"
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
71 elif state['pretrain_choice'] == 1:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
72 nom_serie="series_P07.h5"
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
73
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
74 series = create_series(state.num_hidden_layers,nom_serie)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
75
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
76
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
77 print "Creating optimizer with state, ", state
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
78
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
79 optimizer = CSdASgdOptimizer(dataset=datasets.nist_all(),
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
80 hyperparameters=state, \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
81 n_ins=n_ins, n_outs=n_outs,\
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
82 examples_per_epoch=examples_per_epoch, \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
83 series=series,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
84 max_minibatches=rtt)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
85
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
86 parameters=[]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
87 #Number of files of P07 used for pretraining
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
88 nb_file=0
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
89 if state['pretrain_choice'] == 0:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
90 print('\n\tpretraining with NIST\n')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
91 optimizer.pretrain(datasets.nist_all())
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
92 elif state['pretrain_choice'] == 1:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
93 #To know how many file will be used during pretraining
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
94 nb_file = int(state['pretraining_epochs_per_layer'])
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
95 state['pretraining_epochs_per_layer'] = 1 #Only 1 time over the dataset
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
96 if nb_file >=100:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
97 sys.exit("The code does not support this much pretraining epoch (99 max with P07).\n"+
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
98 "You have to correct the code (and be patient, P07 is huge !!)\n"+
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
99 "or reduce the number of pretraining epoch to run the code (better idea).\n")
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
100 print('\n\tpretraining with P07')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
101 optimizer.pretrain(datasets.nist_P07(min_file=0,max_file=nb_file))
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
102 channel.save()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
103
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
104 #Set some of the parameters used for the finetuning
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
105 if state.has_key('finetune_set'):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
106 finetune_choice=state['finetune_set']
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
107 else:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
108 finetune_choice=FINETUNE_SET
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
109
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
110 if state.has_key('max_finetuning_epochs'):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
111 max_finetune_epoch_NIST=state['max_finetuning_epochs']
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
112 else:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
113 max_finetune_epoch_NIST=MAX_FINETUNING_EPOCHS
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
114
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
115 if state.has_key('max_finetuning_epochs_P07'):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
116 max_finetune_epoch_P07=state['max_finetuning_epochs_P07']
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
117 else:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
118 max_finetune_epoch_P07=max_finetune_epoch_NIST
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
119
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
120 #Decide how the finetune is done
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
121
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
122 if finetune_choice == 0:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
123 print('\n\n\tfinetune with NIST\n\n')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
124 optimizer.reload_parameters('params_pretrain.txt')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
125 optimizer.finetune(datasets.nist_all(),datasets.nist_P07(),max_finetune_epoch_NIST,ind_test=1,decrease=decrease_lr)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
126 channel.save()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
127 if finetune_choice == 1:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
128 print('\n\n\tfinetune with P07\n\n')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
129 optimizer.reload_parameters('params_pretrain.txt')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
130 optimizer.finetune(datasets.nist_P07(),datasets.nist_all(),max_finetune_epoch_P07,ind_test=0,decrease=decrease_lr)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
131 channel.save()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
132 if finetune_choice == 2:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
133 print('\n\n\tfinetune with P07 followed by NIST\n\n')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
134 optimizer.reload_parameters('params_pretrain.txt')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
135 optimizer.finetune(datasets.nist_P07(),datasets.nist_all(),max_finetune_epoch_P07,ind_test=20,decrease=decrease_lr)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
136 optimizer.finetune(datasets.nist_all(),datasets.nist_P07(),max_finetune_epoch_NIST,ind_test=21,decrease=decrease_lr)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
137 channel.save()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
138 if finetune_choice == 3:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
139 print('\n\n\tfinetune with NIST only on the logistic regression on top (but validation on P07).\n\
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
140 All hidden units output are input of the logistic regression\n\n')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
141 optimizer.reload_parameters('params_pretrain.txt')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
142 optimizer.finetune(datasets.nist_all(),datasets.nist_P07(),max_finetune_epoch_NIST,ind_test=1,special=1,decrease=decrease_lr)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
143
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
144
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
145 if finetune_choice==-1:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
146 print('\nSERIE OF 4 DIFFERENT FINETUNINGS')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
147 print('\n\n\tfinetune with NIST\n\n')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
148 sys.stdout.flush()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
149 optimizer.reload_parameters('params_pretrain.txt')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
150 optimizer.finetune(datasets.nist_all(),datasets.nist_P07(),max_finetune_epoch_NIST,ind_test=1,decrease=decrease_lr)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
151 channel.save()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
152 print('\n\n\tfinetune with P07\n\n')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
153 sys.stdout.flush()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
154 optimizer.reload_parameters('params_pretrain.txt')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
155 optimizer.finetune(datasets.nist_P07(),datasets.nist_all(),max_finetune_epoch_P07,ind_test=0,decrease=decrease_lr)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
156 channel.save()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
157 print('\n\n\tfinetune with P07 (done earlier) followed by NIST (written here)\n\n')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
158 sys.stdout.flush()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
159 optimizer.reload_parameters('params_finetune_P07.txt')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
160 optimizer.finetune(datasets.nist_all(),datasets.nist_P07(),max_finetune_epoch_NIST,ind_test=21,decrease=decrease_lr)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
161 channel.save()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
162 print('\n\n\tfinetune with NIST only on the logistic regression on top.\n\
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
163 All hidden units output are input of the logistic regression\n\n')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
164 sys.stdout.flush()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
165 optimizer.reload_parameters('params_pretrain.txt')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
166 optimizer.finetune(datasets.nist_all(),datasets.nist_P07(),max_finetune_epoch_NIST,ind_test=1,special=1,decrease=decrease_lr)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
167 channel.save()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
168
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
169 channel.save()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
170
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
171 return channel.COMPLETE
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
172
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
173 # These Series objects are used to save various statistics
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
174 # during the training.
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
175 def create_series(num_hidden_layers, nom_serie):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
176
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
177 # Replace series we don't want to save with DummySeries, e.g.
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
178 # series['training_error'] = DummySeries()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
179
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
180 series = {}
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
181
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
182 basedir = os.getcwd()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
183
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
184 h5f = tables.openFile(os.path.join(basedir, nom_serie), "w")
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
185
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
186 #REDUCE_EVERY=10
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
187 # reconstruction
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
188 reconstruction_base = \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
189 ErrorSeries(error_name="reconstruction_error",
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
190 table_name="reconstruction_error",
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
191 hdf5_file=h5f,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
192 index_names=('epoch','minibatch'),
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
193 title="Reconstruction error (mean over "+str(REDUCE_EVERY)+" minibatches)")
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
194 series['reconstruction_error'] = \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
195 AccumulatorSeriesWrapper(base_series=reconstruction_base,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
196 reduce_every=REDUCE_EVERY)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
197
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
198 # train
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
199 training_base = \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
200 ErrorSeries(error_name="training_error",
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
201 table_name="training_error",
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
202 hdf5_file=h5f,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
203 index_names=('epoch','minibatch'),
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
204 title="Training error (mean over "+str(REDUCE_EVERY)+" minibatches)")
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
205 series['training_error'] = \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
206 AccumulatorSeriesWrapper(base_series=training_base,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
207 reduce_every=REDUCE_EVERY)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
208
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
209 # valid and test are not accumulated/mean, saved directly
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
210 series['validation_error'] = \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
211 ErrorSeries(error_name="validation_error",
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
212 table_name="validation_error",
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
213 hdf5_file=h5f,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
214 index_names=('epoch','minibatch'))
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
215
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
216 series['test_error'] = \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
217 ErrorSeries(error_name="test_error",
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
218 table_name="test_error",
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
219 hdf5_file=h5f,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
220 index_names=('epoch','minibatch'))
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
221
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
222 param_names = []
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
223 for i in range(num_hidden_layers):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
224 param_names += ['layer%d_W'%i, 'layer%d_b'%i, 'layer%d_bprime'%i]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
225 param_names += ['logreg_layer_W', 'logreg_layer_b']
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
226
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
227 # comment out series we don't want to save
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
228 series['params'] = SharedParamsStatisticsWrapper(
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
229 new_group_name="params",
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
230 base_group="/",
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
231 arrays_names=param_names,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
232 hdf5_file=h5f,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
233 index_names=('epoch',))
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
234
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
235 return series
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
236
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
237 # Perform insertion into the Postgre DB based on combination
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
238 # of hyperparameter values above
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
239 # (see comment for produit_cartesien_jobs() to know how it works)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
240 def jobman_insert_nist():
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
241 jobs = produit_cartesien_jobs(JOB_VALS)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
242
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
243 db = jobman.sql.db(JOBDB)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
244 for job in jobs:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
245 job.update({jobman.sql.EXPERIMENT: EXPERIMENT_PATH})
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
246 jobman.sql.insert_dict(job, db)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
247
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
248 print "inserted"
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
249
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
250 if __name__ == '__main__':
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
251
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
252 args = sys.argv[1:]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
253
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
254 #if len(args) > 0 and args[0] == 'load_nist':
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
255 # test_load_nist()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
256
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
257 if len(args) > 0 and args[0] == 'jobman_insert':
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
258 jobman_insert_nist()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
259
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
260 elif len(args) > 0 and args[0] == 'test_jobman_entrypoint':
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
261 chanmock = DD({'COMPLETE':0,'save':(lambda:None)})
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
262 jobman_entrypoint(DD(DEFAULT_HP_NIST), chanmock)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
263
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
264 else:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
265 print "Bad arguments"
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
266