# HG changeset patch # User Xavier Glorot # Date 1249930720 14400 # Node ID a48ec81bec9d204d608a9f10a60ce4cfe9a83225 # Parent c61d775f4f9543417aadebbdc2761719b424bf61 independent auxiliary tie weigths behavior DAA inputs groups diff -r c61d775f4f95 -r a48ec81bec9d pylearn/algorithms/sandbox/DAA_inputs_groups.py --- a/pylearn/algorithms/sandbox/DAA_inputs_groups.py Thu Jul 30 18:09:17 2009 -0400 +++ b/pylearn/algorithms/sandbox/DAA_inputs_groups.py Mon Aug 10 14:58:40 2009 -0400 @@ -112,7 +112,7 @@ def __init__(self, input = None, auxinput = None, in_size=None, auxin_size= None, n_hid=1, - regularize = False, tie_weights = False, hid_fn = 'tanh_act', + regularize = False, tie_weights = False, tie_weights_aux = None, hid_fn = 'tanh_act', rec_fn = 'tanh_act',reconstruction_cost_function='cross_entropy', interface = True, ignore_missing=None, reconstruct_missing=False, corruption_pattern=None, **init): @@ -152,18 +152,6 @@ missing inputs will be backpropagated. Otherwise, it will not. :todo: Default noise level for all daa levels """ - print '\t\t**** DAAig.__init__ ****' - print '\t\tinput = ', input - print '\t\tauxinput = ', auxinput - print '\t\tin_size = ', in_size - print '\t\tauxin_size = ', auxin_size - print '\t\tn_hid = ', n_hid - print '\t\tregularize = ', regularize - print '\t\ttie_weights = ', tie_weights - print '\t\thid_fn = ', hid_fn - print '\t\trec_fn = ', rec_fn - print '\t\treconstruction_cost_function = ', reconstruction_cost_function - super(DAAig, self).__init__() self.random = T.RandomStreams() @@ -173,6 +161,7 @@ self.n_hid = n_hid self.regularize = regularize self.tie_weights = tie_weights + self.tie_weights_aux = tie_weights_aux if tie_weights_aux is not None else tie_weights self.interface = interface self.ignore_missing = ignore_missing self.reconstruct_missing = reconstruct_missing @@ -189,6 +178,19 @@ self.reconstruction_cost_function = eval(reconstruction_cost_function) self.reconstruction_cost_function_name = reconstruction_cost_function + print '\t\t**** DAAig.__init__ ****' + print '\t\tinput = ', input + print '\t\tauxinput = ', auxinput + print '\t\tin_size = ', self.in_size + print '\t\tauxin_size = ', self.auxin_size + print '\t\tn_hid = ', self.n_hid + print '\t\tregularize = ', self.regularize + print '\t\ttie_weights = ', self.tie_weights + print '\t\ttie_weights_aux = ', self.tie_weights_aux + print '\t\thid_fn = ', hid_fn + print '\t\trec_fn = ', rec_fn + print '\t\treconstruction_cost_function = ', reconstruction_cost_function + ### DECLARE MODEL VARIABLES and default self.input = input if self.ignore_missing is not None and self.input is not None: @@ -212,7 +214,8 @@ if self.auxinput is not None: self.wauxenc = [T.dmatrix('wauxenc%s'%i) for i in range(len(auxin_size))] - self.wauxdec = [self.wauxenc[i].T if tie_weights else T.dmatrix('wauxdec%s'%i) for i in range(len(auxin_size))] + self.wauxdec =[ self.wauxenc[i].T if tie_weights_aux else T.dmatrix('wauxdec%s'%i) for i in\ + range(len(auxin_size))] self.bauxdec = [T.dvector('bauxdec%s'%i) for i in range(len(auxin_size))] #hyper-parameters @@ -370,6 +373,7 @@ if not(self.tie_weights): if self.input is not None: self.params += [self.wdec] + if not(self.tie_weights_aux): if self.auxinput is not None: self.params += self.wauxdec @@ -391,8 +395,8 @@ self.update = theano.Method(listin, self.noise.cost, self.updates) self.compute_cost = theano.Method(listin, self.noise.cost) self.noisify = theano.Method(listin, listout) - self.recactivation = theano.Method(listin, self.clean.rec_activation) - self.reconstruction = theano.Method(listin, self.clean.rec) + self.recactivation = theano.Method(listin, self.noise.rec_activation) + self.reconstruction = theano.Method(listin, self.noise.rec) self.activation = theano.Method(listin, self.clean.hidden_activation) self.representation = theano.Method(listin, self.clean.hidden) @@ -441,12 +445,12 @@ wauxdecshp = [tuple(reversed(i)) for i in wauxencshp] obj.bauxdec = [numpy.zeros(i) for i in self.auxin_size] obj.wauxenc = [self.R.uniform(size=i, low = -self.inf, high = self.inf) for i in wauxencshp] - if not(self.tie_weights): + if not(self.tie_weights_aux): obj.wauxdec = [copy.copy(obj.wauxenc[i].T) for i in range(len(wauxdecshp))] if tieinit else\ [self.R.uniform(size=i, low=-self.hif, high=self.hif) for i in wauxdecshp] if orthoinit: obj.wauxenc = [orthogonalinit(w) for w in obj.wauxenc] - if not(self.tie_weights): + if not(self.tie_weights_aux): obj.wauxdec = [orthogonalinit(w,0) for w in obj.wauxdec] print 'wauxencshp = ', wauxencshp print 'wauxdecshp = ', wauxdecshp @@ -462,7 +466,7 @@ class StackedDAAig(module.Module): def __init__(self, depth = 1, input = T.dmatrix('input'), auxinput = [None], in_size = None, auxin_size = [None], n_hid = [1], - regularize = False, tie_weights = False, hid_fn = 'tanh_act', + regularize = False, tie_weights = False, tie_weights_aux = None, hid_fn = 'tanh_act', rec_fn = 'tanh_act',reconstruction_cost_function='cross_entropy', n_out = 2, target = None, debugmethod = False, totalupdatebool=False, ignore_missing=None, reconstruct_missing=False, @@ -487,6 +491,7 @@ self.n_hid = listify(n_hid,depth) self.regularize = regularize tie_weights = listify(tie_weights,depth) + tie_weights_aux = listify(tie_weights_aux,depth) hid_fn = listify(hid_fn,depth) rec_fn = listify(rec_fn,depth) reconstruction_cost_function = listify(reconstruction_cost_function,depth) @@ -507,6 +512,7 @@ print '\tn_hid = ', self.n_hid print '\tregularize = ', self.regularize print '\ttie_weights = ', tie_weights + print '\ttie_weights_aux = ', tie_weights_aux print '\thid_fn = ', hid_fn print '\trec_fn = ', rec_fn print '\treconstruction_cost_function = ', reconstruction_cost_function @@ -570,7 +576,8 @@ for i in range(self.depth): dict_params = dict(input = inputprec, in_size = in_sizeprec, auxin_size = auxin_size[i], - n_hid = self.n_hid[i], regularize = False, tie_weights = tie_weights[i], hid_fn = hid_fn[i], + n_hid = self.n_hid[i], regularize = False, tie_weights = tie_weights[i], + tie_weights_aux = tie_weights_aux[i], hid_fn = hid_fn[i], rec_fn = rec_fn[i], reconstruction_cost_function = reconstruction_cost_function[i], interface = False, ignore_missing = self.ignore_missing, reconstruct_missing = self.reconstruct_missing,corruption_pattern = self.corruption_pattern) @@ -634,8 +641,8 @@ if self.debugmethod: self.activation[i] = theano.Method(self.inputs[i],self.daaig[i].clean.hidden_activation) self.representation[i] = theano.Method(self.inputs[i],self.daaig[i].clean.hidden) - self.recactivation[i] = theano.Method(self.inputs[i],self.daaig[i].clean.rec_activation) - self.reconstruction[i] = theano.Method(self.inputs[i],self.daaig[i].clean.rec) + self.recactivation[i] = theano.Method(self.inputs[i],self.daaig[i].noise.rec_activation) + self.reconstruction[i] = theano.Method(self.inputs[i],self.daaig[i].noise.rec) self.noisyinputs[i] =theano.Method(self.inputs[i], noisyout) self.compute_localcost[i] = theano.Method(self.inputs[i],self.localcost[i]) self.compute_localgradients[i] = theano.Method(self.inputs[i],self.localgradients[i]) @@ -748,11 +755,12 @@ save_mat('wenc%s.ft'%(i) ,inst.daaig[i].wenc, save_dir) save_mat('bdec%s.ft'%(i) ,inst.daaig[i].bdec, save_dir) - if not self.daaig[i].tie_weights: + if not self.daaig[i].tie_weights_aux: if self.daaig[i].auxinput is not None: for j in range(len(inst.daaig[i].wauxdec)): save_mat('wauxdec%s_%s.ft'%(i,j) ,inst.daaig[i].wauxdec[j], save_dir) - + + if not self.daaig[i].tie_weights: if self.daaig[i].input is not None: save_mat('wdec%s.ft'%(i) ,inst.daaig[i].wdec, save_dir) i=i+1 @@ -778,7 +786,7 @@ inst.daaig[i].wenc = load_mat('wenc%s.ft'%(i),save_dir)/coefenc[i] inst.daaig[i].bdec = load_mat('bdec%s.ft'%(i),save_dir)/coefdec[i] - if not self.daaig[i].tie_weights: + if not self.daaig[i].tie_weights_aux: if self.daaig[i].auxinput is not None: for j in range(len(inst.daaig[i].wauxdec)): if 'wauxdec%s_%s.ft'%(i,j) in os.listdir(save_dir): @@ -786,7 +794,8 @@ else: print "WARNING: no decoding 'wauxdec%s_%s.ft' file use 'wauxenc%s_%s.ft' instead"%(i,j,i,j) inst.daaig[i].wauxdec[j] = numpy.transpose(load_mat('wauxenc%s_%s.ft'%(i,j),save_dir)/coefdec[i]) - + + if not self.daaig[i].tie_weights: if self.daaig[i].input is not None: if 'wdec%s.ft'%(i) in os.listdir(save_dir): inst.daaig[i].wdec = load_mat('wdec%s.ft'%(i),save_dir)/coefdec[i]