Mercurial > pylearn

comparison linear_regression.py @ 110:8fa1ef2411a0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Worked on OneShotTLearner and implementation of LinearRegression
author bengioy@bengiomac.local
date Tue, 06 May 2008 22:24:55 -0400
parents c4916445e025
children 88257dfedf8c
comparison
equal deleted inserted replaced
109:d97f6fe6bdf9 110:8fa1ef2411a0
53 input_dataset rather than those learned during 'update'; currently no support 53 input_dataset rather than those learned during 'update'; currently no support
54 for providing these to update): 54 for providing these to update):
55 55
56 - 'lambda' 56 - 'lambda'
57 - 'b' 57 - 'b'
58 - 'W' 58 - 'W'
59 - 'regularization_term' 59 - 'parameters' = (b, W) tuple
60 - 'regularization_term'
61 - 'XtX'
62 - 'XtY'
60 63
61 """ 64 """
62 65
63 def attributeNames(self): 66 def attributeNames(self):
64 return ["lambda","b","W","regularization_term"] 67 return ["lambda","parameters","b","W","regularization_term","XtX","XtY"]
68
69 def useInputAttributes(self):
70 return ["b","W"]
71
72 def useOutputAttributes(self):
73 return []
74
75 def updateInputAttributes(self):
76 return ["lambda","XtX","XtY"]
77
78 def updateOutputAttributes(self):
79 return ["parameters"] + self.updateMinibatchOutputAttributes() + self.updateEndOutputAttributes()
80
81 def updateMinibatchInputFields(self):
82 return ["input","target"]
65 83
84 def updateMinibatchInputAttributes(self):
85 return ["XtX","XtY"]
86
87 def updateMinibatchOutputAttributes(self):
88 return ["new_XtX","new_XtY"]
89
90 def updateEndInputAttributes(self):
91 return ["theta","XtX","XtY"]
66 92
67 def __init__(self): 93 def updateEndOutputAttributes(self):
68 self.input = t.matrix('input') # n_examples x n_inputs 94 return ["new_theta","b","W","regularization_term"] # CHECK: WILL b AND W CONTAIN OLD OR NEW THETA? @todo i.e. order of computation = ?
69 self.target = t.matrix('target') # n_examples x n_outputs
70 self.lambda = as_scalar(0.,'lambda')
71 self.theta = t.matrix('theta')
72 self.W = self.theta[:,1:]
73 self.b = self.theta[:,0]
74 self.XtX = t.matrix('XtX')
75 self.XtY = t.matrix('XtY')
76 self.regularizer = self.lambda * t.dot(self.W,self.W)
77 self.squared_error =
78 self.loss = self.regularizer + t.sum(self.squared_error) # this only makes sense if the whole training set fits in memory in a minibatch
79 self.loss_function = Function([self.W,self.lambda,self.squared_error],[self.loss])
80 self.new_XtX = self.XtX + t.dot(self.extended_input.T,self.extended_input)
81 self.new_XtY = self.XtY + t.dot(self.extended_input.T,self.target)
82 self.new_theta = t.solve(self.XtX,self.XtY)
83
84 def initialize(self):
85 self.XtX.resize((1+self.n_inputs,1+self.n_inputs))
86 self.XtY.resize((1+self.n_inputs,self.n_outputs))
87 self.XtX.data[:,:]=0
88 self.XtY.data[:,:]=0
89 numpy.diag(self.XtX.data)[1:]=self.lambda.data
90
91 def updated_variables(self):
92
93 def minibatch_wise_inputs(self):
94 def minibatch_wise_outputs(self):
95 # self.input is a (n_examples, n_inputs) minibatch matrix
96 self.extended_input = t.prepend_one_to_each_row(self.input)
97 self.output = t.dot(self.input,self.W.T) + self.b # (n_examples , n_outputs) matrix
98 self.squared_error = t.sum_within_rows(t.sqr(self.output-self.target)) # (n_examples ) vector
99
100 def attributeNames(self):
101 return ["lambda","b","W","regularization_term","XtX","XtY"]
102 95
103 def defaultOutputFields(self, input_fields): 96 def defaultOutputFields(self, input_fields):
104 output_fields = ["output"] 97 output_fields = ["output"]
105 if "target" in input_fields: 98 if "target" in input_fields:
106 output_fields.append("squared_error") 99 output_fields.append("squared_error")
107 return output_fields 100 return output_fields
108 101
109 # poutine generale basee sur ces fonctions 102 def __init__(self):
103 self._input = t.matrix('input') # n_examples x n_inputs
104 self._target = t.matrix('target') # n_examples x n_outputs
105 self._lambda = as_scalar(0.,'lambda')
106 self._theta = t.matrix('theta')
107 self._W = self._theta[:,1:]
108 self._b = self._theta[:,0]
109 self._XtX = t.matrix('XtX')
110 self._XtY = t.matrix('XtY')
111 self._extended_input = t.prepend_one_to_each_row(self._input)
112 self._output = t.dot(self._input,self._W.T) + self._b # (n_examples , n_outputs) matrix
113 self._squared_error = t.sum_within_rows(t.sqr(self._output-self._target)) # (n_examples ) vector
114 self._regularizer = self._lambda * t.dot(self._W,self._W)
115 self._new_XtX = add_inplace(self._XtX,t.dot(self._extended_input.T,self._extended_input))
116 self._new_XtY = add_inplace(self._XtY,t.dot(self._extended_input.T,self._target))
117 self._new_theta = t.solve_inplace(self._theta,self._XtX,self._XtY)
110 118
111 119 OneShotTLearner.__init__(self)
112 def __init__(self,lambda=0.,max_memory_use=500): 120 self.allocate()
113 """ 121
114 @type lambda: float 122 def allocate(self,minibatch):
115 @param lambda: regularization coefficient 123 minibatch_n_inputs = minibatch["input"].shape[1]
116 """ 124 minibatch_n_outputs = minibatch["target"].shape[1]
117 125 if not self._n_inputs:
118 W=t.matrix('W') 126 self._n_inputs = minibatch_n_inputs
119 # b is a broadcastable row vector (can be replicated into 127 self._n_outputs = minibatch_n_outputs
120 # as many rows as there are examples in the minibach) 128 self.XtX = numpy.zeros((1+self._n_inputs,1+self._n_inputs))
121 b=t.row('b') 129 self.XtY = numpy.zeros((1+self._n_inputs,self._n_outputs))
122 minibatch_input = t.matrix('input') # n_examples x n_inputs 130 self.theta = numpy.zeros((self._n_outputs,1+self._n_inputs))
123 minibatch_target = t.matrix('target') # n_examples x n_outputs 131 self.forget()
124 minibatch_output = t.dot(minibatch_input,W.T) + b # n_examples x n_outputs 132 elif self._n_inputs!=minibatch_n_inputs or self._n_outputs!=minibatch_n_outputs:
125 lambda = as_scalar(lambda) 133 # if the input or target changes dimension on the fly, we forget everything
126 regularizer = self.lambda * t.dot(W,W) 134 self.forget()
127 example_squared_error = t.sum_within_rows(t.sqr(minibatch_output-minibatch_target)) 135
128 self.output_function = Function([W,b,minibatch_input],[minibatch_output])
129 self.squared_error_function = Function([minibatch_output,minibatch_target],[self.example_squared_error])
130 self.loss_function = Function([W,squared_error],[self.regularizer + t.sum(self.example_squared_error)])
131 self.W=None
132 self.b=None
133 self.XtX=None
134 self.XtY=None
135
136 def forget(self): 136 def forget(self):
137 if self.W: 137 if self._n_inputs and self._n_outputs:
138 self.XtX *= 0 138 self.XtX.resize((1+self.n_inputs,1+self.n_inputs))
139 self.XtY *= 0 139 self.XtY.resize((1+self.n_inputs,self.n_outputs))
140 self.XtX.data[:,:]=0
141 self.XtY.data[:,:]=0
142 numpy.diag(self.XtX.data)[1:]=self.lambda
140 143
141 def use(self,input_dataset,output_fieldnames=None,copy_inputs=True): 144 def updateEnd(self):
142 input_fieldnames = input_dataset.fieldNames() 145 TLearner.updateEnd(self)
143 assert "input" in input_fieldnames 146 self.parameters = (self.W,self.b)
144 if not output_fields:
145 output_fields = ["output"]
146 if "target" in input_fieldnames:
147 output_fields += ["squared_error"]
148 else:
149 if "squared_error" in output_fields or "total_loss" in output_fields:
150 assert "target" in input_fieldnames
151 147
152 use_functions = []
153 for output_fieldname in output_fieldnames:
154 if output_fieldname=="output":
155 use_functions.append(self.output_function)
156 elif output_fieldname=="squared_error":
157 use_functions.append(lambda self.output_function)
158
159 n_examples = len(input_dataset)
160
161 for minibatch in input_dataset.minibatches(minibatch_size=minibatch_size, allow_odd_last_minibatch=True):
162 use_function(
163