annotate writeup/mlj_submission/mlj_submission.tex @ 615:337253b82409

repair the class/fct that allow to read pnist07 and others by allowing them to read gziped file.
author Frederic Bastien <nouiz@nouiz.org>
date Fri, 07 Jan 2011 11:44:23 -0500
parents 8bd4ff0c5c05
children
rev   line source
587
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1 \RequirePackage{fix-cm} % from template
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
2
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
3 %\documentclass{article} % For LaTeX2e
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
4 \documentclass[smallcondensed]{svjour3} % onecolumn (ditto)
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
5
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
6 \usepackage{times}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
7 \usepackage{wrapfig}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
8 %\usepackage{amsthm} % not to be used with springer tools
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
9 \usepackage{amsmath}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
10 \usepackage{bbm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
11 \usepackage[psamsfonts]{amssymb}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
12 %\usepackage{algorithm,algorithmic} % not used after all
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
13 \usepackage[utf8]{inputenc}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
14 \usepackage{graphicx,subfigure}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
15 \usepackage{natbib} % was [numbers]{natbib}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
16
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
17 \addtolength{\textwidth}{10mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
18 \addtolength{\evensidemargin}{-5mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
19 \addtolength{\oddsidemargin}{-5mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
20
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
21 %\setlength\parindent{0mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
22
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
23 \title{Deep Self-Taught Learning for Handwritten Character Recognition}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
24 \author{
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
25 Yoshua Bengio \and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
26 Frédéric Bastien \and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
27 Arnaud Bergeron \and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
28 Nicolas Boulanger-Lewandowski \and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
29 Thomas Breuel \and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
30 Youssouf Chherawala \and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
31 Moustapha Cisse \and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
32 Myriam Côté \and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
33 Dumitru Erhan \and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
34 Jeremy Eustache \and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
35 Xavier Glorot \and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
36 Xavier Muller \and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
37 Sylvain Pannetier Lebeuf \and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
38 Razvan Pascanu \and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
39 Salah Rifai \and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
40 Francois Savard \and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
41 Guillaume Sicard
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
42 }
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
43 \date{September 30th, submission to MLJ special issue on learning from multi-label data}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
44 \journalname{Machine Learning Journal}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
45 \institute{Frédéric Bastien \and \\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
46 Yoshua Bengio \and \\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
47 Arnaud Bergeron \and \\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
48 Nicolas Boulanger-Lewandowski \and \\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
49 Youssouf Chherawala \and \\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
50 Moustapha Cisse \and \\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
51 Myriam Côté \and \\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
52 Dumitru Erhan \and \\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
53 Jeremy Eustache \and \\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
54 Xavier Glorot \and \\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
55 Xavier Muller \and \\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
56 Sylvain Pannetier-Lebeuf \and \\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
57 Razvan Pascanu \and \\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
58 Salah Rifai \and \\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
59 Francois Savard \and \\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
60 Guillaume Sicard \at
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
61 Dept. IRO, Universite de Montreal, C.P. 6128, Montreal, QC, H3C 3J7, Canada\\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
62 \email{yoshua.bengio@umontreal.ca}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
63 \and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
64 Thomas Breuel \at
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
65 Department of Computer Science, University of Kaiserslautern, Postfach 3049, 67653 Kaiserslautern, Germany
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
66 }
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
67
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
68
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
69 \begin{document}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
70
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
71 %\makeanontitle
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
72 \maketitle
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
73
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
74 %\vspace*{-2mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
75 \begin{abstract}
591
8bd4ff0c5c05 Correction de merge
fsavard
parents: 587
diff changeset
76 Recent theoretical and empirical work in statistical machine learning has demonstrated the potential of learning algorithms for deep architectures, i.e., function classes obtained by composing multiple levels of representation. Self-taught learning (exploiting unlabeled examples or examples from other distributions) has already been applied to deep learners, but mostly to show the advantage of unlabeled examples. Here we explore the advantage brought by {\em out-of-distribution examples}. For this purpose we developed a powerful generator of stochastic variations and noise processes for character images, including not only affine transformations but also slant, local elastic deformations, changes in thickness, background images, grey level changes, contrast, occlusion, and various types of noise. The out-of-distribution examples are obtained from these highly distorted images or by including examples of object classes different from those in the target test set. We show that {\em deep learners benefit more from out-of-distribution examples than a corresponding shallow learner}, at least in the area of handwritten character recognition. In fact, we show that they beat previously published results and reach human-level performance on both handwritten digit classification and 62-class handwritten character recognition.
587
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
77 \end{abstract}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
78 %\vspace*{-3mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
79
591
8bd4ff0c5c05 Correction de merge
fsavard
parents: 587
diff changeset
80 \keywords{self-taught learning \and multi-task learning \and out-of-distribution examples \and deep learning \and handwriting recognition}
587
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
81
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
82 \section{Introduction}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
83 %\vspace*{-1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
84
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
85 {\bf Deep Learning} has emerged as a promising new area of research in
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
86 statistical machine learning (see \citet{Bengio-2009} for a review).
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
87 Learning algorithms for deep architectures are centered on the learning
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
88 of useful representations of data, which are better suited to the task at hand,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
89 and are organized in a hierarchy with multiple levels.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
90 This is in part inspired by observations of the mammalian visual cortex,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
91 which consists of a chain of processing elements, each of which is associated with a
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
92 different representation of the raw visual input. In fact,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
93 it was found recently that the features learnt in deep architectures resemble
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
94 those observed in the first two of these stages (in areas V1 and V2
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
95 of visual cortex) \citep{HonglakL2008}, and that they become more and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
96 more invariant to factors of variation (such as camera movement) in
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
97 higher layers~\citep{Goodfellow2009}.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
98 Learning a hierarchy of features increases the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
99 ease and practicality of developing representations that are at once
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
100 tailored to specific tasks, yet are able to borrow statistical strength
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
101 from other related tasks (e.g., modeling different kinds of objects). Finally, learning the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
102 feature representation can lead to higher-level (more abstract, more
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
103 general) features that are more robust to unanticipated sources of
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
104 variance extant in real data.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
105
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
106 {\bf Self-taught learning}~\citep{RainaR2007} is a paradigm that combines principles
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
107 of semi-supervised and multi-task learning: the learner can exploit examples
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
108 that are unlabeled and possibly come from a distribution different from the target
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
109 distribution, e.g., from other classes than those of interest.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
110 It has already been shown that deep learners can clearly take advantage of
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
111 unsupervised learning and unlabeled examples~\citep{Bengio-2009,WestonJ2008-small},
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
112 but more needs to be done to explore the impact
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
113 of {\em out-of-distribution} examples and of the {\em multi-task} setting
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
114 (one exception is~\citep{CollobertR2008}, which uses a different kind
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
115 of learning algorithm). In particular the {\em relative
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
116 advantage of deep learning} for these settings has not been evaluated.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
117 The hypothesis discussed in the conclusion is that in the context of
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
118 multi-task learning and the availability of out-of-distribution training examples,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
119 a deep hierarchy of features
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
120 may be better able to provide sharing of statistical strength
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
121 between different regions in input space or different tasks, compared to
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
122 a shallow learner.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
123
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
124 Whereas a deep architecture can in principle be more powerful than a
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
125 shallow one in terms of representation, depth appears to render the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
126 training problem more difficult in terms of optimization and local minima.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
127 It is also only recently that successful algorithms were proposed to
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
128 overcome some of these difficulties. All are based on unsupervised
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
129 learning, often in an greedy layer-wise ``unsupervised pre-training''
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
130 stage~\citep{Bengio-2009}. One of these layer initialization techniques,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
131 applied here, is the Denoising
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
132 Auto-encoder~(DA)~\citep{VincentPLarochelleH2008-very-small} (see Figure~\ref{fig:da}),
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
133 which
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
134 performed similarly or better than previously proposed Restricted Boltzmann
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
135 Machines in terms of unsupervised extraction of a hierarchy of features
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
136 useful for classification. Each layer is trained to denoise its
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
137 input, creating a layer of features that can be used as input for the next layer.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
138
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
139 %The principle is that each layer starting from
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
140 %the bottom is trained to encode its input (the output of the previous
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
141 %layer) and to reconstruct it from a corrupted version. After this
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
142 %unsupervised initialization, the stack of DAs can be
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
143 %converted into a deep supervised feedforward neural network and fine-tuned by
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
144 %stochastic gradient descent.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
145
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
146 %
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
147 In this paper we ask the following questions:
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
148
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
149 %\begin{enumerate}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
150 $\bullet$ %\item
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
151 Do the good results previously obtained with deep architectures on the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
152 MNIST digit images generalize to the setting of a much larger and richer (but similar)
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
153 dataset, the NIST special database 19, with 62 classes and around 800k examples?
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
154
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
155 $\bullet$ %\item
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
156 To what extent does the perturbation of input images (e.g. adding
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
157 noise, affine transformations, background images) make the resulting
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
158 classifiers better not only on similarly perturbed images but also on
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
159 the {\em original clean examples}? We study this question in the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
160 context of the 62-class and 10-class tasks of the NIST special database 19.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
161
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
162 $\bullet$ %\item
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
163 Do deep architectures {\em benefit {\bf more} from such out-of-distribution}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
164 examples, i.e. do they benefit more from the self-taught learning~\citep{RainaR2007} framework?
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
165 We use highly perturbed examples to generate out-of-distribution examples.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
166
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
167 $\bullet$ %\item
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
168 Similarly, does the feature learning step in deep learning algorithms benefit {\bf more}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
169 from training with moderately {\em different classes} (i.e. a multi-task learning scenario) than
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
170 a corresponding shallow and purely supervised architecture?
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
171 We train on 62 classes and test on 10 (digits) or 26 (upper case or lower case)
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
172 to answer this question.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
173 %\end{enumerate}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
174
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
175 Our experimental results provide positive evidence towards all of these questions,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
176 as well as classifiers that reach human-level performance on 62-class isolated character
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
177 recognition and beat previously published results on the NIST dataset (special database 19).
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
178 To achieve these results, we introduce in the next section a sophisticated system
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
179 for stochastically transforming character images and then explain the methodology,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
180 which is based on training with or without these transformed images and testing on
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
181 clean ones. We measure the relative advantage of out-of-distribution examples
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
182 (perturbed or out-of-class)
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
183 for a deep learner vs a supervised shallow one.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
184 Code for generating these transformations as well as for the deep learning
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
185 algorithms are made available at {\tt http://hg.assembla.com/ift6266}.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
186 We estimate the relative advantage for deep learners of training with
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
187 other classes than those of interest, by comparing learners trained with
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
188 62 classes with learners trained with only a subset (on which they
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
189 are then tested).
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
190 The conclusion discusses
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
191 the more general question of why deep learners may benefit so much from
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
192 the self-taught learning framework. Since out-of-distribution data
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
193 (perturbed or from other related classes) is very common, this conclusion
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
194 is of practical importance.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
195
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
196 %\vspace*{-3mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
197 %\newpage
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
198 \section{Perturbed and Transformed Character Images}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
199 \label{s:perturbations}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
200 %\vspace*{-2mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
201
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
202 \begin{wrapfigure}[8]{l}{0.15\textwidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
203 %\begin{minipage}[b]{0.14\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
204 %\vspace*{-5mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
205 \begin{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
206 \includegraphics[scale=.4]{Original.png}\\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
207 {\bf Original}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
208 \end{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
209 \end{wrapfigure}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
210 %%\vspace{0.7cm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
211 %\end{minipage}%
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
212 %\hspace{0.3cm}\begin{minipage}[b]{0.86\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
213 This section describes the different transformations we used to stochastically
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
214 transform $32 \times 32$ source images (such as the one on the left)
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
215 in order to obtain data from a larger distribution which
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
216 covers a domain substantially larger than the clean characters distribution from
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
217 which we start.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
218 Although character transformations have been used before to
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
219 improve character recognizers, this effort is on a large scale both
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
220 in number of classes and in the complexity of the transformations, hence
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
221 in the complexity of the learning task.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
222 The code for these transformations (mostly python) is available at
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
223 {\tt http://hg.assembla.com/ift6266}. All the modules in the pipeline share
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
224 a global control parameter ($0 \le complexity \le 1$) that allows one to modulate the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
225 amount of deformation or noise introduced.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
226 There are two main parts in the pipeline. The first one,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
227 from slant to pinch below, performs transformations. The second
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
228 part, from blur to contrast, adds different kinds of noise.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
229 %\end{minipage}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
230
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
231 %\vspace*{1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
232 \subsection{Transformations}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
233 %{\large\bf 2.1 Transformations}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
234 %\vspace*{1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
235
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
236 \subsubsection*{Thickness}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
237
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
238 %\begin{wrapfigure}[7]{l}{0.15\textwidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
239 \begin{minipage}[b]{0.14\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
240 %\centering
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
241 \begin{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
242 \vspace*{-5mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
243 \includegraphics[scale=.4]{Thick_only.png}\\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
244 %{\bf Thickness}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
245 \end{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
246 \vspace{.6cm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
247 \end{minipage}%
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
248 \hspace{0.3cm}\begin{minipage}[b]{0.86\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
249 %\end{wrapfigure}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
250 To change character {\bf thickness}, morphological operators of dilation and erosion~\citep{Haralick87,Serra82}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
251 are applied. The neighborhood of each pixel is multiplied
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
252 element-wise with a {\em structuring element} matrix.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
253 The pixel value is replaced by the maximum or the minimum of the resulting
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
254 matrix, respectively for dilation or erosion. Ten different structural elements with
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
255 increasing dimensions (largest is $5\times5$) were used. For each image,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
256 randomly sample the operator type (dilation or erosion) with equal probability and one structural
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
257 element from a subset of the $n=round(m \times complexity)$ smallest structuring elements
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
258 where $m=10$ for dilation and $m=6$ for erosion (to avoid completely erasing thin characters).
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
259 A neutral element (no transformation)
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
260 is always present in the set.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
261 %%\vspace{.4cm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
262 \end{minipage}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
263
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
264 \vspace{2mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
265
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
266 \subsubsection*{Slant}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
267 \vspace*{2mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
268
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
269 \begin{minipage}[b]{0.14\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
270 \centering
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
271 \includegraphics[scale=.4]{Slant_only.png}\\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
272 %{\bf Slant}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
273 \end{minipage}%
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
274 \hspace{0.3cm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
275 \begin{minipage}[b]{0.83\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
276 %\centering
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
277 To produce {\bf slant}, each row of the image is shifted
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
278 proportionally to its height: $shift = round(slant \times height)$.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
279 $slant \sim U[-complexity,complexity]$.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
280 The shift is randomly chosen to be either to the left or to the right.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
281 \vspace{5mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
282 \end{minipage}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
283 %\vspace*{-4mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
284
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
285 %\newpage
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
286
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
287 \subsubsection*{Affine Transformations}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
288
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
289 \begin{minipage}[b]{0.14\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
290 %\centering
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
291 %\begin{wrapfigure}[8]{l}{0.15\textwidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
292 \begin{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
293 \includegraphics[scale=.4]{Affine_only.png}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
294 \vspace*{6mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
295 %{\small {\bf Affine \mbox{Transformation}}}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
296 \end{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
297 %\end{wrapfigure}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
298 \end{minipage}%
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
299 \hspace{0.3cm}\begin{minipage}[b]{0.86\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
300 \noindent A $2 \times 3$ {\bf affine transform} matrix (with
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
301 parameters $(a,b,c,d,e,f)$) is sampled according to the $complexity$.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
302 Output pixel $(x,y)$ takes the value of input pixel
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
303 nearest to $(ax+by+c,dx+ey+f)$,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
304 producing scaling, translation, rotation and shearing.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
305 Marginal distributions of $(a,b,c,d,e,f)$ have been tuned to
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
306 forbid large rotations (to avoid confusing classes) but to give good
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
307 variability of the transformation: $a$ and $d$ $\sim U[1-3
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
308 complexity,1+3\,complexity]$, $b$ and $e$ $\sim U[-3 \,complexity,3\,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
309 complexity]$, and $c$ and $f \sim U[-4 \,complexity, 4 \,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
310 complexity]$.\\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
311 \end{minipage}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
312
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
313 %\vspace*{-4.5mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
314 \subsubsection*{Local Elastic Deformations}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
315
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
316 %\begin{minipage}[t]{\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
317 %\begin{wrapfigure}[7]{l}{0.15\textwidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
318 %\hspace*{-8mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
319 \begin{minipage}[b]{0.14\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
320 %\centering
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
321 \begin{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
322 \vspace*{5mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
323 \includegraphics[scale=.4]{Localelasticdistorsions_only.png}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
324 %{\bf Local Elastic Deformation}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
325 \end{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
326 %\end{wrapfigure}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
327 \end{minipage}%
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
328 \hspace{3mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
329 \begin{minipage}[b]{0.85\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
330 %%\vspace*{-20mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
331 The {\bf local elastic deformation}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
332 module induces a ``wiggly'' effect in the image, following~\citet{SimardSP03-short},
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
333 which provides more details.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
334 The intensity of the displacement fields is given by
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
335 $\alpha = \sqrt[3]{complexity} \times 10.0$, which are
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
336 convolved with a Gaussian 2D kernel (resulting in a blur) of
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
337 standard deviation $\sigma = 10 - 7 \times\sqrt[3]{complexity}$.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
338 \vspace{2mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
339 \end{minipage}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
340
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
341 \vspace*{4mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
342
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
343 \subsubsection*{Pinch}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
344
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
345 \begin{minipage}[b]{0.14\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
346 %\centering
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
347 %\begin{wrapfigure}[7]{l}{0.15\textwidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
348 %\vspace*{-5mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
349 \begin{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
350 \includegraphics[scale=.4]{Pinch_only.png}\\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
351 \vspace*{15mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
352 %{\bf Pinch}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
353 \end{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
354 %\end{wrapfigure}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
355 %%\vspace{.6cm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
356 \end{minipage}%
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
357 \hspace{0.3cm}\begin{minipage}[b]{0.86\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
358 The {\bf pinch} module applies the ``Whirl and pinch'' GIMP filter with whirl set to 0.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
359 A pinch is ``similar to projecting the image onto an elastic
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
360 surface and pressing or pulling on the center of the surface'' (GIMP documentation manual).
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
361 For a square input image, draw a radius-$r$ disk
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
362 around its center $C$. Any pixel $P$ belonging to
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
363 that disk has its value replaced by
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
364 the value of a ``source'' pixel in the original image,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
365 on the line that goes through $C$ and $P$, but
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
366 at some other distance $d_2$. Define $d_1=distance(P,C)$
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
367 and $d_2 = sin(\frac{\pi{}d_1}{2r})^{-pinch} \times
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
368 d_1$, where $pinch$ is a parameter of the filter.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
369 The actual value is given by bilinear interpolation considering the pixels
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
370 around the (non-integer) source position thus found.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
371 Here $pinch \sim U[-complexity, 0.7 \times complexity]$.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
372 %%\vspace{1.5cm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
373 \end{minipage}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
374
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
375 %\vspace{1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
376
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
377 %{\large\bf 2.2 Injecting Noise}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
378 \subsection{Injecting Noise}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
379 %\vspace{2mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
380
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
381 \subsubsection*{Motion Blur}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
382
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
383 %%\vspace*{-.2cm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
384 \begin{minipage}[t]{0.14\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
385 \centering
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
386 \vspace*{0mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
387 \includegraphics[scale=.4]{Motionblur_only.png}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
388 %{\bf Motion Blur}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
389 \end{minipage}%
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
390 \hspace{0.3cm}\begin{minipage}[t]{0.83\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
391 %%\vspace*{.5mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
392 \vspace*{2mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
393 The {\bf motion blur} module is GIMP's ``linear motion blur'', which
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
394 has parameters $length$ and $angle$. The value of
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
395 a pixel in the final image is approximately the mean of the first $length$ pixels
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
396 found by moving in the $angle$ direction,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
397 $angle \sim U[0,360]$ degrees, and $length \sim {\rm Normal}(0,(3 \times complexity)^2)$.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
398 %\vspace{5mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
399 \end{minipage}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
400
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
401 %\vspace*{1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
402
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
403 \subsubsection*{Occlusion}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
404
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
405 \begin{minipage}[t]{0.14\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
406 \centering
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
407 \vspace*{3mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
408 \includegraphics[scale=.4]{occlusion_only.png}\\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
409 %{\bf Occlusion}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
410 %%\vspace{.5cm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
411 \end{minipage}%
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
412 \hspace{0.3cm}\begin{minipage}[t]{0.83\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
413 %\vspace*{-18mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
414 The {\bf occlusion} module selects a random rectangle from an {\em occluder} character
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
415 image and places it over the original {\em occluded}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
416 image. Pixels are combined by taking the max(occluder, occluded),
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
417 i.e. keeping the lighter ones.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
418 The rectangle corners
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
419 are sampled so that larger complexity gives larger rectangles.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
420 The destination position in the occluded image are also sampled
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
421 according to a normal distribution.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
422 This module is skipped with probability 60\%.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
423 %%\vspace{7mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
424 \end{minipage}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
425
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
426 %\vspace*{1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
427 \subsubsection*{Gaussian Smoothing}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
428
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
429 %\begin{wrapfigure}[8]{l}{0.15\textwidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
430 %\vspace*{-6mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
431 \begin{minipage}[t]{0.14\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
432 \begin{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
433 %\centering
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
434 \vspace*{6mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
435 \includegraphics[scale=.4]{Bruitgauss_only.png}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
436 %{\bf Gaussian Smoothing}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
437 \end{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
438 %\end{wrapfigure}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
439 %%\vspace{.5cm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
440 \end{minipage}%
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
441 \hspace{0.3cm}\begin{minipage}[t]{0.86\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
442 With the {\bf Gaussian smoothing} module,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
443 different regions of the image are spatially smoothed.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
444 This is achieved by first convolving
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
445 the image with an isotropic Gaussian kernel of
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
446 size and variance chosen uniformly in the ranges $[12,12 + 20 \times
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
447 complexity]$ and $[2,2 + 6 \times complexity]$. This filtered image is normalized
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
448 between $0$ and $1$. We also create an isotropic weighted averaging window, of the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
449 kernel size, with maximum value at the center. For each image we sample
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
450 uniformly from $3$ to $3 + 10 \times complexity$ pixels that will be
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
451 averaging centers between the original image and the filtered one. We
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
452 initialize to zero a mask matrix of the image size. For each selected pixel
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
453 we add to the mask the averaging window centered on it. The final image is
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
454 computed from the following element-wise operation: $\frac{image + filtered\_image
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
455 \times mask}{mask+1}$.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
456 This module is skipped with probability 75\%.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
457 \end{minipage}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
458
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
459 %\newpage
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
460
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
461 %\vspace*{-9mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
462 \subsubsection*{Permute Pixels}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
463
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
464 %\hspace*{-3mm}\begin{minipage}[t]{0.18\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
465 %\centering
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
466 \begin{minipage}[t]{0.14\textwidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
467 %\begin{wrapfigure}[7]{l}{
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
468 %\vspace*{-5mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
469 \begin{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
470 \vspace*{1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
471 \includegraphics[scale=.4]{Permutpixel_only.png}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
472 %{\small\bf Permute Pixels}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
473 \end{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
474 %\end{wrapfigure}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
475 \end{minipage}%
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
476 \hspace{3mm}\begin{minipage}[t]{0.86\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
477 \vspace*{1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
478 %%\vspace*{-20mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
479 This module {\bf permutes neighbouring pixels}. It first selects a
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
480 fraction $\frac{complexity}{3}$ of pixels randomly in the image. Each
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
481 of these pixels is then sequentially exchanged with a random pixel
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
482 among its four nearest neighbors (on its left, right, top or bottom).
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
483 This module is skipped with probability 80\%.\\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
484 %\vspace*{1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
485 \end{minipage}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
486
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
487 %\vspace{-3mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
488
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
489 \subsubsection*{Gaussian Noise}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
490
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
491 \begin{minipage}[t]{0.14\textwidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
492 %\begin{wrapfigure}[7]{l}{
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
493 %%\vspace*{-3mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
494 \begin{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
495 %\hspace*{-3mm}\begin{minipage}[t]{0.18\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
496 %\centering
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
497 \vspace*{0mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
498 \includegraphics[scale=.4]{Distorsiongauss_only.png}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
499 %{\small \bf Gauss. Noise}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
500 \end{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
501 %\end{wrapfigure}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
502 \end{minipage}%
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
503 \hspace{0.3cm}\begin{minipage}[t]{0.86\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
504 \vspace*{1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
505 %\vspace*{12mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
506 The {\bf Gaussian noise} module simply adds, to each pixel of the image independently, a
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
507 noise $\sim Normal(0,(\frac{complexity}{10})^2)$.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
508 This module is skipped with probability 70\%.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
509 %%\vspace{1.1cm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
510 \end{minipage}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
511
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
512 %\vspace*{1.2cm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
513
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
514 \subsubsection*{Background Image Addition}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
515
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
516 \begin{minipage}[t]{\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
517 \begin{minipage}[t]{0.14\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
518 \centering
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
519 \vspace*{0mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
520 \includegraphics[scale=.4]{background_other_only.png}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
521 %{\small \bf Bg Image}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
522 \end{minipage}%
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
523 \hspace{0.3cm}\begin{minipage}[t]{0.83\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
524 \vspace*{1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
525 Following~\citet{Larochelle-jmlr-2009}, the {\bf background image} module adds a random
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
526 background image behind the letter, from a randomly chosen natural image,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
527 with contrast adjustments depending on $complexity$, to preserve
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
528 more or less of the original character image.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
529 %%\vspace{.8cm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
530 \end{minipage}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
531 \end{minipage}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
532 %%\vspace{-.7cm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
533
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
534 \subsubsection*{Salt and Pepper Noise}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
535
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
536 \begin{minipage}[t]{0.14\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
537 \centering
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
538 \vspace*{0mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
539 \includegraphics[scale=.4]{Poivresel_only.png}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
540 %{\small \bf Salt \& Pepper}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
541 \end{minipage}%
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
542 \hspace{0.3cm}\begin{minipage}[t]{0.83\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
543 \vspace*{1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
544 The {\bf salt and pepper noise} module adds noise $\sim U[0,1]$ to random subsets of pixels.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
545 The number of selected pixels is $0.2 \times complexity$.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
546 This module is skipped with probability 75\%.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
547 %%\vspace{.9cm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
548 \end{minipage}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
549 %%\vspace{-.7cm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
550
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
551 %\vspace{1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
552 \subsubsection*{Scratches}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
553
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
554 \begin{minipage}[t]{0.14\textwidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
555 %\begin{wrapfigure}[7]{l}{
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
556 %\begin{minipage}[t]{0.14\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
557 %\centering
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
558 \begin{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
559 \vspace*{4mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
560 %\hspace*{-1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
561 \includegraphics[scale=.4]{Rature_only.png}\\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
562 %{\bf Scratches}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
563 \end{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
564 \end{minipage}%
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
565 %\end{wrapfigure}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
566 \hspace{0.3cm}\begin{minipage}[t]{0.86\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
567 %%\vspace{.4cm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
568 The {\bf scratches} module places line-like white patches on the image. The
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
569 lines are heavily transformed images of the digit ``1'' (one), chosen
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
570 at random among 500 such 1 images,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
571 randomly cropped and rotated by an angle $\sim Normal(0,(100 \times
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
572 complexity)^2$ (in degrees), using bi-cubic interpolation.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
573 Two passes of a grey-scale morphological erosion filter
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
574 are applied, reducing the width of the line
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
575 by an amount controlled by $complexity$.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
576 This module is skipped with probability 85\%. The probabilities
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
577 of applying 1, 2, or 3 patches are (50\%,30\%,20\%).
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
578 \end{minipage}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
579
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
580 %\vspace*{1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
581
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
582 \subsubsection*{Grey Level and Contrast Changes}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
583
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
584 \begin{minipage}[t]{0.15\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
585 \centering
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
586 \vspace*{0mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
587 \includegraphics[scale=.4]{Contrast_only.png}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
588 %{\bf Grey Level \& Contrast}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
589 \end{minipage}%
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
590 \hspace{3mm}\begin{minipage}[t]{0.85\linewidth}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
591 \vspace*{1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
592 The {\bf grey level and contrast} module changes the contrast by changing grey levels, and may invert the image polarity (white
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
593 to black and black to white). The contrast is $C \sim U[1-0.85 \times complexity,1]$
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
594 so the image is normalized into $[\frac{1-C}{2},1-\frac{1-C}{2}]$. The
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
595 polarity is inverted with probability 50\%.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
596 %%\vspace{.7cm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
597 \end{minipage}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
598 %\vspace{2mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
599
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
600
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
601 \iffalse
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
602 \begin{figure}[ht]
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
603 \centerline{\resizebox{.9\textwidth}{!}{\includegraphics{example_t.png}}}\\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
604 \caption{Illustration of the pipeline of stochastic
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
605 transformations applied to the image of a lower-case \emph{t}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
606 (the upper left image). Each image in the pipeline (going from
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
607 left to right, first top line, then bottom line) shows the result
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
608 of applying one of the modules in the pipeline. The last image
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
609 (bottom right) is used as training example.}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
610 \label{fig:pipeline}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
611 \end{figure}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
612 \fi
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
613
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
614 %\vspace*{-3mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
615 \section{Experimental Setup}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
616 %\vspace*{-1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
617
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
618 Much previous work on deep learning had been performed on
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
619 the MNIST digits task~\citep{Hinton06,ranzato-07-small,Bengio-nips-2006,Salakhutdinov+Hinton-2009},
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
620 with 60~000 examples, and variants involving 10~000
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
621 examples~\citep{Larochelle-jmlr-toappear-2008,VincentPLarochelleH2008}.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
622 The focus here is on much larger training sets, from 10 times to
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
623 to 1000 times larger, and 62 classes.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
624
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
625 The first step in constructing the larger datasets (called NISTP and P07) is to sample from
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
626 a {\em data source}: {\bf NIST} (NIST database 19), {\bf Fonts}, {\bf Captchas},
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
627 and {\bf OCR data} (scanned machine printed characters). Once a character
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
628 is sampled from one of these sources (chosen randomly), the second step is to
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
629 apply a pipeline of transformations and/or noise processes described in section \ref{s:perturbations}.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
630
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
631 To provide a baseline of error rate comparison we also estimate human performance
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
632 on both the 62-class task and the 10-class digits task.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
633 We compare the best Multi-Layer Perceptrons (MLP) against
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
634 the best Stacked Denoising Auto-encoders (SDA), when
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
635 both models' hyper-parameters are selected to minimize the validation set error.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
636 We also provide a comparison against a precise estimate
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
637 of human performance obtained via Amazon's Mechanical Turk (AMT)
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
638 service (http://mturk.com).
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
639 AMT users are paid small amounts
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
640 of money to perform tasks for which human intelligence is required.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
641 Mechanical Turk has been used extensively in natural language processing and vision.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
642 %processing \citep{SnowEtAl2008} and vision
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
643 %\citep{SorokinAndForsyth2008,whitehill09}.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
644 AMT users were presented
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
645 with 10 character images (from a test set) and asked to choose 10 corresponding ASCII
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
646 characters. They were forced to choose a single character class (either among the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
647 62 or 10 character classes) for each image.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
648 80 subjects classified 2500 images per (dataset,task) pair.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
649 Different humans labelers sometimes provided a different label for the same
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
650 example, and we were able to estimate the error variance due to this effect
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
651 because each image was classified by 3 different persons.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
652 The average error of humans on the 62-class task NIST test set
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
653 is 18.2\%, with a standard error of 0.1\%.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
654
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
655 %\vspace*{-3mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
656 \subsection{Data Sources}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
657 %\vspace*{-2mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
658
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
659 %\begin{itemize}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
660 %\item
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
661 {\bf NIST.}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
662 Our main source of characters is the NIST Special Database 19~\citep{Grother-1995},
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
663 widely used for training and testing character
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
664 recognition systems~\citep{Granger+al-2007,Cortes+al-2000,Oliveira+al-2002-short,Milgram+al-2005}.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
665 The dataset is composed of 814255 digits and characters (upper and lower cases), with hand checked classifications,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
666 extracted from handwritten sample forms of 3600 writers. The characters are labelled by one of the 62 classes
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
667 corresponding to ``0''-``9'',``A''-``Z'' and ``a''-``z''. The dataset contains 8 parts (partitions) of varying complexity.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
668 The fourth partition (called $hsf_4$, 82587 examples),
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
669 experimentally recognized to be the most difficult one, is the one recommended
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
670 by NIST as a testing set and is used in our work as well as some previous work~\citep{Granger+al-2007,Cortes+al-2000,Oliveira+al-2002-short,Milgram+al-2005}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
671 for that purpose. We randomly split the remainder (731668 examples) into a training set and a validation set for
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
672 model selection.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
673 The performances reported by previous work on that dataset mostly use only the digits.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
674 Here we use all the classes both in the training and testing phase. This is especially
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
675 useful to estimate the effect of a multi-task setting.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
676 The distribution of the classes in the NIST training and test sets differs
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
677 substantially, with relatively many more digits in the test set, and a more uniform distribution
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
678 of letters in the test set (whereas in the training set they are distributed
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
679 more like in natural text).
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
680 %\vspace*{-1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
681
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
682 %\item
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
683 {\bf Fonts.}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
684 In order to have a good variety of sources we downloaded an important number of free fonts from:
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
685 {\tt http://cg.scs.carleton.ca/\textasciitilde luc/freefonts.html}.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
686 % TODO: pointless to anonymize, it's not pointing to our work
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
687 Including the operating system's (Windows 7) fonts, there is a total of $9817$ different fonts that we can choose uniformly from.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
688 The chosen {\tt ttf} file is either used as input of the Captcha generator (see next item) or, by producing a corresponding image,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
689 directly as input to our models.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
690 %\vspace*{-1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
691
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
692 %\item
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
693 {\bf Captchas.}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
694 The Captcha data source is an adaptation of the \emph{pycaptcha} library (a python based captcha generator library) for
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
695 generating characters of the same format as the NIST dataset. This software is based on
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
696 a random character class generator and various kinds of transformations similar to those described in the previous sections.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
697 In order to increase the variability of the data generated, many different fonts are used for generating the characters.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
698 Transformations (slant, distortions, rotation, translation) are applied to each randomly generated character with a complexity
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
699 depending on the value of the complexity parameter provided by the user of the data source.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
700 %Two levels of complexity are allowed and can be controlled via an easy to use facade class. %TODO: what's a facade class?
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
701 %\vspace*{-1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
702
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
703 %\item
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
704 {\bf OCR data.}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
705 A large set (2 million) of scanned, OCRed and manually verified machine-printed
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
706 characters where included as an
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
707 additional source. This set is part of a larger corpus being collected by the Image Understanding
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
708 Pattern Recognition Research group led by Thomas Breuel at University of Kaiserslautern
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
709 ({\tt http://www.iupr.com}), and which will be publicly released.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
710 %TODO: let's hope that Thomas is not a reviewer! :) Seriously though, maybe we should anonymize this
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
711 %\end{itemize}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
712
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
713 %\vspace*{-3mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
714 \subsection{Data Sets}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
715 %\vspace*{-2mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
716
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
717 All data sets contain 32$\times$32 grey-level images (values in $[0,1]$) associated with a label
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
718 from one of the 62 character classes.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
719 %\begin{itemize}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
720 %\vspace*{-1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
721
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
722 %\item
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
723 {\bf NIST.} This is the raw NIST special database 19~\citep{Grother-1995}. It has
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
724 \{651668 / 80000 / 82587\} \{training / validation / test\} examples.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
725 %\vspace*{-1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
726
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
727 %\item
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
728 {\bf P07.} This dataset is obtained by taking raw characters from all four of the above sources
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
729 and sending them through the transformation pipeline described in section \ref{s:perturbations}.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
730 For each new example to generate, a data source is selected with probability $10\%$ from the fonts,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
731 $25\%$ from the captchas, $25\%$ from the OCR data and $40\%$ from NIST. We apply all the transformations in the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
732 order given above, and for each of them we sample uniformly a \emph{complexity} in the range $[0,0.7]$.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
733 It has \{81920000 / 80000 / 20000\} \{training / validation / test\} examples.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
734 %\vspace*{-1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
735
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
736 %\item
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
737 {\bf NISTP.} This one is equivalent to P07 (complexity parameter of $0.7$ with the same proportions of data sources)
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
738 except that we only apply
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
739 transformations from slant to pinch. Therefore, the character is
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
740 transformed but no additional noise is added to the image, giving images
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
741 closer to the NIST dataset.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
742 It has \{81920000 / 80000 / 20000\} \{training / validation / test\} examples.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
743 %\end{itemize}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
744
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
745 %\vspace*{-3mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
746 \subsection{Models and their Hyperparameters}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
747 %\vspace*{-2mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
748
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
749 The experiments are performed using MLPs (with a single
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
750 hidden layer) and SDAs.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
751 \emph{Hyper-parameters are selected based on the {\bf NISTP} validation set error.}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
752
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
753 {\bf Multi-Layer Perceptrons (MLP).}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
754 Whereas previous work had compared deep architectures to both shallow MLPs and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
755 SVMs, we only compared to MLPs here because of the very large datasets used
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
756 (making the use of SVMs computationally challenging because of their quadratic
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
757 scaling behavior). Preliminary experiments on training SVMs (libSVM) with subsets of the training
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
758 set allowing the program to fit in memory yielded substantially worse results
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
759 than those obtained with MLPs. For training on nearly a billion examples
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
760 (with the perturbed data), the MLPs and SDA are much more convenient than
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
761 classifiers based on kernel methods.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
762 The MLP has a single hidden layer with $\tanh$ activation functions, and softmax (normalized
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
763 exponentials) on the output layer for estimating $P(class | image)$.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
764 The number of hidden units is taken in $\{300,500,800,1000,1500\}$.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
765 Training examples are presented in minibatches of size 20. A constant learning
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
766 rate was chosen among $\{0.001, 0.01, 0.025, 0.075, 0.1, 0.5\}$.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
767 %through preliminary experiments (measuring performance on a validation set),
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
768 %and $0.1$ (which was found to work best) was then selected for optimizing on
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
769 %the whole training sets.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
770 %\vspace*{-1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
771
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
772
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
773 {\bf Stacked Denoising Auto-Encoders (SDA).}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
774 Various auto-encoder variants and Restricted Boltzmann Machines (RBMs)
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
775 can be used to initialize the weights of each layer of a deep MLP (with many hidden
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
776 layers)~\citep{Hinton06,ranzato-07-small,Bengio-nips-2006},
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
777 apparently setting parameters in the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
778 basin of attraction of supervised gradient descent yielding better
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
779 generalization~\citep{Erhan+al-2010}. This initial {\em unsupervised
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
780 pre-training phase} uses all of the training images but not the training labels.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
781 Each layer is trained in turn to produce a new representation of its input
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
782 (starting from the raw pixels).
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
783 It is hypothesized that the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
784 advantage brought by this procedure stems from a better prior,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
785 on the one hand taking advantage of the link between the input
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
786 distribution $P(x)$ and the conditional distribution of interest
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
787 $P(y|x)$ (like in semi-supervised learning), and on the other hand
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
788 taking advantage of the expressive power and bias implicit in the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
789 deep architecture (whereby complex concepts are expressed as
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
790 compositions of simpler ones through a deep hierarchy).
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
791
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
792 \begin{figure}[ht]
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
793 %\vspace*{-2mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
794 \centerline{\resizebox{0.8\textwidth}{!}{\includegraphics{denoising_autoencoder_small.pdf}}}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
795 %\vspace*{-2mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
796 \caption{Illustration of the computations and training criterion for the denoising
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
797 auto-encoder used to pre-train each layer of the deep architecture. Input $x$ of
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
798 the layer (i.e. raw input or output of previous layer)
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
799 s corrupted into $\tilde{x}$ and encoded into code $y$ by the encoder $f_\theta(\cdot)$.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
800 The decoder $g_{\theta'}(\cdot)$ maps $y$ to reconstruction $z$, which
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
801 is compared to the uncorrupted input $x$ through the loss function
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
802 $L_H(x,z)$, whose expected value is approximately minimized during training
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
803 by tuning $\theta$ and $\theta'$.}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
804 \label{fig:da}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
805 %\vspace*{-2mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
806 \end{figure}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
807
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
808 Here we chose to use the Denoising
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
809 Auto-encoder~\citep{VincentPLarochelleH2008} as the building block for
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
810 these deep hierarchies of features, as it is simple to train and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
811 explain (see Figure~\ref{fig:da}, as well as
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
812 tutorial and code there: {\tt http://deeplearning.net/tutorial}),
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
813 provides efficient inference, and yielded results
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
814 comparable or better than RBMs in series of experiments
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
815 \citep{VincentPLarochelleH2008}. During training, a Denoising
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
816 Auto-encoder is presented with a stochastically corrupted version
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
817 of the input and trained to reconstruct the uncorrupted input,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
818 forcing the hidden units to represent the leading regularities in
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
819 the data. Here we use the random binary masking corruption
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
820 (which sets to 0 a random subset of the inputs).
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
821 Once it is trained, in a purely unsupervised way,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
822 its hidden units' activations can
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
823 be used as inputs for training a second one, etc.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
824 After this unsupervised pre-training stage, the parameters
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
825 are used to initialize a deep MLP, which is fine-tuned by
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
826 the same standard procedure used to train them (see previous section).
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
827 The SDA hyper-parameters are the same as for the MLP, with the addition of the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
828 amount of corruption noise (we used the masking noise process, whereby a
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
829 fixed proportion of the input values, randomly selected, are zeroed), and a
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
830 separate learning rate for the unsupervised pre-training stage (selected
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
831 from the same above set). The fraction of inputs corrupted was selected
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
832 among $\{10\%, 20\%, 50\%\}$. Another hyper-parameter is the number
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
833 of hidden layers but it was fixed to 3 based on previous work with
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
834 SDAs on MNIST~\citep{VincentPLarochelleH2008}. The size of the hidden
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
835 layers was kept constant across hidden layers, and the best results
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
836 were obtained with the largest values that we could experiment
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
837 with given our patience, with 1000 hidden units.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
838
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
839 %\vspace*{-1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
840
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
841 \begin{figure}[ht]
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
842 %\vspace*{-2mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
843 \centerline{\resizebox{.99\textwidth}{!}{\includegraphics{error_rates_charts.pdf}}}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
844 %\vspace*{-3mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
845 \caption{SDAx are the {\bf deep} models. Error bars indicate a 95\% confidence interval. 0 indicates that the model was trained
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
846 on NIST, 1 on NISTP, and 2 on P07. Left: overall results
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
847 of all models, on NIST and NISTP test sets.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
848 Right: error rates on NIST test digits only, along with the previous results from
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
849 literature~\citep{Granger+al-2007,Cortes+al-2000,Oliveira+al-2002-short,Milgram+al-2005}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
850 respectively based on ART, nearest neighbors, MLPs, and SVMs.}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
851 \label{fig:error-rates-charts}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
852 %\vspace*{-2mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
853 \end{figure}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
854
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
855
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
856 \begin{figure}[ht]
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
857 %\vspace*{-3mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
858 \centerline{\resizebox{.99\textwidth}{!}{\includegraphics{improvements_charts.pdf}}}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
859 %\vspace*{-3mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
860 \caption{Relative improvement in error rate due to self-taught learning.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
861 Left: Improvement (or loss, when negative)
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
862 induced by out-of-distribution examples (perturbed data).
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
863 Right: Improvement (or loss, when negative) induced by multi-task
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
864 learning (training on all classes and testing only on either digits,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
865 upper case, or lower-case). The deep learner (SDA) benefits more from
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
866 both self-taught learning scenarios, compared to the shallow MLP.}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
867 \label{fig:improvements-charts}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
868 %\vspace*{-2mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
869 \end{figure}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
870
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
871 \section{Experimental Results}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
872 %\vspace*{-2mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
873
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
874 %%\vspace*{-1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
875 %\subsection{SDA vs MLP vs Humans}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
876 %%\vspace*{-1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
877 The models are either trained on NIST (MLP0 and SDA0),
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
878 NISTP (MLP1 and SDA1), or P07 (MLP2 and SDA2), and tested
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
879 on either NIST, NISTP or P07, either on the 62-class task
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
880 or on the 10-digits task. Training (including about half
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
881 for unsupervised pre-training, for DAs) on the larger
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
882 datasets takes around one day on a GPU-285.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
883 Figure~\ref{fig:error-rates-charts} summarizes the results obtained,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
884 comparing humans, the three MLPs (MLP0, MLP1, MLP2) and the three SDAs (SDA0, SDA1,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
885 SDA2), along with the previous results on the digits NIST special database
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
886 19 test set from the literature, respectively based on ARTMAP neural
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
887 networks ~\citep{Granger+al-2007}, fast nearest-neighbor search
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
888 ~\citep{Cortes+al-2000}, MLPs ~\citep{Oliveira+al-2002-short}, and SVMs
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
889 ~\citep{Milgram+al-2005}. More detailed and complete numerical results
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
890 (figures and tables, including standard errors on the error rates) can be
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
891 found in Appendix.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
892 The deep learner not only outperformed the shallow ones and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
893 previously published performance (in a statistically and qualitatively
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
894 significant way) but when trained with perturbed data
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
895 reaches human performance on both the 62-class task
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
896 and the 10-class (digits) task.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
897 17\% error (SDA1) or 18\% error (humans) may seem large but a large
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
898 majority of the errors from humans and from SDA1 are from out-of-context
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
899 confusions (e.g. a vertical bar can be a ``1'', an ``l'' or an ``L'', and a
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
900 ``c'' and a ``C'' are often indistinguishible).
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
901
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
902 In addition, as shown in the left of
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
903 Figure~\ref{fig:improvements-charts}, the relative improvement in error
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
904 rate brought by self-taught learning is greater for the SDA, and these
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
905 differences with the MLP are statistically and qualitatively
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
906 significant.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
907 The left side of the figure shows the improvement to the clean
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
908 NIST test set error brought by the use of out-of-distribution examples
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
909 (i.e. the perturbed examples examples from NISTP or P07).
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
910 Relative percent change is measured by taking
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
911 $100 \% \times$ (original model's error / perturbed-data model's error - 1).
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
912 The right side of
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
913 Figure~\ref{fig:improvements-charts} shows the relative improvement
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
914 brought by the use of a multi-task setting, in which the same model is
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
915 trained for more classes than the target classes of interest (i.e. training
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
916 with all 62 classes when the target classes are respectively the digits,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
917 lower-case, or upper-case characters). Again, whereas the gain from the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
918 multi-task setting is marginal or negative for the MLP, it is substantial
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
919 for the SDA. Note that to simplify these multi-task experiments, only the original
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
920 NIST dataset is used. For example, the MLP-digits bar shows the relative
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
921 percent improvement in MLP error rate on the NIST digits test set
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
922 is $100\% \times$ (single-task
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
923 model's error / multi-task model's error - 1). The single-task model is
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
924 trained with only 10 outputs (one per digit), seeing only digit examples,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
925 whereas the multi-task model is trained with 62 outputs, with all 62
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
926 character classes as examples. Hence the hidden units are shared across
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
927 all tasks. For the multi-task model, the digit error rate is measured by
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
928 comparing the correct digit class with the output class associated with the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
929 maximum conditional probability among only the digit classes outputs. The
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
930 setting is similar for the other two target classes (lower case characters
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
931 and upper case characters).
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
932 %%\vspace*{-1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
933 %\subsection{Perturbed Training Data More Helpful for SDA}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
934 %%\vspace*{-1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
935
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
936 %%\vspace*{-1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
937 %\subsection{Multi-Task Learning Effects}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
938 %%\vspace*{-1mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
939
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
940 \iffalse
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
941 As previously seen, the SDA is better able to benefit from the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
942 transformations applied to the data than the MLP. In this experiment we
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
943 define three tasks: recognizing digits (knowing that the input is a digit),
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
944 recognizing upper case characters (knowing that the input is one), and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
945 recognizing lower case characters (knowing that the input is one). We
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
946 consider the digit classification task as the target task and we want to
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
947 evaluate whether training with the other tasks can help or hurt, and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
948 whether the effect is different for MLPs versus SDAs. The goal is to find
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
949 out if deep learning can benefit more (or less) from multiple related tasks
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
950 (i.e. the multi-task setting) compared to a corresponding purely supervised
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
951 shallow learner.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
952
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
953 We use a single hidden layer MLP with 1000 hidden units, and a SDA
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
954 with 3 hidden layers (1000 hidden units per layer), pre-trained and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
955 fine-tuned on NIST.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
956
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
957 Our results show that the MLP benefits marginally from the multi-task setting
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
958 in the case of digits (5\% relative improvement) but is actually hurt in the case
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
959 of characters (respectively 3\% and 4\% worse for lower and upper class characters).
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
960 On the other hand the SDA benefited from the multi-task setting, with relative
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
961 error rate improvements of 27\%, 15\% and 13\% respectively for digits,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
962 lower and upper case characters, as shown in Table~\ref{tab:multi-task}.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
963 \fi
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
964
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
965
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
966 %\vspace*{-2mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
967 \section{Conclusions and Discussion}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
968 %\vspace*{-2mm}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
969
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
970 We have found that the self-taught learning framework is more beneficial
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
971 to a deep learner than to a traditional shallow and purely
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
972 supervised learner. More precisely,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
973 the answers are positive for all the questions asked in the introduction.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
974 %\begin{itemize}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
975
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
976 $\bullet$ %\item
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
977 {\bf Do the good results previously obtained with deep architectures on the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
978 MNIST digits generalize to a much larger and richer (but similar)
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
979 dataset, the NIST special database 19, with 62 classes and around 800k examples}?
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
980 Yes, the SDA {\em systematically outperformed the MLP and all the previously
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
981 published results on this dataset} (the ones that we are aware of), {\em in fact reaching human-level
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
982 performance} at around 17\% error on the 62-class task and 1.4\% on the digits,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
983 and beating previously published results on the same data.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
984
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
985 $\bullet$ %\item
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
986 {\bf To what extent do self-taught learning scenarios help deep learners,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
987 and do they help them more than shallow supervised ones}?
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
988 We found that distorted training examples not only made the resulting
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
989 classifier better on similarly perturbed images but also on
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
990 the {\em original clean examples}, and more importantly and more novel,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
991 that deep architectures benefit more from such {\em out-of-distribution}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
992 examples. MLPs were helped by perturbed training examples when tested on perturbed input
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
993 images (65\% relative improvement on NISTP)
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
994 but only marginally helped (5\% relative improvement on all classes)
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
995 or even hurt (10\% relative loss on digits)
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
996 with respect to clean examples . On the other hand, the deep SDAs
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
997 were significantly boosted by these out-of-distribution examples.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
998 Similarly, whereas the improvement due to the multi-task setting was marginal or
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
999 negative for the MLP (from +5.6\% to -3.6\% relative change),
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1000 it was quite significant for the SDA (from +13\% to +27\% relative change),
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1001 which may be explained by the arguments below.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1002 %\end{itemize}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1003
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1004 In the original self-taught learning framework~\citep{RainaR2007}, the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1005 out-of-sample examples were used as a source of unsupervised data, and
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1006 experiments showed its positive effects in a \emph{limited labeled data}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1007 scenario. However, many of the results by \citet{RainaR2007} (who used a
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1008 shallow, sparse coding approach) suggest that the {\em relative gain of self-taught
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1009 learning vs ordinary supervised learning} diminishes as the number of labeled examples increases.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1010 We note instead that, for deep
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1011 architectures, our experiments show that such a positive effect is accomplished
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1012 even in a scenario with a \emph{large number of labeled examples},
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1013 i.e., here, the relative gain of self-taught learning is probably preserved
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1014 in the asymptotic regime.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1015
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1016 {\bf Why would deep learners benefit more from the self-taught learning framework}?
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1017 The key idea is that the lower layers of the predictor compute a hierarchy
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1018 of features that can be shared across tasks or across variants of the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1019 input distribution. A theoretical analysis of generalization improvements
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1020 due to sharing of intermediate features across tasks already points
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1021 towards that explanation~\cite{baxter95a}.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1022 Intermediate features that can be used in different
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1023 contexts can be estimated in a way that allows to share statistical
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1024 strength. Features extracted through many levels are more likely to
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1025 be more abstract (as the experiments in~\citet{Goodfellow2009} suggest),
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1026 increasing the likelihood that they would be useful for a larger array
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1027 of tasks and input conditions.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1028 Therefore, we hypothesize that both depth and unsupervised
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1029 pre-training play a part in explaining the advantages observed here, and future
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1030 experiments could attempt at teasing apart these factors.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1031 And why would deep learners benefit from the self-taught learning
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1032 scenarios even when the number of labeled examples is very large?
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1033 We hypothesize that this is related to the hypotheses studied
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1034 in~\citet{Erhan+al-2010}. Whereas in~\citet{Erhan+al-2010}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1035 it was found that online learning on a huge dataset did not make the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1036 advantage of the deep learning bias vanish, a similar phenomenon
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1037 may be happening here. We hypothesize that unsupervised pre-training
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1038 of a deep hierarchy with self-taught learning initializes the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1039 model in the basin of attraction of supervised gradient descent
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1040 that corresponds to better generalization. Furthermore, such good
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1041 basins of attraction are not discovered by pure supervised learning
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1042 (with or without self-taught settings), and more labeled examples
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1043 does not allow the model to go from the poorer basins of attraction discovered
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1044 by the purely supervised shallow models to the kind of better basins associated
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1045 with deep learning and self-taught learning.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1046
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1047 A Flash demo of the recognizer (where both the MLP and the SDA can be compared)
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1048 can be executed on-line at {\tt http://deep.host22.com}.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1049
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1050
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1051 \section*{Appendix I: Detailed Numerical Results}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1052
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1053 These tables correspond to Figures 2 and 3 and contain the raw error rates for each model and dataset considered.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1054 They also contain additional data such as test errors on P07 and standard errors.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1055
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1056 \begin{table}[ht]
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1057 \caption{Overall comparison of error rates ($\pm$ std.err.) on 62 character classes (10 digits +
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1058 26 lower + 26 upper), except for last columns -- digits only, between deep architecture with pre-training
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1059 (SDA=Stacked Denoising Autoencoder) and ordinary shallow architecture
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1060 (MLP=Multi-Layer Perceptron). The models shown are all trained using perturbed data (NISTP or P07)
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1061 and using a validation set to select hyper-parameters and other training choices.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1062 \{SDA,MLP\}0 are trained on NIST,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1063 \{SDA,MLP\}1 are trained on NISTP, and \{SDA,MLP\}2 are trained on P07.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1064 The human error rate on digits is a lower bound because it does not count digits that were
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1065 recognized as letters. For comparison, the results found in the literature
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1066 on NIST digits classification using the same test set are included.}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1067 \label{tab:sda-vs-mlp-vs-humans}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1068 \begin{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1069 \begin{tabular}{|l|r|r|r|r|} \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1070 & NIST test & NISTP test & P07 test & NIST test digits \\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1071 Humans& 18.2\% $\pm$.1\% & 39.4\%$\pm$.1\% & 46.9\%$\pm$.1\% & $1.4\%$ \\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1072 SDA0 & 23.7\% $\pm$.14\% & 65.2\%$\pm$.34\% & 97.45\%$\pm$.06\% & 2.7\% $\pm$.14\%\\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1073 SDA1 & 17.1\% $\pm$.13\% & 29.7\%$\pm$.3\% & 29.7\%$\pm$.3\% & 1.4\% $\pm$.1\%\\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1074 SDA2 & 18.7\% $\pm$.13\% & 33.6\%$\pm$.3\% & 39.9\%$\pm$.17\% & 1.7\% $\pm$.1\%\\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1075 MLP0 & 24.2\% $\pm$.15\% & 68.8\%$\pm$.33\% & 78.70\%$\pm$.14\% & 3.45\% $\pm$.15\% \\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1076 MLP1 & 23.0\% $\pm$.15\% & 41.8\%$\pm$.35\% & 90.4\%$\pm$.1\% & 3.85\% $\pm$.16\% \\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1077 MLP2 & 24.3\% $\pm$.15\% & 46.0\%$\pm$.35\% & 54.7\%$\pm$.17\% & 4.85\% $\pm$.18\% \\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1078 \citep{Granger+al-2007} & & & & 4.95\% $\pm$.18\% \\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1079 \citep{Cortes+al-2000} & & & & 3.71\% $\pm$.16\% \\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1080 \citep{Oliveira+al-2002} & & & & 2.4\% $\pm$.13\% \\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1081 \citep{Milgram+al-2005} & & & & 2.1\% $\pm$.12\% \\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1082 \end{tabular}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1083 \end{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1084 \end{table}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1085
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1086 \begin{table}[ht]
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1087 \caption{Relative change in error rates due to the use of perturbed training data,
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1088 either using NISTP, for the MLP1/SDA1 models, or using P07, for the MLP2/SDA2 models.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1089 A positive value indicates that training on the perturbed data helped for the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1090 given test set (the first 3 columns on the 62-class tasks and the last one is
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1091 on the clean 10-class digits). Clearly, the deep learning models did benefit more
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1092 from perturbed training data, even when testing on clean data, whereas the MLP
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1093 trained on perturbed data performed worse on the clean digits and about the same
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1094 on the clean characters. }
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1095 \label{tab:perturbation-effect}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1096 \begin{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1097 \begin{tabular}{|l|r|r|r|r|} \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1098 & NIST test & NISTP test & P07 test & NIST test digits \\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1099 SDA0/SDA1-1 & 38\% & 84\% & 228\% & 93\% \\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1100 SDA0/SDA2-1 & 27\% & 94\% & 144\% & 59\% \\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1101 MLP0/MLP1-1 & 5.2\% & 65\% & -13\% & -10\% \\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1102 MLP0/MLP2-1 & -0.4\% & 49\% & 44\% & -29\% \\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1103 \end{tabular}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1104 \end{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1105 \end{table}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1106
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1107 \begin{table}[ht]
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1108 \caption{Test error rates and relative change in error rates due to the use of
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1109 a multi-task setting, i.e., training on each task in isolation vs training
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1110 for all three tasks together, for MLPs vs SDAs. The SDA benefits much
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1111 more from the multi-task setting. All experiments on only on the
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1112 unperturbed NIST data, using validation error for model selection.
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1113 Relative improvement is 1 - single-task error / multi-task error.}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1114 \label{tab:multi-task}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1115 \begin{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1116 \begin{tabular}{|l|r|r|r|} \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1117 & single-task & multi-task & relative \\
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1118 & setting & setting & improvement \\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1119 MLP-digits & 3.77\% & 3.99\% & 5.6\% \\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1120 MLP-lower & 17.4\% & 16.8\% & -4.1\% \\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1121 MLP-upper & 7.84\% & 7.54\% & -3.6\% \\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1122 SDA-digits & 2.6\% & 3.56\% & 27\% \\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1123 SDA-lower & 12.3\% & 14.4\% & 15\% \\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1124 SDA-upper & 5.93\% & 6.78\% & 13\% \\ \hline
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1125 \end{tabular}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1126 \end{center}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1127 \end{table}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1128
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1129 %\afterpage{\clearpage}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1130 \clearpage
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1131 {
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1132 \bibliographystyle{spbasic} % basic style, author-year citations
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1133 \bibliography{strings,strings-short,strings-shorter,ift6266_ml,specials,aigaion-shorter}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1134 %\bibliographystyle{plainnat}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1135 %\bibliographystyle{unsrtnat}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1136 %\bibliographystyle{apalike}
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1137 }
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1138
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1139
b1be957dd1be Added mlj_submission to group every file needed for that.
fsavard
parents:
diff changeset
1140 \end{document}