annotate writeup/jmlr_submission.tex @ 625:128bc92897f2

Script to generate resized version of mnist in two different ways: rescaling and zeropadding
author Salah Rifai <salahmeister@gmail.com>
date Wed, 16 Mar 2011 00:05:39 -0400
parents 537f8b786655
children
rev   line source
594
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
1 %\documentclass[twoside,11pt]{article} % For LaTeX2e
593
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
2 \documentclass{article} % For LaTeX2e
594
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
3 \usepackage{jmlr2e}
593
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
4 \usepackage{times}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
5 \usepackage{wrapfig}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
6 %\usepackage{amsthm} % not to be used with springer tools
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
7 \usepackage{amsmath}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
8 \usepackage{bbm}
594
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
9 \usepackage[utf8]{inputenc}
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
10 %\usepackage[psamsfonts]{amssymb}
593
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
11 %\usepackage{algorithm,algorithmic} % not used after all
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
12 \usepackage{graphicx,subfigure}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
13 \usepackage{natbib} % was [numbers]{natbib}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
14
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
15 \addtolength{\textwidth}{10mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
16 \addtolength{\evensidemargin}{-5mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
17 \addtolength{\oddsidemargin}{-5mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
18
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
19 %\setlength\parindent{0mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
20
594
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
21 \begin{document}
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
22
593
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
23 \title{Deep Self-Taught Learning for Handwritten Character Recognition}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
24 \author{
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
25 Yoshua Bengio \and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
26 Frédéric Bastien \and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
27 Arnaud Bergeron \and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
28 Nicolas Boulanger-Lewandowski \and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
29 Thomas Breuel \and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
30 Youssouf Chherawala \and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
31 Moustapha Cisse \and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
32 Myriam Côté \and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
33 Dumitru Erhan \and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
34 Jeremy Eustache \and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
35 Xavier Glorot \and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
36 Xavier Muller \and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
37 Sylvain Pannetier Lebeuf \and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
38 Razvan Pascanu \and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
39 Salah Rifai \and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
40 Francois Savard \and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
41 Guillaume Sicard
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
42 }
594
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
43 \date{{\tt bengioy@iro.umontreal.ca}, Dept. IRO, U. Montreal, P.O. Box 6128, Centre-Ville branch, H3C 3J7, Montreal (Qc), Canada}
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
44 \jmlrheading{}{2010}{}{10/2010}{XX/2011}{Yoshua Bengio et al}
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
45 \editor{}
593
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
46
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
47 %\makeanontitle
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
48 \maketitle
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
49
594
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
50 {\bf Running title: Deep Self-Taught Learning}
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
51
593
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
52 %\vspace*{-2mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
53 \begin{abstract}
594
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
54 Recent theoretical and empirical work in statistical machine learning has demonstrated the potential of learning algorithms for deep architectures, i.e., function classes obtained by composing multiple levels of representation. Self-taught learning (exploiting unlabeled examples or examples from other distributions) has already been applied to deep learners, but mostly to show the advantage of unlabeled examples. Here we explore the advantage brought by {\em out-of-distribution examples}. For this purpose we developed a powerful generator of stochastic variations and noise processes for character images, including not only affine transformations but also slant, local elastic deformations, changes in thickness, background images, grey level changes, contrast, occlusion, and various types of noise. The out-of-distribution examples are obtained from these highly distorted images or by including examples of object classes different from those in the target test set. We show that {\em deep learners benefit more from out-of-distribution examples than a corresponding shallow learner}, at least in a large-scale handwritten character recognition setting. In fact, we show that they {\em beat previously published results and reach human-level performance}.
593
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
55 \end{abstract}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
56 %\vspace*{-3mm}
594
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
57
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
58 \begin{keywords}
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
59 Deep learning, self-taught learning, out-of-distribution examples, handwritten character recognition, multi-task learning
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
60 \end{keywords}
593
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
61 %\keywords{self-taught learning \and multi-task learning \and out-of-distribution examples \and deep learning \and handwriting recognition}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
62
594
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
63
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
64
593
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
65 \section{Introduction}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
66 %\vspace*{-1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
67
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
68 {\bf Deep Learning} has emerged as a promising new area of research in
594
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
69 statistical machine learning~\citep{Hinton06,ranzato-07-small,Bengio-nips-2006,VincentPLarochelleH2008,ranzato-08,TaylorHintonICML2009,Larochelle-jmlr-2009,Salakhutdinov+Hinton-2009,HonglakL2009,HonglakLNIPS2009,Jarrett-ICCV2009,Taylor-cvpr-2010}. See \citet{Bengio-2009} for a review.
593
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
70 Learning algorithms for deep architectures are centered on the learning
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
71 of useful representations of data, which are better suited to the task at hand,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
72 and are organized in a hierarchy with multiple levels.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
73 This is in part inspired by observations of the mammalian visual cortex,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
74 which consists of a chain of processing elements, each of which is associated with a
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
75 different representation of the raw visual input. In fact,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
76 it was found recently that the features learnt in deep architectures resemble
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
77 those observed in the first two of these stages (in areas V1 and V2
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
78 of visual cortex) \citep{HonglakL2008}, and that they become more and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
79 more invariant to factors of variation (such as camera movement) in
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
80 higher layers~\citep{Goodfellow2009}.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
81 Learning a hierarchy of features increases the
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
82 ease and practicality of developing representations that are at once
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
83 tailored to specific tasks, yet are able to borrow statistical strength
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
84 from other related tasks (e.g., modeling different kinds of objects). Finally, learning the
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
85 feature representation can lead to higher-level (more abstract, more
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
86 general) features that are more robust to unanticipated sources of
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
87 variance extant in real data.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
88
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
89 {\bf Self-taught learning}~\citep{RainaR2007} is a paradigm that combines principles
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
90 of semi-supervised and multi-task learning: the learner can exploit examples
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
91 that are unlabeled and possibly come from a distribution different from the target
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
92 distribution, e.g., from other classes than those of interest.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
93 It has already been shown that deep learners can clearly take advantage of
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
94 unsupervised learning and unlabeled examples~\citep{Bengio-2009,WestonJ2008-small},
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
95 but more needs to be done to explore the impact
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
96 of {\em out-of-distribution} examples and of the {\em multi-task} setting
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
97 (one exception is~\citep{CollobertR2008}, which uses a different kind
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
98 of learning algorithm). In particular the {\em relative
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
99 advantage of deep learning} for these settings has not been evaluated.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
100 The hypothesis discussed in the conclusion is that in the context of
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
101 multi-task learning and the availability of out-of-distribution training examples,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
102 a deep hierarchy of features
594
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
103 may be better able to provide {\em sharing of statistical strength}
593
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
104 between different regions in input space or different tasks, compared to
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
105 a shallow learner.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
106
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
107 Whereas a deep architecture can in principle be more powerful than a
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
108 shallow one in terms of representation, depth appears to render the
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
109 training problem more difficult in terms of optimization and local minima.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
110 It is also only recently that successful algorithms were proposed to
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
111 overcome some of these difficulties. All are based on unsupervised
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
112 learning, often in an greedy layer-wise ``unsupervised pre-training''
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
113 stage~\citep{Bengio-2009}. One of these layer initialization techniques,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
114 applied here, is the Denoising
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
115 Auto-encoder~(DA)~\citep{VincentPLarochelleH2008-very-small} (see Figure~\ref{fig:da}),
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
116 which
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
117 performed similarly or better than previously proposed Restricted Boltzmann
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
118 Machines in terms of unsupervised extraction of a hierarchy of features
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
119 useful for classification. Each layer is trained to denoise its
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
120 input, creating a layer of features that can be used as input for the next layer.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
121
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
122 %The principle is that each layer starting from
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
123 %the bottom is trained to encode its input (the output of the previous
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
124 %layer) and to reconstruct it from a corrupted version. After this
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
125 %unsupervised initialization, the stack of DAs can be
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
126 %converted into a deep supervised feedforward neural network and fine-tuned by
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
127 %stochastic gradient descent.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
128
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
129 %
594
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
130 The {\bf main claim} of this paper is that deep learners (with several levels of representation) can
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
131 {\bf benefit more from self-taught learning than shallow learners} (with a single
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
132 level), both in the context of the multi-task setting and from {\em
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
133 out-of-distribution examples} in general. Because we are able to improve on state-of-the-art
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
134 performance and reach human-level performance
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
135 on a large-scale task, we consider that this paper is also a contribution
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
136 to advance the application of machine learning to handwritten character recognition.
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
137 More precisely, we ask and answer the following questions:
593
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
138
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
139 %\begin{enumerate}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
140 $\bullet$ %\item
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
141 Do the good results previously obtained with deep architectures on the
594
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
142 MNIST digit images generalize to the setting of a similar but much larger and richer
593
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
143 dataset, the NIST special database 19, with 62 classes and around 800k examples?
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
144
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
145 $\bullet$ %\item
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
146 To what extent does the perturbation of input images (e.g. adding
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
147 noise, affine transformations, background images) make the resulting
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
148 classifiers better not only on similarly perturbed images but also on
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
149 the {\em original clean examples}? We study this question in the
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
150 context of the 62-class and 10-class tasks of the NIST special database 19.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
151
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
152 $\bullet$ %\item
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
153 Do deep architectures {\em benefit {\bf more} from such out-of-distribution}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
154 examples, i.e. do they benefit more from the self-taught learning~\citep{RainaR2007} framework?
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
155 We use highly perturbed examples to generate out-of-distribution examples.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
156
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
157 $\bullet$ %\item
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
158 Similarly, does the feature learning step in deep learning algorithms benefit {\bf more}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
159 from training with moderately {\em different classes} (i.e. a multi-task learning scenario) than
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
160 a corresponding shallow and purely supervised architecture?
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
161 We train on 62 classes and test on 10 (digits) or 26 (upper case or lower case)
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
162 to answer this question.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
163 %\end{enumerate}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
164
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
165 Our experimental results provide positive evidence towards all of these questions,
594
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
166 as well as {\em classifiers that reach human-level performance on 62-class isolated character
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
167 recognition and beat previously published results on the NIST dataset (special database 19)}.
593
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
168 To achieve these results, we introduce in the next section a sophisticated system
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
169 for stochastically transforming character images and then explain the methodology,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
170 which is based on training with or without these transformed images and testing on
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
171 clean ones. We measure the relative advantage of out-of-distribution examples
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
172 (perturbed or out-of-class)
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
173 for a deep learner vs a supervised shallow one.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
174 Code for generating these transformations as well as for the deep learning
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
175 algorithms are made available at {\tt http://hg.assembla.com/ift6266}.
594
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
176 We also estimate the relative advantage for deep learners of training with
593
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
177 other classes than those of interest, by comparing learners trained with
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
178 62 classes with learners trained with only a subset (on which they
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
179 are then tested).
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
180 The conclusion discusses
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
181 the more general question of why deep learners may benefit so much from
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
182 the self-taught learning framework. Since out-of-distribution data
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
183 (perturbed or from other related classes) is very common, this conclusion
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
184 is of practical importance.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
185
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
186 %\vspace*{-3mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
187 %\newpage
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
188 \section{Perturbed and Transformed Character Images}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
189 \label{s:perturbations}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
190 %\vspace*{-2mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
191
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
192 \begin{wrapfigure}[8]{l}{0.15\textwidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
193 %\begin{minipage}[b]{0.14\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
194 %\vspace*{-5mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
195 \begin{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
196 \includegraphics[scale=.4]{images/Original.png}\\
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
197 {\bf Original}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
198 \end{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
199 \end{wrapfigure}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
200 %%\vspace{0.7cm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
201 %\end{minipage}%
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
202 %\hspace{0.3cm}\begin{minipage}[b]{0.86\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
203 This section describes the different transformations we used to stochastically
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
204 transform $32 \times 32$ source images (such as the one on the left)
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
205 in order to obtain data from a larger distribution which
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
206 covers a domain substantially larger than the clean characters distribution from
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
207 which we start.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
208 Although character transformations have been used before to
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
209 improve character recognizers, this effort is on a large scale both
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
210 in number of classes and in the complexity of the transformations, hence
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
211 in the complexity of the learning task.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
212 The code for these transformations (mostly python) is available at
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
213 {\tt http://hg.assembla.com/ift6266}. All the modules in the pipeline share
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
214 a global control parameter ($0 \le complexity \le 1$) that allows one to modulate the
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
215 amount of deformation or noise introduced.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
216 There are two main parts in the pipeline. The first one,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
217 from slant to pinch below, performs transformations. The second
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
218 part, from blur to contrast, adds different kinds of noise.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
219 %\end{minipage}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
220
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
221 %\vspace*{1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
222 \subsection{Transformations}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
223 %{\large\bf 2.1 Transformations}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
224 %\vspace*{1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
225
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
226 \subsubsection*{Thickness}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
227
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
228 %\begin{wrapfigure}[7]{l}{0.15\textwidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
229 \begin{minipage}[b]{0.14\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
230 %\centering
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
231 \begin{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
232 \vspace*{-5mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
233 \includegraphics[scale=.4]{images/Thick_only.png}\\
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
234 %{\bf Thickness}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
235 \end{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
236 \vspace{.6cm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
237 \end{minipage}%
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
238 \hspace{0.3cm}\begin{minipage}[b]{0.86\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
239 %\end{wrapfigure}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
240 To change character {\bf thickness}, morphological operators of dilation and erosion~\citep{Haralick87,Serra82}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
241 are applied. The neighborhood of each pixel is multiplied
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
242 element-wise with a {\em structuring element} matrix.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
243 The pixel value is replaced by the maximum or the minimum of the resulting
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
244 matrix, respectively for dilation or erosion. Ten different structural elements with
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
245 increasing dimensions (largest is $5\times5$) were used. For each image,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
246 randomly sample the operator type (dilation or erosion) with equal probability and one structural
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
247 element from a subset of the $n=round(m \times complexity)$ smallest structuring elements
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
248 where $m=10$ for dilation and $m=6$ for erosion (to avoid completely erasing thin characters).
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
249 A neutral element (no transformation)
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
250 is always present in the set.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
251 %%\vspace{.4cm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
252 \end{minipage}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
253
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
254 \vspace{2mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
255
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
256 \subsubsection*{Slant}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
257 \vspace*{2mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
258
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
259 \begin{minipage}[b]{0.14\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
260 \centering
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
261 \includegraphics[scale=.4]{images/Slant_only.png}\\
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
262 %{\bf Slant}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
263 \end{minipage}%
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
264 \hspace{0.3cm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
265 \begin{minipage}[b]{0.83\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
266 %\centering
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
267 To produce {\bf slant}, each row of the image is shifted
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
268 proportionally to its height: $shift = round(slant \times height)$.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
269 $slant \sim U[-complexity,complexity]$.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
270 The shift is randomly chosen to be either to the left or to the right.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
271 \vspace{5mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
272 \end{minipage}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
273 %\vspace*{-4mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
274
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
275 %\newpage
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
276
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
277 \subsubsection*{Affine Transformations}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
278
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
279 \begin{minipage}[b]{0.14\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
280 %\centering
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
281 %\begin{wrapfigure}[8]{l}{0.15\textwidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
282 \begin{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
283 \includegraphics[scale=.4]{images/Affine_only.png}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
284 \vspace*{6mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
285 %{\small {\bf Affine \mbox{Transformation}}}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
286 \end{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
287 %\end{wrapfigure}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
288 \end{minipage}%
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
289 \hspace{0.3cm}\begin{minipage}[b]{0.86\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
290 \noindent A $2 \times 3$ {\bf affine transform} matrix (with
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
291 parameters $(a,b,c,d,e,f)$) is sampled according to the $complexity$.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
292 Output pixel $(x,y)$ takes the value of input pixel
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
293 nearest to $(ax+by+c,dx+ey+f)$,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
294 producing scaling, translation, rotation and shearing.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
295 Marginal distributions of $(a,b,c,d,e,f)$ have been tuned to
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
296 forbid large rotations (to avoid confusing classes) but to give good
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
297 variability of the transformation: $a$ and $d$ $\sim U[1-3
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
298 complexity,1+3\,complexity]$, $b$ and $e$ $\sim U[-3 \,complexity,3\,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
299 complexity]$, and $c$ and $f \sim U[-4 \,complexity, 4 \,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
300 complexity]$.\\
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
301 \end{minipage}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
302
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
303 %\vspace*{-4.5mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
304 \subsubsection*{Local Elastic Deformations}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
305
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
306 %\begin{minipage}[t]{\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
307 %\begin{wrapfigure}[7]{l}{0.15\textwidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
308 %\hspace*{-8mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
309 \begin{minipage}[b]{0.14\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
310 %\centering
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
311 \begin{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
312 \vspace*{5mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
313 \includegraphics[scale=.4]{images/Localelasticdistorsions_only.png}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
314 %{\bf Local Elastic Deformation}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
315 \end{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
316 %\end{wrapfigure}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
317 \end{minipage}%
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
318 \hspace{3mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
319 \begin{minipage}[b]{0.85\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
320 %%\vspace*{-20mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
321 The {\bf local elastic deformation}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
322 module induces a ``wiggly'' effect in the image, following~\citet{SimardSP03-short},
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
323 which provides more details.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
324 The intensity of the displacement fields is given by
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
325 $\alpha = \sqrt[3]{complexity} \times 10.0$, which are
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
326 convolved with a Gaussian 2D kernel (resulting in a blur) of
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
327 standard deviation $\sigma = 10 - 7 \times\sqrt[3]{complexity}$.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
328 \vspace{2mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
329 \end{minipage}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
330
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
331 \vspace*{4mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
332
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
333 \subsubsection*{Pinch}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
334
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
335 \begin{minipage}[b]{0.14\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
336 %\centering
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
337 %\begin{wrapfigure}[7]{l}{0.15\textwidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
338 %\vspace*{-5mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
339 \begin{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
340 \includegraphics[scale=.4]{images/Pinch_only.png}\\
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
341 \vspace*{15mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
342 %{\bf Pinch}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
343 \end{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
344 %\end{wrapfigure}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
345 %%\vspace{.6cm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
346 \end{minipage}%
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
347 \hspace{0.3cm}\begin{minipage}[b]{0.86\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
348 The {\bf pinch} module applies the ``Whirl and pinch'' GIMP filter with whirl set to 0.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
349 A pinch is ``similar to projecting the image onto an elastic
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
350 surface and pressing or pulling on the center of the surface'' (GIMP documentation manual).
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
351 For a square input image, draw a radius-$r$ disk
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
352 around its center $C$. Any pixel $P$ belonging to
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
353 that disk has its value replaced by
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
354 the value of a ``source'' pixel in the original image,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
355 on the line that goes through $C$ and $P$, but
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
356 at some other distance $d_2$. Define $d_1=distance(P,C)$
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
357 and $d_2 = sin(\frac{\pi{}d_1}{2r})^{-pinch} \times
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
358 d_1$, where $pinch$ is a parameter of the filter.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
359 The actual value is given by bilinear interpolation considering the pixels
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
360 around the (non-integer) source position thus found.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
361 Here $pinch \sim U[-complexity, 0.7 \times complexity]$.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
362 %%\vspace{1.5cm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
363 \end{minipage}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
364
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
365 %\vspace{1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
366
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
367 %{\large\bf 2.2 Injecting Noise}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
368 \subsection{Injecting Noise}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
369 %\vspace{2mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
370
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
371 \subsubsection*{Motion Blur}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
372
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
373 %%\vspace*{-.2cm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
374 \begin{minipage}[t]{0.14\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
375 \centering
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
376 \vspace*{0mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
377 \includegraphics[scale=.4]{images/Motionblur_only.png}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
378 %{\bf Motion Blur}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
379 \end{minipage}%
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
380 \hspace{0.3cm}\begin{minipage}[t]{0.83\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
381 %%\vspace*{.5mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
382 \vspace*{2mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
383 The {\bf motion blur} module is GIMP's ``linear motion blur'', which
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
384 has parameters $length$ and $angle$. The value of
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
385 a pixel in the final image is approximately the mean of the first $length$ pixels
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
386 found by moving in the $angle$ direction,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
387 $angle \sim U[0,360]$ degrees, and $length \sim {\rm Normal}(0,(3 \times complexity)^2)$.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
388 %\vspace{5mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
389 \end{minipage}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
390
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
391 %\vspace*{1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
392
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
393 \subsubsection*{Occlusion}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
394
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
395 \begin{minipage}[t]{0.14\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
396 \centering
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
397 \vspace*{3mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
398 \includegraphics[scale=.4]{images/occlusion_only.png}\\
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
399 %{\bf Occlusion}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
400 %%\vspace{.5cm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
401 \end{minipage}%
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
402 \hspace{0.3cm}\begin{minipage}[t]{0.83\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
403 %\vspace*{-18mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
404 The {\bf occlusion} module selects a random rectangle from an {\em occluder} character
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
405 image and places it over the original {\em occluded}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
406 image. Pixels are combined by taking the max(occluder, occluded),
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
407 i.e. keeping the lighter ones.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
408 The rectangle corners
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
409 are sampled so that larger complexity gives larger rectangles.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
410 The destination position in the occluded image are also sampled
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
411 according to a normal distribution.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
412 This module is skipped with probability 60\%.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
413 %%\vspace{7mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
414 \end{minipage}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
415
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
416 %\vspace*{1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
417 \subsubsection*{Gaussian Smoothing}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
418
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
419 %\begin{wrapfigure}[8]{l}{0.15\textwidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
420 %\vspace*{-6mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
421 \begin{minipage}[t]{0.14\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
422 \begin{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
423 %\centering
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
424 \vspace*{6mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
425 \includegraphics[scale=.4]{images/Bruitgauss_only.png}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
426 %{\bf Gaussian Smoothing}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
427 \end{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
428 %\end{wrapfigure}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
429 %%\vspace{.5cm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
430 \end{minipage}%
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
431 \hspace{0.3cm}\begin{minipage}[t]{0.86\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
432 With the {\bf Gaussian smoothing} module,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
433 different regions of the image are spatially smoothed.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
434 This is achieved by first convolving
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
435 the image with an isotropic Gaussian kernel of
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
436 size and variance chosen uniformly in the ranges $[12,12 + 20 \times
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
437 complexity]$ and $[2,2 + 6 \times complexity]$. This filtered image is normalized
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
438 between $0$ and $1$. We also create an isotropic weighted averaging window, of the
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
439 kernel size, with maximum value at the center. For each image we sample
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
440 uniformly from $3$ to $3 + 10 \times complexity$ pixels that will be
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
441 averaging centers between the original image and the filtered one. We
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
442 initialize to zero a mask matrix of the image size. For each selected pixel
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
443 we add to the mask the averaging window centered on it. The final image is
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
444 computed from the following element-wise operation: $\frac{image + filtered\_image
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
445 \times mask}{mask+1}$.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
446 This module is skipped with probability 75\%.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
447 \end{minipage}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
448
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
449 %\newpage
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
450
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
451 %\vspace*{-9mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
452 \subsubsection*{Permute Pixels}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
453
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
454 %\hspace*{-3mm}\begin{minipage}[t]{0.18\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
455 %\centering
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
456 \begin{minipage}[t]{0.14\textwidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
457 %\begin{wrapfigure}[7]{l}{
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
458 %\vspace*{-5mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
459 \begin{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
460 \vspace*{1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
461 \includegraphics[scale=.4]{images/Permutpixel_only.png}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
462 %{\small\bf Permute Pixels}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
463 \end{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
464 %\end{wrapfigure}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
465 \end{minipage}%
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
466 \hspace{3mm}\begin{minipage}[t]{0.86\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
467 \vspace*{1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
468 %%\vspace*{-20mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
469 This module {\bf permutes neighbouring pixels}. It first selects a
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
470 fraction $\frac{complexity}{3}$ of pixels randomly in the image. Each
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
471 of these pixels is then sequentially exchanged with a random pixel
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
472 among its four nearest neighbors (on its left, right, top or bottom).
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
473 This module is skipped with probability 80\%.\\
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
474 %\vspace*{1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
475 \end{minipage}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
476
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
477 %\vspace{-3mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
478
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
479 \subsubsection*{Gaussian Noise}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
480
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
481 \begin{minipage}[t]{0.14\textwidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
482 %\begin{wrapfigure}[7]{l}{
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
483 %%\vspace*{-3mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
484 \begin{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
485 %\hspace*{-3mm}\begin{minipage}[t]{0.18\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
486 %\centering
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
487 \vspace*{0mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
488 \includegraphics[scale=.4]{images/Distorsiongauss_only.png}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
489 %{\small \bf Gauss. Noise}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
490 \end{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
491 %\end{wrapfigure}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
492 \end{minipage}%
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
493 \hspace{0.3cm}\begin{minipage}[t]{0.86\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
494 \vspace*{1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
495 %\vspace*{12mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
496 The {\bf Gaussian noise} module simply adds, to each pixel of the image independently, a
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
497 noise $\sim Normal(0,(\frac{complexity}{10})^2)$.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
498 This module is skipped with probability 70\%.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
499 %%\vspace{1.1cm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
500 \end{minipage}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
501
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
502 %\vspace*{1.2cm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
503
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
504 \subsubsection*{Background Image Addition}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
505
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
506 \begin{minipage}[t]{\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
507 \begin{minipage}[t]{0.14\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
508 \centering
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
509 \vspace*{0mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
510 \includegraphics[scale=.4]{images/background_other_only.png}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
511 %{\small \bf Bg Image}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
512 \end{minipage}%
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
513 \hspace{0.3cm}\begin{minipage}[t]{0.83\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
514 \vspace*{1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
515 Following~\citet{Larochelle-jmlr-2009}, the {\bf background image} module adds a random
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
516 background image behind the letter, from a randomly chosen natural image,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
517 with contrast adjustments depending on $complexity$, to preserve
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
518 more or less of the original character image.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
519 %%\vspace{.8cm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
520 \end{minipage}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
521 \end{minipage}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
522 %%\vspace{-.7cm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
523
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
524 \subsubsection*{Salt and Pepper Noise}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
525
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
526 \begin{minipage}[t]{0.14\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
527 \centering
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
528 \vspace*{0mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
529 \includegraphics[scale=.4]{images/Poivresel_only.png}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
530 %{\small \bf Salt \& Pepper}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
531 \end{minipage}%
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
532 \hspace{0.3cm}\begin{minipage}[t]{0.83\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
533 \vspace*{1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
534 The {\bf salt and pepper noise} module adds noise $\sim U[0,1]$ to random subsets of pixels.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
535 The number of selected pixels is $0.2 \times complexity$.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
536 This module is skipped with probability 75\%.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
537 %%\vspace{.9cm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
538 \end{minipage}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
539 %%\vspace{-.7cm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
540
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
541 %\vspace{1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
542 \subsubsection*{Scratches}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
543
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
544 \begin{minipage}[t]{0.14\textwidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
545 %\begin{wrapfigure}[7]{l}{
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
546 %\begin{minipage}[t]{0.14\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
547 %\centering
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
548 \begin{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
549 \vspace*{4mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
550 %\hspace*{-1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
551 \includegraphics[scale=.4]{images/Rature_only.png}\\
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
552 %{\bf Scratches}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
553 \end{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
554 \end{minipage}%
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
555 %\end{wrapfigure}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
556 \hspace{0.3cm}\begin{minipage}[t]{0.86\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
557 %%\vspace{.4cm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
558 The {\bf scratches} module places line-like white patches on the image. The
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
559 lines are heavily transformed images of the digit ``1'' (one), chosen
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
560 at random among 500 such 1 images,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
561 randomly cropped and rotated by an angle $\sim Normal(0,(100 \times
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
562 complexity)^2$ (in degrees), using bi-cubic interpolation.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
563 Two passes of a grey-scale morphological erosion filter
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
564 are applied, reducing the width of the line
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
565 by an amount controlled by $complexity$.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
566 This module is skipped with probability 85\%. The probabilities
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
567 of applying 1, 2, or 3 patches are (50\%,30\%,20\%).
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
568 \end{minipage}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
569
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
570 %\vspace*{1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
571
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
572 \subsubsection*{Grey Level and Contrast Changes}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
573
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
574 \begin{minipage}[t]{0.15\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
575 \centering
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
576 \vspace*{0mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
577 \includegraphics[scale=.4]{images/Contrast_only.png}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
578 %{\bf Grey Level \& Contrast}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
579 \end{minipage}%
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
580 \hspace{3mm}\begin{minipage}[t]{0.85\linewidth}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
581 \vspace*{1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
582 The {\bf grey level and contrast} module changes the contrast by changing grey levels, and may invert the image polarity (white
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
583 to black and black to white). The contrast is $C \sim U[1-0.85 \times complexity,1]$
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
584 so the image is normalized into $[\frac{1-C}{2},1-\frac{1-C}{2}]$. The
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
585 polarity is inverted with probability 50\%.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
586 %%\vspace{.7cm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
587 \end{minipage}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
588 %\vspace{2mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
589
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
590
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
591 \iffalse
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
592 \begin{figure}[ht]
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
593 \centerline{\resizebox{.9\textwidth}{!}{\includegraphics{images/example_t.png}}}\\
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
594 \caption{Illustration of the pipeline of stochastic
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
595 transformations applied to the image of a lower-case \emph{t}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
596 (the upper left image). Each image in the pipeline (going from
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
597 left to right, first top line, then bottom line) shows the result
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
598 of applying one of the modules in the pipeline. The last image
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
599 (bottom right) is used as training example.}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
600 \label{fig:pipeline}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
601 \end{figure}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
602 \fi
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
603
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
604 %\vspace*{-3mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
605 \section{Experimental Setup}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
606 %\vspace*{-1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
607
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
608 Much previous work on deep learning had been performed on
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
609 the MNIST digits task~\citep{Hinton06,ranzato-07-small,Bengio-nips-2006,Salakhutdinov+Hinton-2009},
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
610 with 60~000 examples, and variants involving 10~000
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
611 examples~\citep{Larochelle-jmlr-toappear-2008,VincentPLarochelleH2008}.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
612 The focus here is on much larger training sets, from 10 times to
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
613 to 1000 times larger, and 62 classes.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
614
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
615 The first step in constructing the larger datasets (called NISTP and P07) is to sample from
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
616 a {\em data source}: {\bf NIST} (NIST database 19), {\bf Fonts}, {\bf Captchas},
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
617 and {\bf OCR data} (scanned machine printed characters). Once a character
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
618 is sampled from one of these sources (chosen randomly), the second step is to
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
619 apply a pipeline of transformations and/or noise processes described in section \ref{s:perturbations}.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
620
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
621 To provide a baseline of error rate comparison we also estimate human performance
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
622 on both the 62-class task and the 10-class digits task.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
623 We compare the best Multi-Layer Perceptrons (MLP) against
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
624 the best Stacked Denoising Auto-encoders (SDA), when
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
625 both models' hyper-parameters are selected to minimize the validation set error.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
626 We also provide a comparison against a precise estimate
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
627 of human performance obtained via Amazon's Mechanical Turk (AMT)
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
628 service (http://mturk.com).
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
629 AMT users are paid small amounts
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
630 of money to perform tasks for which human intelligence is required.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
631 Mechanical Turk has been used extensively in natural language processing and vision.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
632 %processing \citep{SnowEtAl2008} and vision
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
633 %\citep{SorokinAndForsyth2008,whitehill09}.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
634 AMT users were presented
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
635 with 10 character images (from a test set) and asked to choose 10 corresponding ASCII
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
636 characters. They were forced to choose a single character class (either among the
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
637 62 or 10 character classes) for each image.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
638 80 subjects classified 2500 images per (dataset,task) pair.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
639 Different humans labelers sometimes provided a different label for the same
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
640 example, and we were able to estimate the error variance due to this effect
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
641 because each image was classified by 3 different persons.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
642 The average error of humans on the 62-class task NIST test set
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
643 is 18.2\%, with a standard error of 0.1\%.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
644
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
645 %\vspace*{-3mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
646 \subsection{Data Sources}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
647 %\vspace*{-2mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
648
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
649 %\begin{itemize}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
650 %\item
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
651 {\bf NIST.}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
652 Our main source of characters is the NIST Special Database 19~\citep{Grother-1995},
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
653 widely used for training and testing character
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
654 recognition systems~\citep{Granger+al-2007,Cortes+al-2000,Oliveira+al-2002-short,Milgram+al-2005}.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
655 The dataset is composed of 814255 digits and characters (upper and lower cases), with hand checked classifications,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
656 extracted from handwritten sample forms of 3600 writers. The characters are labelled by one of the 62 classes
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
657 corresponding to ``0''-``9'',``A''-``Z'' and ``a''-``z''. The dataset contains 8 parts (partitions) of varying complexity.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
658 The fourth partition (called $hsf_4$, 82587 examples),
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
659 experimentally recognized to be the most difficult one, is the one recommended
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
660 by NIST as a testing set and is used in our work as well as some previous work~\citep{Granger+al-2007,Cortes+al-2000,Oliveira+al-2002-short,Milgram+al-2005}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
661 for that purpose. We randomly split the remainder (731668 examples) into a training set and a validation set for
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
662 model selection.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
663 The performances reported by previous work on that dataset mostly use only the digits.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
664 Here we use all the classes both in the training and testing phase. This is especially
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
665 useful to estimate the effect of a multi-task setting.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
666 The distribution of the classes in the NIST training and test sets differs
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
667 substantially, with relatively many more digits in the test set, and a more uniform distribution
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
668 of letters in the test set (whereas in the training set they are distributed
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
669 more like in natural text).
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
670 %\vspace*{-1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
671
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
672 %\item
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
673 {\bf Fonts.}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
674 In order to have a good variety of sources we downloaded an important number of free fonts from:
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
675 {\tt http://cg.scs.carleton.ca/\textasciitilde luc/freefonts.html}.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
676 % TODO: pointless to anonymize, it's not pointing to our work
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
677 Including the operating system's (Windows 7) fonts, there is a total of $9817$ different fonts that we can choose uniformly from.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
678 The chosen {\tt ttf} file is either used as input of the Captcha generator (see next item) or, by producing a corresponding image,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
679 directly as input to our models.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
680 %\vspace*{-1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
681
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
682 %\item
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
683 {\bf Captchas.}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
684 The Captcha data source is an adaptation of the \emph{pycaptcha} library (a python based captcha generator library) for
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
685 generating characters of the same format as the NIST dataset. This software is based on
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
686 a random character class generator and various kinds of transformations similar to those described in the previous sections.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
687 In order to increase the variability of the data generated, many different fonts are used for generating the characters.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
688 Transformations (slant, distortions, rotation, translation) are applied to each randomly generated character with a complexity
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
689 depending on the value of the complexity parameter provided by the user of the data source.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
690 %Two levels of complexity are allowed and can be controlled via an easy to use facade class. %TODO: what's a facade class?
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
691 %\vspace*{-1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
692
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
693 %\item
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
694 {\bf OCR data.}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
695 A large set (2 million) of scanned, OCRed and manually verified machine-printed
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
696 characters where included as an
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
697 additional source. This set is part of a larger corpus being collected by the Image Understanding
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
698 Pattern Recognition Research group led by Thomas Breuel at University of Kaiserslautern
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
699 ({\tt http://www.iupr.com}), and which will be publicly released.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
700 %TODO: let's hope that Thomas is not a reviewer! :) Seriously though, maybe we should anonymize this
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
701 %\end{itemize}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
702
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
703 %\vspace*{-3mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
704 \subsection{Data Sets}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
705 %\vspace*{-2mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
706
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
707 All data sets contain 32$\times$32 grey-level images (values in $[0,1]$) associated with a label
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
708 from one of the 62 character classes.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
709 %\begin{itemize}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
710 %\vspace*{-1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
711
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
712 %\item
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
713 {\bf NIST.} This is the raw NIST special database 19~\citep{Grother-1995}. It has
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
714 \{651668 / 80000 / 82587\} \{training / validation / test\} examples.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
715 %\vspace*{-1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
716
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
717 %\item
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
718 {\bf P07.} This dataset is obtained by taking raw characters from all four of the above sources
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
719 and sending them through the transformation pipeline described in section \ref{s:perturbations}.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
720 For each new example to generate, a data source is selected with probability $10\%$ from the fonts,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
721 $25\%$ from the captchas, $25\%$ from the OCR data and $40\%$ from NIST. We apply all the transformations in the
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
722 order given above, and for each of them we sample uniformly a \emph{complexity} in the range $[0,0.7]$.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
723 It has \{81920000 / 80000 / 20000\} \{training / validation / test\} examples.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
724 %\vspace*{-1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
725
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
726 %\item
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
727 {\bf NISTP.} This one is equivalent to P07 (complexity parameter of $0.7$ with the same proportions of data sources)
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
728 except that we only apply
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
729 transformations from slant to pinch. Therefore, the character is
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
730 transformed but no additional noise is added to the image, giving images
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
731 closer to the NIST dataset.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
732 It has \{81920000 / 80000 / 20000\} \{training / validation / test\} examples.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
733 %\end{itemize}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
734
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
735 %\vspace*{-3mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
736 \subsection{Models and their Hyperparameters}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
737 %\vspace*{-2mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
738
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
739 The experiments are performed using MLPs (with a single
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
740 hidden layer) and SDAs.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
741 \emph{Hyper-parameters are selected based on the {\bf NISTP} validation set error.}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
742
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
743 {\bf Multi-Layer Perceptrons (MLP).}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
744 Whereas previous work had compared deep architectures to both shallow MLPs and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
745 SVMs, we only compared to MLPs here because of the very large datasets used
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
746 (making the use of SVMs computationally challenging because of their quadratic
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
747 scaling behavior). Preliminary experiments on training SVMs (libSVM) with subsets of the training
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
748 set allowing the program to fit in memory yielded substantially worse results
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
749 than those obtained with MLPs. For training on nearly a billion examples
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
750 (with the perturbed data), the MLPs and SDA are much more convenient than
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
751 classifiers based on kernel methods.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
752 The MLP has a single hidden layer with $\tanh$ activation functions, and softmax (normalized
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
753 exponentials) on the output layer for estimating $P(class | image)$.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
754 The number of hidden units is taken in $\{300,500,800,1000,1500\}$.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
755 Training examples are presented in minibatches of size 20. A constant learning
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
756 rate was chosen among $\{0.001, 0.01, 0.025, 0.075, 0.1, 0.5\}$.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
757 %through preliminary experiments (measuring performance on a validation set),
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
758 %and $0.1$ (which was found to work best) was then selected for optimizing on
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
759 %the whole training sets.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
760 %\vspace*{-1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
761
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
762
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
763 {\bf Stacked Denoising Auto-Encoders (SDA).}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
764 Various auto-encoder variants and Restricted Boltzmann Machines (RBMs)
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
765 can be used to initialize the weights of each layer of a deep MLP (with many hidden
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
766 layers)~\citep{Hinton06,ranzato-07-small,Bengio-nips-2006},
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
767 apparently setting parameters in the
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
768 basin of attraction of supervised gradient descent yielding better
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
769 generalization~\citep{Erhan+al-2010}. This initial {\em unsupervised
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
770 pre-training phase} uses all of the training images but not the training labels.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
771 Each layer is trained in turn to produce a new representation of its input
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
772 (starting from the raw pixels).
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
773 It is hypothesized that the
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
774 advantage brought by this procedure stems from a better prior,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
775 on the one hand taking advantage of the link between the input
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
776 distribution $P(x)$ and the conditional distribution of interest
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
777 $P(y|x)$ (like in semi-supervised learning), and on the other hand
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
778 taking advantage of the expressive power and bias implicit in the
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
779 deep architecture (whereby complex concepts are expressed as
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
780 compositions of simpler ones through a deep hierarchy).
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
781
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
782 \begin{figure}[ht]
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
783 %\vspace*{-2mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
784 \centerline{\resizebox{0.8\textwidth}{!}{\includegraphics{images/denoising_autoencoder_small.pdf}}}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
785 %\vspace*{-2mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
786 \caption{Illustration of the computations and training criterion for the denoising
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
787 auto-encoder used to pre-train each layer of the deep architecture. Input $x$ of
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
788 the layer (i.e. raw input or output of previous layer)
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
789 s corrupted into $\tilde{x}$ and encoded into code $y$ by the encoder $f_\theta(\cdot)$.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
790 The decoder $g_{\theta'}(\cdot)$ maps $y$ to reconstruction $z$, which
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
791 is compared to the uncorrupted input $x$ through the loss function
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
792 $L_H(x,z)$, whose expected value is approximately minimized during training
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
793 by tuning $\theta$ and $\theta'$.}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
794 \label{fig:da}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
795 %\vspace*{-2mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
796 \end{figure}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
797
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
798 Here we chose to use the Denoising
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
799 Auto-encoder~\citep{VincentPLarochelleH2008} as the building block for
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
800 these deep hierarchies of features, as it is simple to train and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
801 explain (see Figure~\ref{fig:da}, as well as
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
802 tutorial and code there: {\tt http://deeplearning.net/tutorial}),
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
803 provides efficient inference, and yielded results
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
804 comparable or better than RBMs in series of experiments
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
805 \citep{VincentPLarochelleH2008}. During training, a Denoising
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
806 Auto-encoder is presented with a stochastically corrupted version
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
807 of the input and trained to reconstruct the uncorrupted input,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
808 forcing the hidden units to represent the leading regularities in
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
809 the data. Here we use the random binary masking corruption
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
810 (which sets to 0 a random subset of the inputs).
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
811 Once it is trained, in a purely unsupervised way,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
812 its hidden units' activations can
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
813 be used as inputs for training a second one, etc.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
814 After this unsupervised pre-training stage, the parameters
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
815 are used to initialize a deep MLP, which is fine-tuned by
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
816 the same standard procedure used to train them (see previous section).
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
817 The SDA hyper-parameters are the same as for the MLP, with the addition of the
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
818 amount of corruption noise (we used the masking noise process, whereby a
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
819 fixed proportion of the input values, randomly selected, are zeroed), and a
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
820 separate learning rate for the unsupervised pre-training stage (selected
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
821 from the same above set). The fraction of inputs corrupted was selected
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
822 among $\{10\%, 20\%, 50\%\}$. Another hyper-parameter is the number
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
823 of hidden layers but it was fixed to 3 based on previous work with
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
824 SDAs on MNIST~\citep{VincentPLarochelleH2008}. The size of the hidden
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
825 layers was kept constant across hidden layers, and the best results
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
826 were obtained with the largest values that we could experiment
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
827 with given our patience, with 1000 hidden units.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
828
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
829 %\vspace*{-1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
830
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
831 \begin{figure}[ht]
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
832 %\vspace*{-2mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
833 \centerline{\resizebox{.99\textwidth}{!}{\includegraphics{images/error_rates_charts.pdf}}}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
834 %\vspace*{-3mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
835 \caption{SDAx are the {\bf deep} models. Error bars indicate a 95\% confidence interval. 0 indicates that the model was trained
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
836 on NIST, 1 on NISTP, and 2 on P07. Left: overall results
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
837 of all models, on NIST and NISTP test sets.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
838 Right: error rates on NIST test digits only, along with the previous results from
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
839 literature~\citep{Granger+al-2007,Cortes+al-2000,Oliveira+al-2002-short,Milgram+al-2005}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
840 respectively based on ART, nearest neighbors, MLPs, and SVMs.}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
841 \label{fig:error-rates-charts}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
842 %\vspace*{-2mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
843 \end{figure}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
844
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
845
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
846 \begin{figure}[ht]
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
847 %\vspace*{-3mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
848 \centerline{\resizebox{.99\textwidth}{!}{\includegraphics{images/improvements_charts.pdf}}}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
849 %\vspace*{-3mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
850 \caption{Relative improvement in error rate due to self-taught learning.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
851 Left: Improvement (or loss, when negative)
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
852 induced by out-of-distribution examples (perturbed data).
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
853 Right: Improvement (or loss, when negative) induced by multi-task
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
854 learning (training on all classes and testing only on either digits,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
855 upper case, or lower-case). The deep learner (SDA) benefits more from
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
856 both self-taught learning scenarios, compared to the shallow MLP.}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
857 \label{fig:improvements-charts}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
858 %\vspace*{-2mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
859 \end{figure}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
860
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
861 \section{Experimental Results}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
862 %\vspace*{-2mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
863
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
864 %%\vspace*{-1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
865 %\subsection{SDA vs MLP vs Humans}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
866 %%\vspace*{-1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
867 The models are either trained on NIST (MLP0 and SDA0),
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
868 NISTP (MLP1 and SDA1), or P07 (MLP2 and SDA2), and tested
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
869 on either NIST, NISTP or P07, either on the 62-class task
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
870 or on the 10-digits task. Training (including about half
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
871 for unsupervised pre-training, for DAs) on the larger
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
872 datasets takes around one day on a GPU-285.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
873 Figure~\ref{fig:error-rates-charts} summarizes the results obtained,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
874 comparing humans, the three MLPs (MLP0, MLP1, MLP2) and the three SDAs (SDA0, SDA1,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
875 SDA2), along with the previous results on the digits NIST special database
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
876 19 test set from the literature, respectively based on ARTMAP neural
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
877 networks ~\citep{Granger+al-2007}, fast nearest-neighbor search
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
878 ~\citep{Cortes+al-2000}, MLPs ~\citep{Oliveira+al-2002-short}, and SVMs
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
879 ~\citep{Milgram+al-2005}. More detailed and complete numerical results
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
880 (figures and tables, including standard errors on the error rates) can be
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
881 found in Appendix.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
882 The deep learner not only outperformed the shallow ones and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
883 previously published performance (in a statistically and qualitatively
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
884 significant way) but when trained with perturbed data
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
885 reaches human performance on both the 62-class task
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
886 and the 10-class (digits) task.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
887 17\% error (SDA1) or 18\% error (humans) may seem large but a large
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
888 majority of the errors from humans and from SDA1 are from out-of-context
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
889 confusions (e.g. a vertical bar can be a ``1'', an ``l'' or an ``L'', and a
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
890 ``c'' and a ``C'' are often indistinguishible).
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
891
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
892 In addition, as shown in the left of
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
893 Figure~\ref{fig:improvements-charts}, the relative improvement in error
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
894 rate brought by self-taught learning is greater for the SDA, and these
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
895 differences with the MLP are statistically and qualitatively
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
896 significant.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
897 The left side of the figure shows the improvement to the clean
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
898 NIST test set error brought by the use of out-of-distribution examples
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
899 (i.e. the perturbed examples examples from NISTP or P07).
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
900 Relative percent change is measured by taking
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
901 $100 \% \times$ (original model's error / perturbed-data model's error - 1).
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
902 The right side of
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
903 Figure~\ref{fig:improvements-charts} shows the relative improvement
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
904 brought by the use of a multi-task setting, in which the same model is
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
905 trained for more classes than the target classes of interest (i.e. training
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
906 with all 62 classes when the target classes are respectively the digits,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
907 lower-case, or upper-case characters). Again, whereas the gain from the
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
908 multi-task setting is marginal or negative for the MLP, it is substantial
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
909 for the SDA. Note that to simplify these multi-task experiments, only the original
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
910 NIST dataset is used. For example, the MLP-digits bar shows the relative
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
911 percent improvement in MLP error rate on the NIST digits test set
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
912 is $100\% \times$ (single-task
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
913 model's error / multi-task model's error - 1). The single-task model is
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
914 trained with only 10 outputs (one per digit), seeing only digit examples,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
915 whereas the multi-task model is trained with 62 outputs, with all 62
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
916 character classes as examples. Hence the hidden units are shared across
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
917 all tasks. For the multi-task model, the digit error rate is measured by
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
918 comparing the correct digit class with the output class associated with the
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
919 maximum conditional probability among only the digit classes outputs. The
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
920 setting is similar for the other two target classes (lower case characters
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
921 and upper case characters).
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
922 %%\vspace*{-1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
923 %\subsection{Perturbed Training Data More Helpful for SDA}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
924 %%\vspace*{-1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
925
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
926 %%\vspace*{-1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
927 %\subsection{Multi-Task Learning Effects}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
928 %%\vspace*{-1mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
929
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
930 \iffalse
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
931 As previously seen, the SDA is better able to benefit from the
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
932 transformations applied to the data than the MLP. In this experiment we
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
933 define three tasks: recognizing digits (knowing that the input is a digit),
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
934 recognizing upper case characters (knowing that the input is one), and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
935 recognizing lower case characters (knowing that the input is one). We
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
936 consider the digit classification task as the target task and we want to
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
937 evaluate whether training with the other tasks can help or hurt, and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
938 whether the effect is different for MLPs versus SDAs. The goal is to find
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
939 out if deep learning can benefit more (or less) from multiple related tasks
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
940 (i.e. the multi-task setting) compared to a corresponding purely supervised
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
941 shallow learner.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
942
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
943 We use a single hidden layer MLP with 1000 hidden units, and a SDA
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
944 with 3 hidden layers (1000 hidden units per layer), pre-trained and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
945 fine-tuned on NIST.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
946
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
947 Our results show that the MLP benefits marginally from the multi-task setting
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
948 in the case of digits (5\% relative improvement) but is actually hurt in the case
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
949 of characters (respectively 3\% and 4\% worse for lower and upper class characters).
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
950 On the other hand the SDA benefited from the multi-task setting, with relative
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
951 error rate improvements of 27\%, 15\% and 13\% respectively for digits,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
952 lower and upper case characters, as shown in Table~\ref{tab:multi-task}.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
953 \fi
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
954
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
955
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
956 %\vspace*{-2mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
957 \section{Conclusions and Discussion}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
958 %\vspace*{-2mm}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
959
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
960 We have found that the self-taught learning framework is more beneficial
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
961 to a deep learner than to a traditional shallow and purely
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
962 supervised learner. More precisely,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
963 the answers are positive for all the questions asked in the introduction.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
964 %\begin{itemize}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
965
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
966 $\bullet$ %\item
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
967 {\bf Do the good results previously obtained with deep architectures on the
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
968 MNIST digits generalize to a much larger and richer (but similar)
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
969 dataset, the NIST special database 19, with 62 classes and around 800k examples}?
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
970 Yes, the SDA {\em systematically outperformed the MLP and all the previously
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
971 published results on this dataset} (the ones that we are aware of), {\em in fact reaching human-level
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
972 performance} at around 17\% error on the 62-class task and 1.4\% on the digits,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
973 and beating previously published results on the same data.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
974
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
975 $\bullet$ %\item
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
976 {\bf To what extent do self-taught learning scenarios help deep learners,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
977 and do they help them more than shallow supervised ones}?
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
978 We found that distorted training examples not only made the resulting
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
979 classifier better on similarly perturbed images but also on
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
980 the {\em original clean examples}, and more importantly and more novel,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
981 that deep architectures benefit more from such {\em out-of-distribution}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
982 examples. MLPs were helped by perturbed training examples when tested on perturbed input
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
983 images (65\% relative improvement on NISTP)
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
984 but only marginally helped (5\% relative improvement on all classes)
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
985 or even hurt (10\% relative loss on digits)
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
986 with respect to clean examples . On the other hand, the deep SDAs
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
987 were significantly boosted by these out-of-distribution examples.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
988 Similarly, whereas the improvement due to the multi-task setting was marginal or
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
989 negative for the MLP (from +5.6\% to -3.6\% relative change),
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
990 it was quite significant for the SDA (from +13\% to +27\% relative change),
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
991 which may be explained by the arguments below.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
992 %\end{itemize}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
993
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
994 In the original self-taught learning framework~\citep{RainaR2007}, the
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
995 out-of-sample examples were used as a source of unsupervised data, and
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
996 experiments showed its positive effects in a \emph{limited labeled data}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
997 scenario. However, many of the results by \citet{RainaR2007} (who used a
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
998 shallow, sparse coding approach) suggest that the {\em relative gain of self-taught
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
999 learning vs ordinary supervised learning} diminishes as the number of labeled examples increases.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1000 We note instead that, for deep
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1001 architectures, our experiments show that such a positive effect is accomplished
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1002 even in a scenario with a \emph{large number of labeled examples},
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1003 i.e., here, the relative gain of self-taught learning is probably preserved
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1004 in the asymptotic regime.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1005
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1006 {\bf Why would deep learners benefit more from the self-taught learning framework}?
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1007 The key idea is that the lower layers of the predictor compute a hierarchy
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1008 of features that can be shared across tasks or across variants of the
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1009 input distribution. A theoretical analysis of generalization improvements
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1010 due to sharing of intermediate features across tasks already points
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1011 towards that explanation~\cite{baxter95a}.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1012 Intermediate features that can be used in different
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1013 contexts can be estimated in a way that allows to share statistical
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1014 strength. Features extracted through many levels are more likely to
594
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
1015 be more abstract and more invariant to some of the factors of variation
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
1016 in the underlying distribution (as the experiments in~\citet{Goodfellow2009} suggest),
593
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1017 increasing the likelihood that they would be useful for a larger array
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1018 of tasks and input conditions.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1019 Therefore, we hypothesize that both depth and unsupervised
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1020 pre-training play a part in explaining the advantages observed here, and future
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1021 experiments could attempt at teasing apart these factors.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1022 And why would deep learners benefit from the self-taught learning
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1023 scenarios even when the number of labeled examples is very large?
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1024 We hypothesize that this is related to the hypotheses studied
594
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
1025 in~\citet{Erhan+al-2010}. In~\citet{Erhan+al-2010}
593
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1026 it was found that online learning on a huge dataset did not make the
594
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
1027 advantage of the deep learning bias vanish, and a similar phenomenon
593
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1028 may be happening here. We hypothesize that unsupervised pre-training
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1029 of a deep hierarchy with self-taught learning initializes the
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1030 model in the basin of attraction of supervised gradient descent
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1031 that corresponds to better generalization. Furthermore, such good
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1032 basins of attraction are not discovered by pure supervised learning
594
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
1033 (with or without self-taught settings) from random initialization, and more labeled examples
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
1034 does not allow the shallow or purely supervised models to discover
537f8b786655 submitted JMLR paper
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 593
diff changeset
1035 the kind of better basins associated
593
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1036 with deep learning and self-taught learning.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1037
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1038 A Flash demo of the recognizer (where both the MLP and the SDA can be compared)
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1039 can be executed on-line at {\tt http://deep.host22.com}.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1040
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1041
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1042 \section*{Appendix I: Detailed Numerical Results}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1043
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1044 These tables correspond to Figures 2 and 3 and contain the raw error rates for each model and dataset considered.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1045 They also contain additional data such as test errors on P07 and standard errors.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1046
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1047 \begin{table}[ht]
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1048 \caption{Overall comparison of error rates ($\pm$ std.err.) on 62 character classes (10 digits +
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1049 26 lower + 26 upper), except for last columns -- digits only, between deep architecture with pre-training
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1050 (SDA=Stacked Denoising Autoencoder) and ordinary shallow architecture
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1051 (MLP=Multi-Layer Perceptron). The models shown are all trained using perturbed data (NISTP or P07)
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1052 and using a validation set to select hyper-parameters and other training choices.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1053 \{SDA,MLP\}0 are trained on NIST,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1054 \{SDA,MLP\}1 are trained on NISTP, and \{SDA,MLP\}2 are trained on P07.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1055 The human error rate on digits is a lower bound because it does not count digits that were
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1056 recognized as letters. For comparison, the results found in the literature
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1057 on NIST digits classification using the same test set are included.}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1058 \label{tab:sda-vs-mlp-vs-humans}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1059 \begin{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1060 \begin{tabular}{|l|r|r|r|r|} \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1061 & NIST test & NISTP test & P07 test & NIST test digits \\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1062 Humans& 18.2\% $\pm$.1\% & 39.4\%$\pm$.1\% & 46.9\%$\pm$.1\% & $1.4\%$ \\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1063 SDA0 & 23.7\% $\pm$.14\% & 65.2\%$\pm$.34\% & 97.45\%$\pm$.06\% & 2.7\% $\pm$.14\%\\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1064 SDA1 & 17.1\% $\pm$.13\% & 29.7\%$\pm$.3\% & 29.7\%$\pm$.3\% & 1.4\% $\pm$.1\%\\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1065 SDA2 & 18.7\% $\pm$.13\% & 33.6\%$\pm$.3\% & 39.9\%$\pm$.17\% & 1.7\% $\pm$.1\%\\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1066 MLP0 & 24.2\% $\pm$.15\% & 68.8\%$\pm$.33\% & 78.70\%$\pm$.14\% & 3.45\% $\pm$.15\% \\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1067 MLP1 & 23.0\% $\pm$.15\% & 41.8\%$\pm$.35\% & 90.4\%$\pm$.1\% & 3.85\% $\pm$.16\% \\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1068 MLP2 & 24.3\% $\pm$.15\% & 46.0\%$\pm$.35\% & 54.7\%$\pm$.17\% & 4.85\% $\pm$.18\% \\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1069 \citep{Granger+al-2007} & & & & 4.95\% $\pm$.18\% \\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1070 \citep{Cortes+al-2000} & & & & 3.71\% $\pm$.16\% \\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1071 \citep{Oliveira+al-2002} & & & & 2.4\% $\pm$.13\% \\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1072 \citep{Milgram+al-2005} & & & & 2.1\% $\pm$.12\% \\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1073 \end{tabular}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1074 \end{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1075 \end{table}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1076
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1077 \begin{table}[ht]
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1078 \caption{Relative change in error rates due to the use of perturbed training data,
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1079 either using NISTP, for the MLP1/SDA1 models, or using P07, for the MLP2/SDA2 models.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1080 A positive value indicates that training on the perturbed data helped for the
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1081 given test set (the first 3 columns on the 62-class tasks and the last one is
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1082 on the clean 10-class digits). Clearly, the deep learning models did benefit more
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1083 from perturbed training data, even when testing on clean data, whereas the MLP
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1084 trained on perturbed data performed worse on the clean digits and about the same
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1085 on the clean characters. }
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1086 \label{tab:perturbation-effect}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1087 \begin{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1088 \begin{tabular}{|l|r|r|r|r|} \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1089 & NIST test & NISTP test & P07 test & NIST test digits \\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1090 SDA0/SDA1-1 & 38\% & 84\% & 228\% & 93\% \\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1091 SDA0/SDA2-1 & 27\% & 94\% & 144\% & 59\% \\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1092 MLP0/MLP1-1 & 5.2\% & 65\% & -13\% & -10\% \\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1093 MLP0/MLP2-1 & -0.4\% & 49\% & 44\% & -29\% \\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1094 \end{tabular}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1095 \end{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1096 \end{table}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1097
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1098 \begin{table}[ht]
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1099 \caption{Test error rates and relative change in error rates due to the use of
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1100 a multi-task setting, i.e., training on each task in isolation vs training
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1101 for all three tasks together, for MLPs vs SDAs. The SDA benefits much
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1102 more from the multi-task setting. All experiments on only on the
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1103 unperturbed NIST data, using validation error for model selection.
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1104 Relative improvement is 1 - single-task error / multi-task error.}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1105 \label{tab:multi-task}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1106 \begin{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1107 \begin{tabular}{|l|r|r|r|} \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1108 & single-task & multi-task & relative \\
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1109 & setting & setting & improvement \\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1110 MLP-digits & 3.77\% & 3.99\% & 5.6\% \\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1111 MLP-lower & 17.4\% & 16.8\% & -4.1\% \\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1112 MLP-upper & 7.84\% & 7.54\% & -3.6\% \\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1113 SDA-digits & 2.6\% & 3.56\% & 27\% \\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1114 SDA-lower & 12.3\% & 14.4\% & 15\% \\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1115 SDA-upper & 5.93\% & 6.78\% & 13\% \\ \hline
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1116 \end{tabular}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1117 \end{center}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1118 \end{table}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1119
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1120 %\afterpage{\clearpage}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1121 \clearpage
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1122 {
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1123 %\bibliographystyle{spbasic} % basic style, author-year citations
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1124 \bibliographystyle{plainnat}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1125 \bibliography{strings,strings-short,strings-shorter,ift6266_ml,specials,aigaion-shorter}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1126 %\bibliographystyle{unsrtnat}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1127 %\bibliographystyle{apalike}
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1128 }
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1129
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1130
18a7e7fdea4d jmlr_submission
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents:
diff changeset
1131 \end{document}