Mercurial > ift6266

annotate writeup/nips2010_submission_supplementary.tex @ 596:f6a3b28b002c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
nips2010_submission.pdf
author Yoshua Bengio <bengioy@iro.umontreal.ca>
date Thu, 14 Oct 2010 15:52:02 -0400
parents a7193b092b0a
children
rev   line source
488
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
1 \documentclass{article} % For LaTeX2e
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
2 \usepackage{nips10submit_e,times}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
3
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
4 \usepackage{amsthm,amsmath,amssymb,bbold,bbm}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
5 \usepackage{algorithm,algorithmic}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
6 \usepackage[utf8]{inputenc}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
7 \usepackage{graphicx,subfigure}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
8 \usepackage[numbers]{natbib}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
9
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
10 \title{Deep Self-Taught Learning for Handwritten Character Recognition\\
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
11 \emph{Supplementary Material}}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
12
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
13 \begin{document}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
14
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
15 \maketitle
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
16
492
bf481414ba9c adding a section title
dumitru@boldaslove.sbo.corp.google.com
parents: 488
diff changeset
17 \section*{Appendix I: Full results}
bf481414ba9c adding a section title
dumitru@boldaslove.sbo.corp.google.com
parents: 488
diff changeset
18
556
a7193b092b0a cleaner le supplementary material
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 525
diff changeset
19 These tables correspond to Figures 2 and 3 and contain the raw error rates for each model and dataset considered.
a7193b092b0a cleaner le supplementary material
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 525
diff changeset
20 They also contain additional data such as test errors on P07 and standard errors.
488
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
21
556
a7193b092b0a cleaner le supplementary material
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 525
diff changeset
22 \begin{table}[ht]
488
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
23 \caption{Overall comparison of error rates ($\pm$ std.err.) on 62 character classes (10 digits +
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
24 26 lower + 26 upper), except for last columns -- digits only, between deep architecture with pre-training
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
25 (SDA=Stacked Denoising Autoencoder) and ordinary shallow architecture
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
26 (MLP=Multi-Layer Perceptron). The models shown are all trained using perturbed data (NISTP or P07)
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
27 and using a validation set to select hyper-parameters and other training choices.
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
28 \{SDA,MLP\}0 are trained on NIST,
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
29 \{SDA,MLP\}1 are trained on NISTP, and \{SDA,MLP\}2 are trained on P07.
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
30 The human error rate on digits is a lower bound because it does not count digits that were
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
31 recognized as letters. For comparison, the results found in the literature
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
32 on NIST digits classification using the same test set are included.}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
33 \label{tab:sda-vs-mlp-vs-humans}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
34 \begin{center}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
35 \begin{tabular}{|l|r|r|r|r|} \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
36 & NIST test & NISTP test & P07 test & NIST test digits \\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
37 Humans& 18.2\% $\pm$.1\% & 39.4\%$\pm$.1\% & 46.9\%$\pm$.1\% & $1.4\%$ \\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
38 SDA0 & 23.7\% $\pm$.14\% & 65.2\%$\pm$.34\% & 97.45\%$\pm$.06\% & 2.7\% $\pm$.14\%\\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
39 SDA1 & 17.1\% $\pm$.13\% & 29.7\%$\pm$.3\% & 29.7\%$\pm$.3\% & 1.4\% $\pm$.1\%\\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
40 SDA2 & 18.7\% $\pm$.13\% & 33.6\%$\pm$.3\% & 39.9\%$\pm$.17\% & 1.7\% $\pm$.1\%\\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
41 MLP0 & 24.2\% $\pm$.15\% & 68.8\%$\pm$.33\% & 78.70\%$\pm$.14\% & 3.45\% $\pm$.15\% \\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
42 MLP1 & 23.0\% $\pm$.15\% & 41.8\%$\pm$.35\% & 90.4\%$\pm$.1\% & 3.85\% $\pm$.16\% \\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
43 MLP2 & 24.3\% $\pm$.15\% & 46.0\%$\pm$.35\% & 54.7\%$\pm$.17\% & 4.85\% $\pm$.18\% \\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
44 \citep{Granger+al-2007} & & & & 4.95\% $\pm$.18\% \\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
45 \citep{Cortes+al-2000} & & & & 3.71\% $\pm$.16\% \\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
46 \citep{Oliveira+al-2002} & & & & 2.4\% $\pm$.13\% \\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
47 \citep{Milgram+al-2005} & & & & 2.1\% $\pm$.12\% \\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
48 \end{tabular}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
49 \end{center}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
50 \end{table}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
51
556
a7193b092b0a cleaner le supplementary material
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 525
diff changeset
52 \begin{table}[ht]
488
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
53 \caption{Relative change in error rates due to the use of perturbed training data,
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
54 either using NISTP, for the MLP1/SDA1 models, or using P07, for the MLP2/SDA2 models.
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
55 A positive value indicates that training on the perturbed data helped for the
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
56 given test set (the first 3 columns on the 62-class tasks and the last one is
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
57 on the clean 10-class digits). Clearly, the deep learning models did benefit more
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
58 from perturbed training data, even when testing on clean data, whereas the MLP
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
59 trained on perturbed data performed worse on the clean digits and about the same
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
60 on the clean characters. }
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
61 \label{tab:perturbation-effect}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
62 \begin{center}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
63 \begin{tabular}{|l|r|r|r|r|} \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
64 & NIST test & NISTP test & P07 test & NIST test digits \\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
65 SDA0/SDA1-1 & 38\% & 84\% & 228\% & 93\% \\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
66 SDA0/SDA2-1 & 27\% & 94\% & 144\% & 59\% \\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
67 MLP0/MLP1-1 & 5.2\% & 65\% & -13\% & -10\% \\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
68 MLP0/MLP2-1 & -0.4\% & 49\% & 44\% & -29\% \\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
69 \end{tabular}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
70 \end{center}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
71 \end{table}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
72
556
a7193b092b0a cleaner le supplementary material
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 525
diff changeset
73 \begin{table}[ht]
488
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
74 \caption{Test error rates and relative change in error rates due to the use of
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
75 a multi-task setting, i.e., training on each task in isolation vs training
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
76 for all three tasks together, for MLPs vs SDAs. The SDA benefits much
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
77 more from the multi-task setting. All experiments on only on the
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
78 unperturbed NIST data, using validation error for model selection.
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
79 Relative improvement is 1 - single-task error / multi-task error.}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
80 \label{tab:multi-task}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
81 \begin{center}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
82 \begin{tabular}{|l|r|r|r|} \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
83 & single-task & multi-task & relative \\
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
84 & setting & setting & improvement \\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
85 MLP-digits & 3.77\% & 3.99\% & 5.6\% \\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
86 MLP-lower & 17.4\% & 16.8\% & -4.1\% \\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
87 MLP-upper & 7.84\% & 7.54\% & -3.6\% \\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
88 SDA-digits & 2.6\% & 3.56\% & 27\% \\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
89 SDA-lower & 12.3\% & 14.4\% & 15\% \\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
90 SDA-upper & 5.93\% & 6.78\% & 13\% \\ \hline
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
91 \end{tabular}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
92 \end{center}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
93 \end{table}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
94
556
a7193b092b0a cleaner le supplementary material
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 525
diff changeset
95
a7193b092b0a cleaner le supplementary material
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 525
diff changeset
96 \newpage
a7193b092b0a cleaner le supplementary material
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 525
diff changeset
97
a7193b092b0a cleaner le supplementary material
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 525
diff changeset
98 \vspace*{10mm}
a7193b092b0a cleaner le supplementary material
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 525
diff changeset
99 %{\small
525
8b7e054d22bd correct bib for the supplementary
Dumitru Erhan <dumitru.erhan@gmail.com>
parents: 492
diff changeset
100 \bibliography{strings,strings-short,strings-shorter,ift6266_ml,aigaion-shorter,specials}
488
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
101 %\bibliographystyle{plainnat}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
102 \bibliographystyle{unsrtnat}
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
103 %\bibliographystyle{apalike}
556
a7193b092b0a cleaner le supplementary material
Yoshua Bengio <bengioy@iro.umontreal.ca>
parents: 525
diff changeset
104 %}
488
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
105
6c9ff48e15cd Moved the tables into a separate supplementary material file
dumitru@dumitru.mtv.corp.google.com
parents:
diff changeset
106 \end{document}