We show how to use “complementary priors” to
eliminate the explaining- away effects that make
inference difficult in densely connected belief nets
that have many hidden layers. Using complementary
priors, we derive a fast, greedy algorithm that can
learn deep, directed belief networks one layer at a
time, provided the top two layers form an undirected
associa- tive memory. The fast, greedy algorithm is
used to initialize a slower learning procedure that
fine-tunes the weights using a contrastive ver- sion of
the wake-sleep algorithm. After fine-tuning, a network
with three hidden layers forms a very good generative
model of the joint distribu- tion of handwritten digit
images and their labels. This generative model gives
better digit classification than the best
discriminative learning al- gorithms. The
low-dimensional manifolds on which the digits lie are
modeled by long ravines in the free-energy landscape of
the top-level associative memory, and it is easy to
explore these ravines by using the directed connections
to display what the associative memory has in mind.
%0 Journal Article
%1 hinton-fast-deep-belief-2006
%A Hinton, Geoffrey
%A Osindero, Simon
%A Teh, Yee-Whye
%D 2006
%I MIT Press
%J Neural Computation
%K boltzmann_machine mnist neural_net
%N 7
%P 1527--1554
%T A fast learning algorithm for deep belief nets
%V 18
%X We show how to use “complementary priors” to
eliminate the explaining- away effects that make
inference difficult in densely connected belief nets
that have many hidden layers. Using complementary
priors, we derive a fast, greedy algorithm that can
learn deep, directed belief networks one layer at a
time, provided the top two layers form an undirected
associa- tive memory. The fast, greedy algorithm is
used to initialize a slower learning procedure that
fine-tunes the weights using a contrastive ver- sion of
the wake-sleep algorithm. After fine-tuning, a network
with three hidden layers forms a very good generative
model of the joint distribu- tion of handwritten digit
images and their labels. This generative model gives
better digit classification than the best
discriminative learning al- gorithms. The
low-dimensional manifolds on which the digits lie are
modeled by long ravines in the free-energy landscape of
the top-level associative memory, and it is easy to
explore these ravines by using the directed connections
to display what the associative memory has in mind.
@article{hinton-fast-deep-belief-2006,
abstract = {We show how to use “complementary priors” to
eliminate the explaining- away effects that make
inference difficult in densely connected belief nets
that have many hidden layers. Using complementary
priors, we derive a fast, greedy algorithm that can
learn deep, directed belief networks one layer at a
time, provided the top two layers form an undirected
associa- tive memory. The fast, greedy algorithm is
used to initialize a slower learning procedure that
fine-tunes the weights using a contrastive ver- sion of
the wake-sleep algorithm. After fine-tuning, a network
with three hidden layers forms a very good generative
model of the joint distribu- tion of handwritten digit
images and their labels. This generative model gives
better digit classification than the best
discriminative learning al- gorithms. The
low-dimensional manifolds on which the digits lie are
modeled by long ravines in the free-energy landscape of
the top-level associative memory, and it is easy to
explore these ravines by using the directed connections
to display what the associative memory has in mind.},
added-at = {2015-08-10T16:43:40.000+0200},
author = {Hinton, Geoffrey and Osindero, Simon and Teh, Yee-Whye},
biburl = {https://www.bibsonomy.org/bibtex/2c1ea7c04054b732b99ebe15a0d905ec8/mhwombat},
file = {:HintonETAL06.pdf:PDF},
groups = {public},
interhash = {e20c213844c6c160f4bcc59cfcdc845d},
intrahash = {c1ea7c04054b732b99ebe15a0d905ec8},
journal = {Neural Computation},
keywords = {boltzmann_machine mnist neural_net},
number = 7,
pages = {1527--1554},
publisher = {MIT Press},
timestamp = {2016-07-12T19:25:30.000+0200},
title = {A fast learning algorithm for deep belief nets},
username = {kappeld},
volume = 18,
year = 2006
}