Bayesian inference was once a gold standard for learning with neural
networks, providing accurate full predictive distributions and well calibrated
uncertainty. However, scaling Bayesian inference techniques to deep neural
networks is challenging due to the high dimensionality of the parameter space.
In this paper, we construct low-dimensional subspaces of parameter space, such
as the first principal components of the stochastic gradient descent (SGD)
trajectory, which contain diverse sets of high performing models. In these
subspaces, we are able to apply elliptical slice sampling and variational
inference, which struggle in the full parameter space. We show that Bayesian
model averaging over the induced posterior in these subspaces produces accurate
predictions and well calibrated predictive uncertainty for both regression and
image classification.
Beschreibung
[1907.07504] Subspace Inference for Bayesian Deep Learning
%0 Journal Article
%1 izmailov2019subspace
%A Izmailov, Pavel
%A Maddox, Wesley J.
%A Kirichenko, Polina
%A Garipov, Timur
%A Vetrov, Dmitry
%A Wilson, Andrew Gordon
%D 2019
%K bayesian deep-learning sampling
%T Subspace Inference for Bayesian Deep Learning
%U http://arxiv.org/abs/1907.07504
%X Bayesian inference was once a gold standard for learning with neural
networks, providing accurate full predictive distributions and well calibrated
uncertainty. However, scaling Bayesian inference techniques to deep neural
networks is challenging due to the high dimensionality of the parameter space.
In this paper, we construct low-dimensional subspaces of parameter space, such
as the first principal components of the stochastic gradient descent (SGD)
trajectory, which contain diverse sets of high performing models. In these
subspaces, we are able to apply elliptical slice sampling and variational
inference, which struggle in the full parameter space. We show that Bayesian
model averaging over the induced posterior in these subspaces produces accurate
predictions and well calibrated predictive uncertainty for both regression and
image classification.
@article{izmailov2019subspace,
abstract = {Bayesian inference was once a gold standard for learning with neural
networks, providing accurate full predictive distributions and well calibrated
uncertainty. However, scaling Bayesian inference techniques to deep neural
networks is challenging due to the high dimensionality of the parameter space.
In this paper, we construct low-dimensional subspaces of parameter space, such
as the first principal components of the stochastic gradient descent (SGD)
trajectory, which contain diverse sets of high performing models. In these
subspaces, we are able to apply elliptical slice sampling and variational
inference, which struggle in the full parameter space. We show that Bayesian
model averaging over the induced posterior in these subspaces produces accurate
predictions and well calibrated predictive uncertainty for both regression and
image classification.},
added-at = {2019-08-16T01:28:10.000+0200},
author = {Izmailov, Pavel and Maddox, Wesley J. and Kirichenko, Polina and Garipov, Timur and Vetrov, Dmitry and Wilson, Andrew Gordon},
biburl = {https://www.bibsonomy.org/bibtex/21b04a97f69bb56d441452aa0cf76716e/kirk86},
description = {[1907.07504] Subspace Inference for Bayesian Deep Learning},
interhash = {e34ae93a863d4e7079abf8a1c5f367e2},
intrahash = {1b04a97f69bb56d441452aa0cf76716e},
keywords = {bayesian deep-learning sampling},
note = {cite arxiv:1907.07504Comment: Published at UAI 2019},
timestamp = {2019-08-16T01:28:10.000+0200},
title = {Subspace Inference for Bayesian Deep Learning},
url = {http://arxiv.org/abs/1907.07504},
year = 2019
}