Automatic Differentiation Variational Inference (ADVI) is a useful tool for
efficiently learning probabilistic models in machine learning. Generally
approximate posteriors learned by ADVI are forced to be unimodal in order to
facilitate use of the reparameterization trick. In this paper, we show how
stratified sampling may be used to enable mixture distributions as the
approximate posterior, and derive a new lower bound on the evidence analogous
to the importance weighted autoencoder (IWAE). We show that this "SIWAE" is a
tighter bound than both IWAE and the traditional ELBO, both of which are
special instances of this bound. We verify empirically that the traditional
ELBO objective disfavors the presence of multimodal posterior distributions and
may therefore not be able to fully capture structure in the latent space. Our
experiments show that using the SIWAE objective allows the encoder to learn
more complex distributions which regularly contain multimodality, resulting in
higher accuracy and better calibration in the presence of incomplete, limited,
or corrupted data.
Описание
[2003.01687] Automatic Differentiation Variational Inference with Mixtures
%0 Journal Article
%1 morningstar2020automatic
%A Morningstar, Warren R.
%A Vikram, Sharad M.
%A Ham, Cusuh
%A Gallagher, Andrew
%A Dillon, Joshua V.
%D 2020
%K bayesian gaussian-proceses readings variational
%T Automatic Differentiation Variational Inference with Mixtures
%U http://arxiv.org/abs/2003.01687
%X Automatic Differentiation Variational Inference (ADVI) is a useful tool for
efficiently learning probabilistic models in machine learning. Generally
approximate posteriors learned by ADVI are forced to be unimodal in order to
facilitate use of the reparameterization trick. In this paper, we show how
stratified sampling may be used to enable mixture distributions as the
approximate posterior, and derive a new lower bound on the evidence analogous
to the importance weighted autoencoder (IWAE). We show that this "SIWAE" is a
tighter bound than both IWAE and the traditional ELBO, both of which are
special instances of this bound. We verify empirically that the traditional
ELBO objective disfavors the presence of multimodal posterior distributions and
may therefore not be able to fully capture structure in the latent space. Our
experiments show that using the SIWAE objective allows the encoder to learn
more complex distributions which regularly contain multimodality, resulting in
higher accuracy and better calibration in the presence of incomplete, limited,
or corrupted data.
@article{morningstar2020automatic,
abstract = {Automatic Differentiation Variational Inference (ADVI) is a useful tool for
efficiently learning probabilistic models in machine learning. Generally
approximate posteriors learned by ADVI are forced to be unimodal in order to
facilitate use of the reparameterization trick. In this paper, we show how
stratified sampling may be used to enable mixture distributions as the
approximate posterior, and derive a new lower bound on the evidence analogous
to the importance weighted autoencoder (IWAE). We show that this "SIWAE" is a
tighter bound than both IWAE and the traditional ELBO, both of which are
special instances of this bound. We verify empirically that the traditional
ELBO objective disfavors the presence of multimodal posterior distributions and
may therefore not be able to fully capture structure in the latent space. Our
experiments show that using the SIWAE objective allows the encoder to learn
more complex distributions which regularly contain multimodality, resulting in
higher accuracy and better calibration in the presence of incomplete, limited,
or corrupted data.},
added-at = {2020-03-09T18:28:30.000+0100},
author = {Morningstar, Warren R. and Vikram, Sharad M. and Ham, Cusuh and Gallagher, Andrew and Dillon, Joshua V.},
biburl = {https://www.bibsonomy.org/bibtex/2ca5e24537b22581a717e0f05e560163a/kirk86},
description = {[2003.01687] Automatic Differentiation Variational Inference with Mixtures},
interhash = {3f7683b7b35a279519082652444ff78c},
intrahash = {ca5e24537b22581a717e0f05e560163a},
keywords = {bayesian gaussian-proceses readings variational},
note = {cite arxiv:2003.01687Comment: Submitted to UAI 2020},
timestamp = {2020-03-09T18:28:30.000+0100},
title = {Automatic Differentiation Variational Inference with Mixtures},
url = {http://arxiv.org/abs/2003.01687},
year = 2020
}