The natural gradient method has been used effectively in conjugate Gaussian
process models, but the non-conjugate case has been largely unexplored. We
examine how natural gradients can be used in non-conjugate stochastic settings,
together with hyperparameter learning. We conclude that the natural gradient
can significantly improve performance in terms of wall-clock time. For
ill-conditioned posteriors the benefit of the natural gradient method is
especially pronounced, and we demonstrate a practical setting where ordinary
gradients are unusable. We show how natural gradients can be computed
efficiently and automatically in any parameterization, using automatic
differentiation. Our code is integrated into the GPflow package.
Description
[1803.09151] Natural Gradients in Practice: Non-Conjugate Variational Inference in Gaussian Process Models
%0 Journal Article
%1 salimbeni2018natural
%A Salimbeni, Hugh
%A Eleftheriadis, Stefanos
%A Hensman, James
%D 2018
%K gaussian-proceses optimization readings variational
%T Natural Gradients in Practice: Non-Conjugate Variational Inference in
Gaussian Process Models
%U http://arxiv.org/abs/1803.09151
%X The natural gradient method has been used effectively in conjugate Gaussian
process models, but the non-conjugate case has been largely unexplored. We
examine how natural gradients can be used in non-conjugate stochastic settings,
together with hyperparameter learning. We conclude that the natural gradient
can significantly improve performance in terms of wall-clock time. For
ill-conditioned posteriors the benefit of the natural gradient method is
especially pronounced, and we demonstrate a practical setting where ordinary
gradients are unusable. We show how natural gradients can be computed
efficiently and automatically in any parameterization, using automatic
differentiation. Our code is integrated into the GPflow package.
@article{salimbeni2018natural,
abstract = {The natural gradient method has been used effectively in conjugate Gaussian
process models, but the non-conjugate case has been largely unexplored. We
examine how natural gradients can be used in non-conjugate stochastic settings,
together with hyperparameter learning. We conclude that the natural gradient
can significantly improve performance in terms of wall-clock time. For
ill-conditioned posteriors the benefit of the natural gradient method is
especially pronounced, and we demonstrate a practical setting where ordinary
gradients are unusable. We show how natural gradients can be computed
efficiently and automatically in any parameterization, using automatic
differentiation. Our code is integrated into the GPflow package.},
added-at = {2020-02-24T04:36:22.000+0100},
author = {Salimbeni, Hugh and Eleftheriadis, Stefanos and Hensman, James},
biburl = {https://www.bibsonomy.org/bibtex/290161205624adc0b1c93fb264fa39b23/kirk86},
description = {[1803.09151] Natural Gradients in Practice: Non-Conjugate Variational Inference in Gaussian Process Models},
interhash = {3972b85e1f456cd4aa3f482c4f95ac4e},
intrahash = {90161205624adc0b1c93fb264fa39b23},
keywords = {gaussian-proceses optimization readings variational},
note = {cite arxiv:1803.09151},
timestamp = {2020-02-24T04:36:22.000+0100},
title = {Natural Gradients in Practice: Non-Conjugate Variational Inference in
Gaussian Process Models},
url = {http://arxiv.org/abs/1803.09151},
year = 2018
}