In industrial machine learning pipelines, data often arrive in parts.
Particularly in the case of deep neural networks, it may be too expensive to
train the model from scratch each time, so one would rather use a previously
learned model and the new data to improve performance. However, deep neural
networks are prone to getting stuck in a suboptimal solution when trained on
only new data as compared to the full dataset. Our work focuses on a continuous
learning setup where the task is always the same and new parts of data arrive
sequentially. We apply a Bayesian approach to update the posterior
approximation with each new piece of data and find this method to outperform
the traditional approach in our experiments.
Description
Bayesian Incremental Learning for Deep Neural Networks
%0 Generic
%1 kochurov2018bayesian
%A Kochurov, Max
%A Garipov, Timur
%A Podoprikhin, Dmitry
%A Molchanov, Dmitry
%A Ashukha, Arsenii
%A Vetrov, Dmitry
%D 2018
%K 2018 arxiv deep-learning machine-learning neural-networks research
%T Bayesian Incremental Learning for Deep Neural Networks
%U http://arxiv.org/abs/1802.07329
%X In industrial machine learning pipelines, data often arrive in parts.
Particularly in the case of deep neural networks, it may be too expensive to
train the model from scratch each time, so one would rather use a previously
learned model and the new data to improve performance. However, deep neural
networks are prone to getting stuck in a suboptimal solution when trained on
only new data as compared to the full dataset. Our work focuses on a continuous
learning setup where the task is always the same and new parts of data arrive
sequentially. We apply a Bayesian approach to update the posterior
approximation with each new piece of data and find this method to outperform
the traditional approach in our experiments.
@misc{kochurov2018bayesian,
abstract = {In industrial machine learning pipelines, data often arrive in parts.
Particularly in the case of deep neural networks, it may be too expensive to
train the model from scratch each time, so one would rather use a previously
learned model and the new data to improve performance. However, deep neural
networks are prone to getting stuck in a suboptimal solution when trained on
only new data as compared to the full dataset. Our work focuses on a continuous
learning setup where the task is always the same and new parts of data arrive
sequentially. We apply a Bayesian approach to update the posterior
approximation with each new piece of data and find this method to outperform
the traditional approach in our experiments.},
added-at = {2018-02-22T19:30:19.000+0100},
author = {Kochurov, Max and Garipov, Timur and Podoprikhin, Dmitry and Molchanov, Dmitry and Ashukha, Arsenii and Vetrov, Dmitry},
biburl = {https://www.bibsonomy.org/bibtex/26bddfedd8370845ac484fe16848e30d7/achakraborty},
description = {Bayesian Incremental Learning for Deep Neural Networks},
interhash = {5c8b7e1021d7072efa645b6a2a989754},
intrahash = {6bddfedd8370845ac484fe16848e30d7},
keywords = {2018 arxiv deep-learning machine-learning neural-networks research},
note = {cite arxiv:1802.07329Comment: ICLR Workshop Submission},
timestamp = {2018-02-22T19:30:19.000+0100},
title = {Bayesian Incremental Learning for Deep Neural Networks},
url = {http://arxiv.org/abs/1802.07329},
year = 2018
}