The learning rate warmup heuristic achieves remarkable success in stabilizing
training, accelerating convergence and improving generalization for adaptive
stochastic optimization algorithms like RMSprop and Adam. Here, we study its
mechanism in details. Pursuing the theory behind warmup, we identify a problem
of the adaptive learning rate (i.e., it has problematically large variance in
the early stage), suggest warmup works as a variance reduction technique, and
provide both empirical and theoretical evidence to verify our hypothesis. We
further propose RAdam, a new variant of Adam, by introducing a term to rectify
the variance of the adaptive learning rate. Extensive experimental results on
image classification, language modeling, and neural machine translation verify
our intuition and demonstrate the effectiveness and robustness of our proposed
method. All implementations are available at:
https://github.com/LiyuanLucasLiu/RAdam.
Description
[1908.03265] On the Variance of the Adaptive Learning Rate and Beyond
%0 Generic
%1 liu2019variance
%A Liu, Liyuan
%A Jiang, Haoming
%A He, Pengcheng
%A Chen, Weizhu
%A Liu, Xiaodong
%A Gao, Jianfeng
%A Han, Jiawei
%D 2019
%K cs.CL stat.ML
%T On the Variance of the Adaptive Learning Rate and Beyond
%U http://arxiv.org/abs/1908.03265
%X The learning rate warmup heuristic achieves remarkable success in stabilizing
training, accelerating convergence and improving generalization for adaptive
stochastic optimization algorithms like RMSprop and Adam. Here, we study its
mechanism in details. Pursuing the theory behind warmup, we identify a problem
of the adaptive learning rate (i.e., it has problematically large variance in
the early stage), suggest warmup works as a variance reduction technique, and
provide both empirical and theoretical evidence to verify our hypothesis. We
further propose RAdam, a new variant of Adam, by introducing a term to rectify
the variance of the adaptive learning rate. Extensive experimental results on
image classification, language modeling, and neural machine translation verify
our intuition and demonstrate the effectiveness and robustness of our proposed
method. All implementations are available at:
https://github.com/LiyuanLucasLiu/RAdam.
@misc{liu2019variance,
abstract = {The learning rate warmup heuristic achieves remarkable success in stabilizing
training, accelerating convergence and improving generalization for adaptive
stochastic optimization algorithms like RMSprop and Adam. Here, we study its
mechanism in details. Pursuing the theory behind warmup, we identify a problem
of the adaptive learning rate (i.e., it has problematically large variance in
the early stage), suggest warmup works as a variance reduction technique, and
provide both empirical and theoretical evidence to verify our hypothesis. We
further propose RAdam, a new variant of Adam, by introducing a term to rectify
the variance of the adaptive learning rate. Extensive experimental results on
image classification, language modeling, and neural machine translation verify
our intuition and demonstrate the effectiveness and robustness of our proposed
method. All implementations are available at:
https://github.com/LiyuanLucasLiu/RAdam.},
added-at = {2021-05-28T09:22:17.000+0200},
author = {Liu, Liyuan and Jiang, Haoming and He, Pengcheng and Chen, Weizhu and Liu, Xiaodong and Gao, Jianfeng and Han, Jiawei},
biburl = {https://www.bibsonomy.org/bibtex/225d828c4971803945ab20ed168895df3/aerover},
description = {[1908.03265] On the Variance of the Adaptive Learning Rate and Beyond},
interhash = {d88b772152e9101b6ebc0645c10985e5},
intrahash = {25d828c4971803945ab20ed168895df3},
keywords = {cs.CL stat.ML},
note = {cite arxiv:1908.03265Comment: ICLR 2020. Fix several typos in the previous version},
timestamp = {2021-05-28T09:22:17.000+0200},
title = {On the Variance of the Adaptive Learning Rate and Beyond},
url = {http://arxiv.org/abs/1908.03265},
year = 2019
}