%0 Generic %1 JasKen17Three %A Jastrzębski, Stanisław %A Kenton, Zachary %A Arpit, Devansh %A Ballas, Nicolas %A Fischer, Asja %A Bengio, Yoshua %A Storkey, Amos %D 2017 %K deep_learning flat_minima large_batch optimization %T Three Factors Influencing Minima in SGD %U http://arxiv.org/abs/1711.04623 %X We investigate the dynamical and convergent properties of stochastic gradient descent (SGD) applied to Deep Neural Networks (DNNs). Characterizing the relation between learning rate, batch size and the properties of the final minima, such as width or generalization, remains an open question. In order to tackle this problem we investigate the previously proposed approximation of SGD by a stochastic differential equation (SDE). We theoretically argue that three factors - learning rate, batch size and gradient covariance - influence the minima found by SGD. In particular we find that the ratio of learning rate to batch size is a key determinant of SGD dynamics and of the width of the final minima, and that higher values of the ratio lead to wider minima and often better generalization. We confirm these findings experimentally. Further, we include experiments which show that learning rate schedules can be replaced with batch size schedules and that the ratio of learning rate to batch size is an important factor influencing the memorization process.

@misc{JasKen17Three, abstract = {We investigate the dynamical and convergent properties of stochastic gradient descent (SGD) applied to Deep Neural Networks (DNNs). Characterizing the relation between learning rate, batch size and the properties of the final minima, such as width or generalization, remains an open question. In order to tackle this problem we investigate the previously proposed approximation of SGD by a stochastic differential equation (SDE). We theoretically argue that three factors - learning rate, batch size and gradient covariance - influence the minima found by SGD. In particular we find that the ratio of learning rate to batch size is a key determinant of SGD dynamics and of the width of the final minima, and that higher values of the ratio lead to wider minima and often better generalization. We confirm these findings experimentally. Further, we include experiments which show that learning rate schedules can be replaced with batch size schedules and that the ratio of learning rate to batch size is an important factor influencing the memorization process.}, added-at = {2018-07-03T10:40:04.000+0200}, author = {Jastrzębski, Stanisław and Kenton, Zachary and Arpit, Devansh and Ballas, Nicolas and Fischer, Asja and Bengio, Yoshua and Storkey, Amos}, biburl = {https://www.bibsonomy.org/bibtex/25aa43c8f36914acdd7633de1d781af1d/loroch}, description = {Three Factors Influencing Minima in SGD}, interhash = {5e095d15b4bf6fb2e5e79cbd7bf2ef6d}, intrahash = {5aa43c8f36914acdd7633de1d781af1d}, keywords = {deep_learning flat_minima large_batch optimization}, note = {cite arxiv:1711.04623Comment: First two authors contributed equally. Short version accepted into ICLR workshop}, timestamp = {2018-07-03T10:40:04.000+0200}, title = {Three Factors Influencing Minima in SGD}, url = {http://arxiv.org/abs/1711.04623}, year = 2017 }