%0 Journal Article %1 wilson2003general %A Wilson, D.Randall %A Martinez, Tony R. %D 2003 %J Neural Networks %K large_batch offline_vs_online %N 10 %P 1429 - 1451 %R http://dx.doi.org/10.1016/S0893-6080(03)00138-2 %T The general inefficiency of batch training for gradient descent learning %U http://www.sciencedirect.com/science/article/pii/S0893608003001382 %V 16 %X Gradient descent training of neural networks can be done in either a batch or on-line manner. A widely held myth in the neural network community is that batch training is as fast or faster and/or more ‘correct’ than on-line training because it supposedly uses a better approximation of the true gradient for its weight updates. This paper explains why batch training is almost always slower than on-line training—often orders of magnitude slower—especially on large training sets. The main reason is due to the ability of on-line training to follow curves in the error surface throughout each epoch, which allows it to safely use a larger learning rate and thus converge with less iterations through the training data. Empirical results on a large (20,000-instance) speech recognition task and on 26 other learning tasks demonstrate that convergence can be reached significantly faster using on-line training than batch training, with no apparent difference in accuracy.

@article{wilson2003general, abstract = {Gradient descent training of neural networks can be done in either a batch or on-line manner. A widely held myth in the neural network community is that batch training is as fast or faster and/or more ‘correct’ than on-line training because it supposedly uses a better approximation of the true gradient for its weight updates. This paper explains why batch training is almost always slower than on-line training—often orders of magnitude slower—especially on large training sets. The main reason is due to the ability of on-line training to follow curves in the error surface throughout each epoch, which allows it to safely use a larger learning rate and thus converge with less iterations through the training data. Empirical results on a large (20,000-instance) speech recognition task and on 26 other learning tasks demonstrate that convergence can be reached significantly faster using on-line training than batch training, with no apparent difference in accuracy. }, added-at = {2018-06-28T17:27:22.000+0200}, author = {Wilson, D.Randall and Martinez, Tony R.}, biburl = {https://www.bibsonomy.org/bibtex/296f9edfa9e22221c26d382ff6e4feae8/pl_itwm}, description = {The general inefficiency of batch training for gradient descent learning}, doi = {http://dx.doi.org/10.1016/S0893-6080(03)00138-2}, interhash = {01b5e7f03c9066b5ffe0667ee96c8aba}, intrahash = {96f9edfa9e22221c26d382ff6e4feae8}, issn = {0893-6080}, journal = {Neural Networks }, keywords = {large_batch offline_vs_online}, number = 10, pages = {1429 - 1451}, timestamp = {2018-06-28T17:27:22.000+0200}, title = {The general inefficiency of batch training for gradient descent learning }, url = {http://www.sciencedirect.com/science/article/pii/S0893608003001382}, volume = 16, year = 2003 }