%0 Journal Article %1 soudry2017implicit %A Soudry, Daniel %A Hoffer, Elad %A Nacson, Mor Shpigel %A Gunasekar, Suriya %A Srebro, Nathan %D 2017 %K deep-learning foundations machine-learning optimization readings regularisation stable theory %T The Implicit Bias of Gradient Descent on Separable Data %U http://arxiv.org/abs/1710.10345 %X We examine gradient descent on unregularized logistic regression problems, with homogeneous linear predictors on linearly separable datasets. We show the predictor converges to the direction of the max-margin (hard margin SVM) solution. The result also generalizes to other monotone decreasing loss functions with an infimum at infinity, to multi-class problems, and to training a weight layer in a deep network in a certain restricted setting. Furthermore, we show this convergence is very slow, and only logarithmic in the convergence of the loss itself. This can help explain the benefit of continuing to optimize the logistic or cross-entropy loss even after the training error is zero and the training loss is extremely small, and, as we show, even if the validation loss increases. Our methodology can also aid in understanding implicit regularization n more complex models and with other optimization methods.

@article{soudry2017implicit, abstract = {We examine gradient descent on unregularized logistic regression problems, with homogeneous linear predictors on linearly separable datasets. We show the predictor converges to the direction of the max-margin (hard margin SVM) solution. The result also generalizes to other monotone decreasing loss functions with an infimum at infinity, to multi-class problems, and to training a weight layer in a deep network in a certain restricted setting. Furthermore, we show this convergence is very slow, and only logarithmic in the convergence of the loss itself. This can help explain the benefit of continuing to optimize the logistic or cross-entropy loss even after the training error is zero and the training loss is extremely small, and, as we show, even if the validation loss increases. Our methodology can also aid in understanding implicit regularization n more complex models and with other optimization methods.}, added-at = {2019-06-10T11:46:19.000+0200}, author = {Soudry, Daniel and Hoffer, Elad and Nacson, Mor Shpigel and Gunasekar, Suriya and Srebro, Nathan}, biburl = {https://www.bibsonomy.org/bibtex/2d8de094706791f0222312a8d981d4739/kirk86}, description = {[1710.10345] The Implicit Bias of Gradient Descent on Separable Data}, interhash = {a9c10d42f692d492bc904cd0ed89a13f}, intrahash = {d8de094706791f0222312a8d981d4739}, keywords = {deep-learning foundations machine-learning optimization readings regularisation stable theory}, note = {cite arxiv:1710.10345Comment: Final JMLR version, with improved discussions over v3. Main improvements in journal version over conference version (v2 appeared in ICLR): We proved the measure zero case for main theorem (with implications for the rates), and the multi-class case}, timestamp = {2019-09-26T16:03:28.000+0200}, title = {The Implicit Bias of Gradient Descent on Separable Data}, url = {http://arxiv.org/abs/1710.10345}, year = 2017 }