Multi-layer neural networks are among the most powerful models in machine
learning, yet the fundamental reasons for this success defy mathematical
understanding. Learning a neural network requires to optimize a non-convex
high-dimensional objective (risk function), a problem which is usually attacked
using stochastic gradient descent (SGD). Does SGD converge to a global optimum
of the risk or only to a local optimum? In the first case, does this happen
because local minima are absent, or because SGD somehow avoids them? In the
second, why do local minima reached by SGD have good generalization properties?
In this paper we consider a simple case, namely two-layers neural networks,
and prove that -in a suitable scaling limit- SGD dynamics is captured by a
certain non-linear partial differential equation (PDE) that we call
distributional dynamics (DD). We then consider several specific examples, and
show how DD can be used to prove convergence of SGD to networks with nearly
ideal generalization error. This description allows to 'average-out' some of
the complexities of the landscape of neural networks, and can be used to prove
a general convergence result for noisy SGD.
Description
[1804.06561] A Mean Field View of the Landscape of Two-Layers Neural Networks
%0 Journal Article
%1 mei2018field
%A Mei, Song
%A Montanari, Andrea
%A Nguyen, Phan-Minh
%D 2018
%K approximate deep-learning generalization optimization readings
%T A Mean Field View of the Landscape of Two-Layers Neural Networks
%U http://arxiv.org/abs/1804.06561
%X Multi-layer neural networks are among the most powerful models in machine
learning, yet the fundamental reasons for this success defy mathematical
understanding. Learning a neural network requires to optimize a non-convex
high-dimensional objective (risk function), a problem which is usually attacked
using stochastic gradient descent (SGD). Does SGD converge to a global optimum
of the risk or only to a local optimum? In the first case, does this happen
because local minima are absent, or because SGD somehow avoids them? In the
second, why do local minima reached by SGD have good generalization properties?
In this paper we consider a simple case, namely two-layers neural networks,
and prove that -in a suitable scaling limit- SGD dynamics is captured by a
certain non-linear partial differential equation (PDE) that we call
distributional dynamics (DD). We then consider several specific examples, and
show how DD can be used to prove convergence of SGD to networks with nearly
ideal generalization error. This description allows to 'average-out' some of
the complexities of the landscape of neural networks, and can be used to prove
a general convergence result for noisy SGD.
@article{mei2018field,
abstract = {Multi-layer neural networks are among the most powerful models in machine
learning, yet the fundamental reasons for this success defy mathematical
understanding. Learning a neural network requires to optimize a non-convex
high-dimensional objective (risk function), a problem which is usually attacked
using stochastic gradient descent (SGD). Does SGD converge to a global optimum
of the risk or only to a local optimum? In the first case, does this happen
because local minima are absent, or because SGD somehow avoids them? In the
second, why do local minima reached by SGD have good generalization properties?
In this paper we consider a simple case, namely two-layers neural networks,
and prove that -in a suitable scaling limit- SGD dynamics is captured by a
certain non-linear partial differential equation (PDE) that we call
distributional dynamics (DD). We then consider several specific examples, and
show how DD can be used to prove convergence of SGD to networks with nearly
ideal generalization error. This description allows to 'average-out' some of
the complexities of the landscape of neural networks, and can be used to prove
a general convergence result for noisy SGD.},
added-at = {2019-09-26T15:12:22.000+0200},
author = {Mei, Song and Montanari, Andrea and Nguyen, Phan-Minh},
biburl = {https://www.bibsonomy.org/bibtex/222c8b3e0060ca2c8727e3eced8d87215/kirk86},
description = {[1804.06561] A Mean Field View of the Landscape of Two-Layers Neural Networks},
interhash = {fd2606c8d69f164875b23674030819d2},
intrahash = {22c8b3e0060ca2c8727e3eced8d87215},
keywords = {approximate deep-learning generalization optimization readings},
note = {cite arxiv:1804.06561Comment: 103 pages},
timestamp = {2019-09-26T15:18:52.000+0200},
title = {A Mean Field View of the Landscape of Two-Layers Neural Networks},
url = {http://arxiv.org/abs/1804.06561},
year = 2018
}