One of the major concerns for neural network training is that the
non-convexity of the associated loss functions may cause bad landscape. The
recent success of neural networks suggests that their loss landscape is not too
bad, but what specific results do we know about the landscape? In this article,
we review recent findings and results on the global landscape of neural
networks. First, we point out that wide neural nets may have sub-optimal local
minima under certain assumptions. Second, we discuss a few rigorous results on
the geometric properties of wide networks such as "no bad basin", and some
modifications that eliminate sub-optimal local minima and/or decreasing paths
to infinity. Third, we discuss visualization and empirical explorations of the
landscape for practical neural nets. Finally, we briefly discuss some
convergence results and their relation to landscape results.
Description
[2007.01429] The Global Landscape of Neural Networks: An Overview
%0 Generic
%1 sun2020global
%A Sun, Ruoyu
%A Li, Dawei
%A Liang, Shiyu
%A Ding, Tian
%A Srikant, R
%D 2020
%K 2020 deep-learning survey
%T The Global Landscape of Neural Networks: An Overview
%U http://arxiv.org/abs/2007.01429
%X One of the major concerns for neural network training is that the
non-convexity of the associated loss functions may cause bad landscape. The
recent success of neural networks suggests that their loss landscape is not too
bad, but what specific results do we know about the landscape? In this article,
we review recent findings and results on the global landscape of neural
networks. First, we point out that wide neural nets may have sub-optimal local
minima under certain assumptions. Second, we discuss a few rigorous results on
the geometric properties of wide networks such as "no bad basin", and some
modifications that eliminate sub-optimal local minima and/or decreasing paths
to infinity. Third, we discuss visualization and empirical explorations of the
landscape for practical neural nets. Finally, we briefly discuss some
convergence results and their relation to landscape results.
@misc{sun2020global,
abstract = {One of the major concerns for neural network training is that the
non-convexity of the associated loss functions may cause bad landscape. The
recent success of neural networks suggests that their loss landscape is not too
bad, but what specific results do we know about the landscape? In this article,
we review recent findings and results on the global landscape of neural
networks. First, we point out that wide neural nets may have sub-optimal local
minima under certain assumptions. Second, we discuss a few rigorous results on
the geometric properties of wide networks such as "no bad basin", and some
modifications that eliminate sub-optimal local minima and/or decreasing paths
to infinity. Third, we discuss visualization and empirical explorations of the
landscape for practical neural nets. Finally, we briefly discuss some
convergence results and their relation to landscape results.},
added-at = {2020-07-07T12:19:47.000+0200},
author = {Sun, Ruoyu and Li, Dawei and Liang, Shiyu and Ding, Tian and Srikant, R},
biburl = {https://www.bibsonomy.org/bibtex/2b8181c46a959810633e0b6ee0559665c/analyst},
description = {[2007.01429] The Global Landscape of Neural Networks: An Overview},
interhash = {7e6cb93754795f59720966e4f8fe3f01},
intrahash = {b8181c46a959810633e0b6ee0559665c},
keywords = {2020 deep-learning survey},
note = {cite arxiv:2007.01429Comment: 16 pages. 8 figures},
timestamp = {2020-07-07T12:19:47.000+0200},
title = {The Global Landscape of Neural Networks: An Overview},
url = {http://arxiv.org/abs/2007.01429},
year = 2020
}