The work "Loss Landscape Sightseeing with Multi-Point Optimization"
(Skorokhodov and Burtsev, 2019) demonstrated that one can empirically find
arbitrary 2D binary patterns inside loss surfaces of popular neural networks.
In this paper we prove that: (i) this is a general property of deep universal
approximators; and (ii) this property holds for arbitrary smooth patterns, for
other dimensionalities, for every dataset, and any neural network that is
sufficiently deep and wide. Our analysis predicts not only the existence of all
such low-dimensional patterns, but also two other properties that were observed
empirically: (i) that it is easy to find these patterns; and (ii) that they
transfer to other data-sets (e.g. a test-set).
Description
[1912.07559v1] A Deep Neural Network's Loss Surface Contains Every Low-dimensional Pattern
%0 Generic
%1 czarnecki2019neural
%A Czarnecki, Wojciech Marian
%A Osindero, Simon
%A Pascanu, Razvan
%A Jaderberg, Max
%D 2019
%K 2019 deep-learning optimization surface
%T A Deep Neural Network's Loss Surface Contains Every Low-dimensional
Pattern
%U http://arxiv.org/abs/1912.07559
%X The work "Loss Landscape Sightseeing with Multi-Point Optimization"
(Skorokhodov and Burtsev, 2019) demonstrated that one can empirically find
arbitrary 2D binary patterns inside loss surfaces of popular neural networks.
In this paper we prove that: (i) this is a general property of deep universal
approximators; and (ii) this property holds for arbitrary smooth patterns, for
other dimensionalities, for every dataset, and any neural network that is
sufficiently deep and wide. Our analysis predicts not only the existence of all
such low-dimensional patterns, but also two other properties that were observed
empirically: (i) that it is easy to find these patterns; and (ii) that they
transfer to other data-sets (e.g. a test-set).
@misc{czarnecki2019neural,
abstract = {The work "Loss Landscape Sightseeing with Multi-Point Optimization"
(Skorokhodov and Burtsev, 2019) demonstrated that one can empirically find
arbitrary 2D binary patterns inside loss surfaces of popular neural networks.
In this paper we prove that: (i) this is a general property of deep universal
approximators; and (ii) this property holds for arbitrary smooth patterns, for
other dimensionalities, for every dataset, and any neural network that is
sufficiently deep and wide. Our analysis predicts not only the existence of all
such low-dimensional patterns, but also two other properties that were observed
empirically: (i) that it is easy to find these patterns; and (ii) that they
transfer to other data-sets (e.g. a test-set).},
added-at = {2019-12-18T06:29:51.000+0100},
author = {Czarnecki, Wojciech Marian and Osindero, Simon and Pascanu, Razvan and Jaderberg, Max},
biburl = {https://www.bibsonomy.org/bibtex/2d1e00ebb9a3303816d6f3d12c1644037/analyst},
description = {[1912.07559v1] A Deep Neural Network's Loss Surface Contains Every Low-dimensional Pattern},
interhash = {c0baf8318dcc3643c59e809c026c79a2},
intrahash = {d1e00ebb9a3303816d6f3d12c1644037},
keywords = {2019 deep-learning optimization surface},
note = {cite arxiv:1912.07559},
timestamp = {2019-12-18T06:29:51.000+0100},
title = {A Deep Neural Network's Loss Surface Contains Every Low-dimensional
Pattern},
url = {http://arxiv.org/abs/1912.07559},
year = 2019
}