We explore the phase diagram of approximation rates for deep neural networks.
The phase diagram describes theoretically optimal accuracy-complexity relations
and their qualitative properties. Our contribution is three-fold. First, we
generalize the existing result on the existence of deep discontinuous phase in
ReLU networks to functional classes of arbitrary positive smoothness, and
identify the boundary between the feasible and infeasible rates. Second, we
demonstrate that standard fully-connected architectures of a fixed width
independent of smoothness can adapt to smoothness and achieve almost optimal
rates. Finally, we discuss how the phase diagram can change in the case of
non-ReLU activation functions. In particular, we prove that using both sine and
ReLU activations theoretically leads to very fast, nearly exponential
approximation rates, thanks to the emerging capability of the network to
implement efficient lookup operations.
Description
[1906.09477] The phase diagram of approximation rates for deep neural networks
%0 Journal Article
%1 yarotsky2019phase
%A Yarotsky, Dmitry
%A Zhevnerchuk, Anton
%D 2019
%K deep-learning machine-learning
%T The phase diagram of approximation rates for deep neural networks
%U http://arxiv.org/abs/1906.09477
%X We explore the phase diagram of approximation rates for deep neural networks.
The phase diagram describes theoretically optimal accuracy-complexity relations
and their qualitative properties. Our contribution is three-fold. First, we
generalize the existing result on the existence of deep discontinuous phase in
ReLU networks to functional classes of arbitrary positive smoothness, and
identify the boundary between the feasible and infeasible rates. Second, we
demonstrate that standard fully-connected architectures of a fixed width
independent of smoothness can adapt to smoothness and achieve almost optimal
rates. Finally, we discuss how the phase diagram can change in the case of
non-ReLU activation functions. In particular, we prove that using both sine and
ReLU activations theoretically leads to very fast, nearly exponential
approximation rates, thanks to the emerging capability of the network to
implement efficient lookup operations.
@article{yarotsky2019phase,
abstract = {We explore the phase diagram of approximation rates for deep neural networks.
The phase diagram describes theoretically optimal accuracy-complexity relations
and their qualitative properties. Our contribution is three-fold. First, we
generalize the existing result on the existence of deep discontinuous phase in
ReLU networks to functional classes of arbitrary positive smoothness, and
identify the boundary between the feasible and infeasible rates. Second, we
demonstrate that standard fully-connected architectures of a fixed width
independent of smoothness can adapt to smoothness and achieve almost optimal
rates. Finally, we discuss how the phase diagram can change in the case of
non-ReLU activation functions. In particular, we prove that using both sine and
ReLU activations theoretically leads to very fast, nearly exponential
approximation rates, thanks to the emerging capability of the network to
implement efficient lookup operations.},
added-at = {2019-06-25T13:26:54.000+0200},
author = {Yarotsky, Dmitry and Zhevnerchuk, Anton},
biburl = {https://www.bibsonomy.org/bibtex/2470679d7baf701563167ce9bb0754f63/kirk86},
description = {[1906.09477] The phase diagram of approximation rates for deep neural networks},
interhash = {53687dde5ea0c31bb022963d33f4fbab},
intrahash = {470679d7baf701563167ce9bb0754f63},
keywords = {deep-learning machine-learning},
note = {cite arxiv:1906.09477Comment: 21 page},
timestamp = {2019-06-25T13:26:54.000+0200},
title = {The phase diagram of approximation rates for deep neural networks},
url = {http://arxiv.org/abs/1906.09477},
year = 2019
}