We develop a variational framework to understand the properties of the
functions learned by neural networks fit to data. We propose and study of a
family of continuous-domain linear inverse problems with total variation-like
regularization in the Radon domain subject to data fitting constraints. We
derive a representer theorem showing that finite-width, single-hidden layer
neural networks are solutions to these inverse problems. We draw on many
techniques from variational spline theory and so we propose the notion of a
ridge spline, which corresponds to fitting data with a single-hidden layer
neural network. The representer theorem is reminiscent of the classical
Reproducing Kernel Hilbert space representer theorem, but the neural network
problem is set in a non-Hilbertian Banach space. Although the learning problems
are posed in the continuous-domain, similar to kernel methods, the problems can
be recast as finite-dimensional neural network training problems. These neural
network training problems have regularizers which are related to the well-known
weight decay and path-norm regularizers. Thus, our result gives insight into
functional characteristics of trained neural networks and also into the design
neural network regularizers. We also show that these regularizers promote
neural network solutions with desirable generalization properties.
Beschreibung
Neural Networks, Ridge Splines, and TV Regularization in the Radon Domain
%0 Generic
%1 parhi2020neural
%A Parhi, Rahul
%A Nowak, Robert D.
%D 2020
%K machine-learinng neural-network
%T Neural Networks, Ridge Splines, and TV Regularization in the Radon
Domain
%U http://arxiv.org/abs/2006.05626
%X We develop a variational framework to understand the properties of the
functions learned by neural networks fit to data. We propose and study of a
family of continuous-domain linear inverse problems with total variation-like
regularization in the Radon domain subject to data fitting constraints. We
derive a representer theorem showing that finite-width, single-hidden layer
neural networks are solutions to these inverse problems. We draw on many
techniques from variational spline theory and so we propose the notion of a
ridge spline, which corresponds to fitting data with a single-hidden layer
neural network. The representer theorem is reminiscent of the classical
Reproducing Kernel Hilbert space representer theorem, but the neural network
problem is set in a non-Hilbertian Banach space. Although the learning problems
are posed in the continuous-domain, similar to kernel methods, the problems can
be recast as finite-dimensional neural network training problems. These neural
network training problems have regularizers which are related to the well-known
weight decay and path-norm regularizers. Thus, our result gives insight into
functional characteristics of trained neural networks and also into the design
neural network regularizers. We also show that these regularizers promote
neural network solutions with desirable generalization properties.
@misc{parhi2020neural,
abstract = {We develop a variational framework to understand the properties of the
functions learned by neural networks fit to data. We propose and study of a
family of continuous-domain linear inverse problems with total variation-like
regularization in the Radon domain subject to data fitting constraints. We
derive a representer theorem showing that finite-width, single-hidden layer
neural networks are solutions to these inverse problems. We draw on many
techniques from variational spline theory and so we propose the notion of a
ridge spline, which corresponds to fitting data with a single-hidden layer
neural network. The representer theorem is reminiscent of the classical
Reproducing Kernel Hilbert space representer theorem, but the neural network
problem is set in a non-Hilbertian Banach space. Although the learning problems
are posed in the continuous-domain, similar to kernel methods, the problems can
be recast as finite-dimensional neural network training problems. These neural
network training problems have regularizers which are related to the well-known
weight decay and path-norm regularizers. Thus, our result gives insight into
functional characteristics of trained neural networks and also into the design
neural network regularizers. We also show that these regularizers promote
neural network solutions with desirable generalization properties.},
added-at = {2020-06-23T22:51:13.000+0200},
author = {Parhi, Rahul and Nowak, Robert D.},
biburl = {https://www.bibsonomy.org/bibtex/2aee65f44cc6d43fe44d1c39e55953445/stdiff},
description = {Neural Networks, Ridge Splines, and TV Regularization in the Radon Domain},
interhash = {0386ac05e85a271d80335f69d69b7e9f},
intrahash = {aee65f44cc6d43fe44d1c39e55953445},
keywords = {machine-learinng neural-network},
note = {cite arxiv:2006.05626},
timestamp = {2020-06-23T22:51:13.000+0200},
title = {Neural Networks, Ridge Splines, and TV Regularization in the Radon
Domain},
url = {http://arxiv.org/abs/2006.05626},
year = 2020
}