A recent strategy to circumvent the exploding and vanishing gradient problem
in RNNs, and to allow the stable propagation of signals over long time scales,
is to constrain recurrent connectivity matrices to be orthogonal or unitary.
This ensures eigenvalues with unit norm and thus stable dynamics and training.
However this comes at the cost of reduced expressivity due to the limited
variety of orthogonal transformations. We propose a novel connectivity
structure based on the Schur decomposition and a splitting of the Schur form
into normal and non-normal parts. This allows to parametrize matrices with
unit-norm eigenspectra without orthogonality constraints on eigenbases. The
resulting architecture ensures access to a larger space of spectrally
constrained matrices, of which orthogonal matrices are a subset. This crucial
difference retains the stability advantages and training speed of orthogonal
RNNs while enhancing expressivity, especially on tasks that require
computations over ongoing input sequences.
Description
[1905.12080] Non-normal Recurrent Neural Network (nnRNN): learning long time dependencies while improving expressivity with transient dynamics
%0 Journal Article
%1 kerg2019nonnormal
%A Kerg, Giancarlo
%A Goyette, Kyle
%A Touzel, Maximilian Puelma
%A Gidel, Gauthier
%A Vorontsov, Eugene
%A Bengio, Yoshua
%A Lajoie, Guillaume
%D 2019
%K deep-learning memory optimization
%T Non-normal Recurrent Neural Network (nnRNN): learning long time
dependencies while improving expressivity with transient dynamics
%U http://arxiv.org/abs/1905.12080
%X A recent strategy to circumvent the exploding and vanishing gradient problem
in RNNs, and to allow the stable propagation of signals over long time scales,
is to constrain recurrent connectivity matrices to be orthogonal or unitary.
This ensures eigenvalues with unit norm and thus stable dynamics and training.
However this comes at the cost of reduced expressivity due to the limited
variety of orthogonal transformations. We propose a novel connectivity
structure based on the Schur decomposition and a splitting of the Schur form
into normal and non-normal parts. This allows to parametrize matrices with
unit-norm eigenspectra without orthogonality constraints on eigenbases. The
resulting architecture ensures access to a larger space of spectrally
constrained matrices, of which orthogonal matrices are a subset. This crucial
difference retains the stability advantages and training speed of orthogonal
RNNs while enhancing expressivity, especially on tasks that require
computations over ongoing input sequences.
@article{kerg2019nonnormal,
abstract = {A recent strategy to circumvent the exploding and vanishing gradient problem
in RNNs, and to allow the stable propagation of signals over long time scales,
is to constrain recurrent connectivity matrices to be orthogonal or unitary.
This ensures eigenvalues with unit norm and thus stable dynamics and training.
However this comes at the cost of reduced expressivity due to the limited
variety of orthogonal transformations. We propose a novel connectivity
structure based on the Schur decomposition and a splitting of the Schur form
into normal and non-normal parts. This allows to parametrize matrices with
unit-norm eigenspectra without orthogonality constraints on eigenbases. The
resulting architecture ensures access to a larger space of spectrally
constrained matrices, of which orthogonal matrices are a subset. This crucial
difference retains the stability advantages and training speed of orthogonal
RNNs while enhancing expressivity, especially on tasks that require
computations over ongoing input sequences.},
added-at = {2019-09-04T15:52:38.000+0200},
author = {Kerg, Giancarlo and Goyette, Kyle and Touzel, Maximilian Puelma and Gidel, Gauthier and Vorontsov, Eugene and Bengio, Yoshua and Lajoie, Guillaume},
biburl = {https://www.bibsonomy.org/bibtex/23947df17726de122f7fad03349cd9465/kirk86},
description = {[1905.12080] Non-normal Recurrent Neural Network (nnRNN): learning long time dependencies while improving expressivity with transient dynamics},
interhash = {bcad4084ff5e8c4e68c758cb7dcd2219},
intrahash = {3947df17726de122f7fad03349cd9465},
keywords = {deep-learning memory optimization},
note = {cite arxiv:1905.12080},
timestamp = {2019-09-04T15:52:38.000+0200},
title = {Non-normal Recurrent Neural Network (nnRNN): learning long time
dependencies while improving expressivity with transient dynamics},
url = {http://arxiv.org/abs/1905.12080},
year = 2019
}