As traditional neural network consumes a significant amount of computing
resources during back propagation, Sun2017mePropSB propose a simple yet
effective technique to alleviate this problem. In this technique, only a small
subset of the full gradients are computed to update the model parameters. In
this paper we extend this technique into the Convolutional Neural Network(CNN)
to reduce calculation in back propagation, and the surprising results verify
its validity in CNN: only 5\% of the gradients are passed back but the model
still achieves the same effect as the traditional CNN, or even better. We also
show that the top-$k$ selection of gradients leads to a sparse calculation in
back propagation, which may bring significant computational benefits for high
computational complexity of convolution operation in CNN.
Description
[1709.05804] Minimal Effort Back Propagation for Convolutional Neural Networks
%0 Generic
%1 wei2017minimal
%A Wei, Bingzhen
%A Sun, Xu
%A Ren, Xuancheng
%A Xu, Jingjing
%D 2017
%K gradient_descent
%T Minimal Effort Back Propagation for Convolutional Neural Networks
%U http://arxiv.org/abs/1709.05804
%X As traditional neural network consumes a significant amount of computing
resources during back propagation, Sun2017mePropSB propose a simple yet
effective technique to alleviate this problem. In this technique, only a small
subset of the full gradients are computed to update the model parameters. In
this paper we extend this technique into the Convolutional Neural Network(CNN)
to reduce calculation in back propagation, and the surprising results verify
its validity in CNN: only 5\% of the gradients are passed back but the model
still achieves the same effect as the traditional CNN, or even better. We also
show that the top-$k$ selection of gradients leads to a sparse calculation in
back propagation, which may bring significant computational benefits for high
computational complexity of convolution operation in CNN.
@misc{wei2017minimal,
abstract = {As traditional neural network consumes a significant amount of computing
resources during back propagation, \citet{Sun2017mePropSB} propose a simple yet
effective technique to alleviate this problem. In this technique, only a small
subset of the full gradients are computed to update the model parameters. In
this paper we extend this technique into the Convolutional Neural Network(CNN)
to reduce calculation in back propagation, and the surprising results verify
its validity in CNN: only 5\% of the gradients are passed back but the model
still achieves the same effect as the traditional CNN, or even better. We also
show that the top-$k$ selection of gradients leads to a sparse calculation in
back propagation, which may bring significant computational benefits for high
computational complexity of convolution operation in CNN.},
added-at = {2019-05-28T16:26:06.000+0200},
author = {Wei, Bingzhen and Sun, Xu and Ren, Xuancheng and Xu, Jingjing},
biburl = {https://www.bibsonomy.org/bibtex/2723ff553eca3b421efa783e07221e3db/straybird321},
description = {[1709.05804] Minimal Effort Back Propagation for Convolutional Neural Networks},
interhash = {1e956936777800377c073367e4a95c2f},
intrahash = {723ff553eca3b421efa783e07221e3db},
keywords = {gradient_descent},
note = {cite arxiv:1709.05804},
timestamp = {2019-05-28T16:26:06.000+0200},
title = {Minimal Effort Back Propagation for Convolutional Neural Networks},
url = {http://arxiv.org/abs/1709.05804},
year = 2017
}