%0 Generic %1 etmann2019closer %A Etmann, Christian %D 2019 %K double-backpropagation from:adulny maths theory %T A Closer Look at Double Backpropagation %U http://arxiv.org/abs/1906.06637 %X In recent years, an increasing number of neural network models have included derivatives with respect to inputs in their loss functions, resulting in so-called double backpropagation for first-order optimization. However, so far no general description of the involved derivatives exists. Here, we cover a wide array of special cases in a very general Hilbert space framework, which allows us to provide optimized backpropagation rules for many real-world scenarios. This includes the reduction of calculations for Frobenius-norm-penalties on Jacobians by roughly a third for locally linear activation functions. Furthermore, we provide a description of the discontinuous loss surface of ReLU networks both in the inputs and the parameters and demonstrate why the discontinuities do not pose a big problem in reality.

@misc{etmann2019closer, abstract = {In recent years, an increasing number of neural network models have included derivatives with respect to inputs in their loss functions, resulting in so-called double backpropagation for first-order optimization. However, so far no general description of the involved derivatives exists. Here, we cover a wide array of special cases in a very general Hilbert space framework, which allows us to provide optimized backpropagation rules for many real-world scenarios. This includes the reduction of calculations for Frobenius-norm-penalties on Jacobians by roughly a third for locally linear activation functions. Furthermore, we provide a description of the discontinuous loss surface of ReLU networks both in the inputs and the parameters and demonstrate why the discontinuities do not pose a big problem in reality.}, added-at = {2021-02-26T16:08:31.000+0100}, author = {Etmann, Christian}, biburl = {https://www.bibsonomy.org/bibtex/266c6dd1ffb001a1514a76d66a8fb1a1c/adulny}, description = {A Closer Look at Double Backpropagation - 1906.06637.pdf}, interhash = {f5629b1d234299d3895c2b35c1e598b1}, intrahash = {66c6dd1ffb001a1514a76d66a8fb1a1c}, keywords = {double-backpropagation from:adulny maths theory}, note = {cite arxiv:1906.06637Comment: 16 pages, 7 figures}, timestamp = {2021-02-26T16:08:31.000+0100}, title = {A Closer Look at Double Backpropagation}, url = {http://arxiv.org/abs/1906.06637}, year = 2019 }