Training Deep Neural Networks (DNNs) that are robust to norm bounded
adversarial attacks remains an elusive problem. While verification based
methods are generally too expensive to robustly train large networks, it was
demonstrated in Gowal et. al. that bounded input intervals can be inexpensively
propagated per layer through large networks. This interval bound propagation
(IBP) approach led to high robustness and was the first to be employed on large
networks. However, due to the very loose nature of the IBP bounds, particularly
for large networks, the required training procedure is complex and involved. In
this paper, we closely examine the bounds of a block of layers composed of an
affine layer followed by a ReLU nonlinearity followed by another affine layer.
In doing so, we propose probabilistic bounds, true bounds in expectation, that
are provably tighter than IBP bounds in expectation. We then extend this result
to deeper networks through blockwise propagation and show that we can achieve
orders of magnitudes tighter bounds compared to IBP. With such tight bounds, we
demonstrate that a simple standard training procedure can achieve the best
robustness-accuracy trade-off across several architectures on both MNIST and
CIFAR10.
Beschreibung
[1905.12418] Probabilistically True and Tight Bounds for Robust Deep Neural Network Training
%0 Journal Article
%1 alsubaihi2019probabilistically
%A Alsubaihi, Salman
%A Bibi, Adel
%A Alfadly, Modar
%A Ghanem, Bernard
%D 2019
%K bounds generalization theory
%T Probabilistically True and Tight Bounds for Robust Deep Neural Network
Training
%U http://arxiv.org/abs/1905.12418
%X Training Deep Neural Networks (DNNs) that are robust to norm bounded
adversarial attacks remains an elusive problem. While verification based
methods are generally too expensive to robustly train large networks, it was
demonstrated in Gowal et. al. that bounded input intervals can be inexpensively
propagated per layer through large networks. This interval bound propagation
(IBP) approach led to high robustness and was the first to be employed on large
networks. However, due to the very loose nature of the IBP bounds, particularly
for large networks, the required training procedure is complex and involved. In
this paper, we closely examine the bounds of a block of layers composed of an
affine layer followed by a ReLU nonlinearity followed by another affine layer.
In doing so, we propose probabilistic bounds, true bounds in expectation, that
are provably tighter than IBP bounds in expectation. We then extend this result
to deeper networks through blockwise propagation and show that we can achieve
orders of magnitudes tighter bounds compared to IBP. With such tight bounds, we
demonstrate that a simple standard training procedure can achieve the best
robustness-accuracy trade-off across several architectures on both MNIST and
CIFAR10.
@article{alsubaihi2019probabilistically,
abstract = {Training Deep Neural Networks (DNNs) that are robust to norm bounded
adversarial attacks remains an elusive problem. While verification based
methods are generally too expensive to robustly train large networks, it was
demonstrated in Gowal et. al. that bounded input intervals can be inexpensively
propagated per layer through large networks. This interval bound propagation
(IBP) approach led to high robustness and was the first to be employed on large
networks. However, due to the very loose nature of the IBP bounds, particularly
for large networks, the required training procedure is complex and involved. In
this paper, we closely examine the bounds of a block of layers composed of an
affine layer followed by a ReLU nonlinearity followed by another affine layer.
In doing so, we propose probabilistic bounds, true bounds in expectation, that
are provably tighter than IBP bounds in expectation. We then extend this result
to deeper networks through blockwise propagation and show that we can achieve
orders of magnitudes tighter bounds compared to IBP. With such tight bounds, we
demonstrate that a simple standard training procedure can achieve the best
robustness-accuracy trade-off across several architectures on both MNIST and
CIFAR10.},
added-at = {2019-07-16T19:24:29.000+0200},
author = {Alsubaihi, Salman and Bibi, Adel and Alfadly, Modar and Ghanem, Bernard},
biburl = {https://www.bibsonomy.org/bibtex/201bfc3dfe0e2f04d5b434567e2266f53/kirk86},
description = {[1905.12418] Probabilistically True and Tight Bounds for Robust Deep Neural Network Training},
interhash = {75dbd7de86ffdb78fb1303bd59f79d4b},
intrahash = {01bfc3dfe0e2f04d5b434567e2266f53},
keywords = {bounds generalization theory},
note = {cite arxiv:1905.12418},
timestamp = {2019-07-16T19:24:29.000+0200},
title = {Probabilistically True and Tight Bounds for Robust Deep Neural Network
Training},
url = {http://arxiv.org/abs/1905.12418},
year = 2019
}