%0 Journal Article %1 polson2018posterior %A Polson, Nicholas %A Rockova, Veronika %D 2018 %K approximate bayesian readings %T Posterior Concentration for Sparse Deep Learning %U http://arxiv.org/abs/1803.09138 %X Spike-and-Slab Deep Learning (SS-DL) is a fully Bayesian alternative to Dropout for improving generalizability of deep ReLU networks. This new type of regularization enables provable recovery of smooth input-output maps with unknown levels of smoothness. Indeed, we show that the posterior distribution concentrates at the near minimax rate for $\alpha$-Hölder smooth maps, performing as well as if we knew the smoothness level $\alpha$ ahead of time. Our result sheds light on architecture design for deep neural networks, namely the choice of depth, width and sparsity level. These network attributes typically depend on unknown smoothness in order to be optimal. We obviate this constraint with the fully Bayes construction. As an aside, we show that SS-DL does not overfit in the sense that the posterior concentrates on smaller networks with fewer (up to the optimal number of) nodes and links. Our results provide new theoretical justifications for deep ReLU networks from a Bayesian point of view.

@article{polson2018posterior, abstract = {Spike-and-Slab Deep Learning (SS-DL) is a fully Bayesian alternative to Dropout for improving generalizability of deep ReLU networks. This new type of regularization enables provable recovery of smooth input-output maps with unknown levels of smoothness. Indeed, we show that the posterior distribution concentrates at the near minimax rate for $\alpha$-H\"older smooth maps, performing as well as if we knew the smoothness level $\alpha$ ahead of time. Our result sheds light on architecture design for deep neural networks, namely the choice of depth, width and sparsity level. These network attributes typically depend on unknown smoothness in order to be optimal. We obviate this constraint with the fully Bayes construction. As an aside, we show that SS-DL does not overfit in the sense that the posterior concentrates on smaller networks with fewer (up to the optimal number of) nodes and links. Our results provide new theoretical justifications for deep ReLU networks from a Bayesian point of view.}, added-at = {2020-01-23T01:31:58.000+0100}, author = {Polson, Nicholas and Rockova, Veronika}, biburl = {https://www.bibsonomy.org/bibtex/2234c82e52ccd8a80b0c23e19dfcb1962/kirk86}, description = {[1803.09138] Posterior Concentration for Sparse Deep Learning}, interhash = {05b3ee70b5c1a548022da11a0dabebe7}, intrahash = {234c82e52ccd8a80b0c23e19dfcb1962}, keywords = {approximate bayesian readings}, note = {cite arxiv:1803.09138}, timestamp = {2020-01-23T01:31:58.000+0100}, title = {Posterior Concentration for Sparse Deep Learning}, url = {http://arxiv.org/abs/1803.09138}, year = 2018 }