%0 Journal Article %1 turner2019blockswap %A Turner, Jack %A Crowley, Elliot J. %A Gray, Gavin %A Storkey, Amos %A O'Boyle, Michael %D 2019 %K compression deep-learning information sparsity %T BlockSwap: Fisher-guided Block Substitution for Network Compression %U http://arxiv.org/abs/1906.04113 %X The desire to run neural networks on low-capacity edge devices has led to the development of a wealth of compression techniques. Moonshine is a simple and powerful example of this: one takes a large pre-trained network and substitutes each of its convolutional blocks with a selected cheap alternative block, then distills the resultant network with the original. However, not all blocks are created equally; for a required parameter budget there may exist a potent combination of many different cheap blocks. In this work, we find these by developing BlockSwap: an algorithm for choosing networks with interleaved block types by passing a single minibatch of training data through randomly initialised networks and gauging their Fisher potential. We show that block-wise cheapening yields more accurate networks than single block-type networks across a spectrum of parameter budgets. Code is available at https://github.com/BayesWatch/pytorch-blockswap.

@article{turner2019blockswap, abstract = {The desire to run neural networks on low-capacity edge devices has led to the development of a wealth of compression techniques. Moonshine is a simple and powerful example of this: one takes a large pre-trained network and substitutes each of its convolutional blocks with a selected cheap alternative block, then distills the resultant network with the original. However, not all blocks are created equally; for a required parameter budget there may exist a potent combination of many different cheap blocks. In this work, we find these by developing BlockSwap: an algorithm for choosing networks with interleaved block types by passing a single minibatch of training data through randomly initialised networks and gauging their Fisher potential. We show that block-wise cheapening yields more accurate networks than single block-type networks across a spectrum of parameter budgets. Code is available at https://github.com/BayesWatch/pytorch-blockswap.}, added-at = {2019-06-11T13:46:16.000+0200}, author = {Turner, Jack and Crowley, Elliot J. and Gray, Gavin and Storkey, Amos and O'Boyle, Michael}, biburl = {https://www.bibsonomy.org/bibtex/297b36c52fca29674e5ce133b5c1fcb3a/kirk86}, description = {[1906.04113] BlockSwap: Fisher-guided Block Substitution for Network Compression}, interhash = {a8088f270e32c50eb2664defdbed2fc6}, intrahash = {97b36c52fca29674e5ce133b5c1fcb3a}, keywords = {compression deep-learning information sparsity}, note = {cite arxiv:1906.04113}, timestamp = {2019-06-11T13:46:16.000+0200}, title = {BlockSwap: Fisher-guided Block Substitution for Network Compression}, url = {http://arxiv.org/abs/1906.04113}, year = 2019 }