%0 Generic %1 TenWoo18High %A Teng, Michael %A Wood, Frank %D 2018 %K HPC SGD distributed workload_balance %T High Throughput Synchronous Distributed Stochastic Gradient Descent %U http://arxiv.org/abs/1803.04209 %X We introduce a new, high-throughput, synchronous, distributed, data-parallel, stochastic-gradient-descent learning algorithm. This algorithm uses amortized inference in a compute-cluster-specific, deep, generative, dynamical model to perform joint posterior predictive inference of the mini-batch gradient computation times of all worker-nodes in a parallel computing cluster. We show that a synchronous parameter server can, by utilizing such a model, choose an optimal cutoff time beyond which mini-batch gradient messages from slow workers are ignored that maximizes overall mini-batch gradient computations per second. In keeping with earlier findings we observe that, under realistic conditions, eagerly discarding the mini-batch gradient computations of stragglers not only increases throughput but actually increases the overall rate of convergence as a function of wall-clock time by virtue of eliminating idleness. The principal novel contribution and finding of this work goes beyond this by demonstrating that using the predicted run-times from a generative model of cluster worker performance to dynamically adjust the cutoff improves substantially over the static-cutoff prior art, leading to, among other things, significantly reduced deep neural net training times on large computer clusters.

@misc{TenWoo18High, abstract = {We introduce a new, high-throughput, synchronous, distributed, data-parallel, stochastic-gradient-descent learning algorithm. This algorithm uses amortized inference in a compute-cluster-specific, deep, generative, dynamical model to perform joint posterior predictive inference of the mini-batch gradient computation times of all worker-nodes in a parallel computing cluster. We show that a synchronous parameter server can, by utilizing such a model, choose an optimal cutoff time beyond which mini-batch gradient messages from slow workers are ignored that maximizes overall mini-batch gradient computations per second. In keeping with earlier findings we observe that, under realistic conditions, eagerly discarding the mini-batch gradient computations of stragglers not only increases throughput but actually increases the overall rate of convergence as a function of wall-clock time by virtue of eliminating idleness. The principal novel contribution and finding of this work goes beyond this by demonstrating that using the predicted run-times from a generative model of cluster worker performance to dynamically adjust the cutoff improves substantially over the static-cutoff prior art, leading to, among other things, significantly reduced deep neural net training times on large computer clusters.}, added-at = {2018-08-24T08:52:55.000+0200}, author = {Teng, Michael and Wood, Frank}, biburl = {https://www.bibsonomy.org/bibtex/297a75e8839f7bdc81c4e5cd5d61d1ac2/loroch}, interhash = {097f76b511fef2d6ca3a31268cd0678a}, intrahash = {97a75e8839f7bdc81c4e5cd5d61d1ac2}, keywords = {HPC SGD distributed workload_balance}, timestamp = {2018-08-24T08:52:55.000+0200}, title = {High Throughput Synchronous Distributed Stochastic Gradient Descent}, url = {http://arxiv.org/abs/1803.04209}, year = 2018 }