%0 Generic %1 ma2015adding %A Ma, Chenxin %A Smith, Virginia %A Jaggi, Martin %A Jordan, Michael I. %A Richtárik, Peter %A Takáč, Martin %D 2015 %K deep dl large-scale networks neural %T Adding vs. Averaging in Distributed Primal-Dual Optimization %U http://arxiv.org/abs/1502.03508 %X Distributed optimization methods for large-scale machine learning suffer from a communication bottleneck. It is difficult to reduce this bottleneck while still efficiently and accurately aggregating partial work from different machines. In this paper, we present a novel generalization of the recent communication-efficient primal-dual framework (CoCoA) for distributed optimization. Our framework, CoCoA+, allows for additive combination of local updates to the global parameters at each iteration, whereas previous schemes with convergence guarantees only allow conservative averaging. We give stronger (primal-dual) convergence rate guarantees for both CoCoA as well as our new variants, and generalize the theory for both methods to cover non-smooth convex loss functions. We provide an extensive experimental comparison that shows the markedly improved performance of CoCoA+ on several real-world distributed datasets, especially when scaling up the number of machines.

@misc{ma2015adding, abstract = {Distributed optimization methods for large-scale machine learning suffer from a communication bottleneck. It is difficult to reduce this bottleneck while still efficiently and accurately aggregating partial work from different machines. In this paper, we present a novel generalization of the recent communication-efficient primal-dual framework (CoCoA) for distributed optimization. Our framework, CoCoA+, allows for additive combination of local updates to the global parameters at each iteration, whereas previous schemes with convergence guarantees only allow conservative averaging. We give stronger (primal-dual) convergence rate guarantees for both CoCoA as well as our new variants, and generalize the theory for both methods to cover non-smooth convex loss functions. We provide an extensive experimental comparison that shows the markedly improved performance of CoCoA+ on several real-world distributed datasets, especially when scaling up the number of machines.}, added-at = {2019-06-04T19:07:30.000+0200}, author = {Ma, Chenxin and Smith, Virginia and Jaggi, Martin and Jordan, Michael I. and Richtárik, Peter and Takáč, Martin}, biburl = {https://www.bibsonomy.org/bibtex/24fefca1109af9516b8b342e892281a31/alrigazzi}, description = {Adding vs. Averaging in Distributed Primal-Dual Optimization}, interhash = {6b5b65b3706c9aeb046ca88ba60d7b4b}, intrahash = {4fefca1109af9516b8b342e892281a31}, keywords = {deep dl large-scale networks neural}, note = {cite arxiv:1502.03508Comment: ICML 2015: JMLR W&CP volume37, Proceedings of The 32nd International Conference on Machine Learning, pp. 1973-1982}, timestamp = {2019-06-04T19:07:30.000+0200}, title = {Adding vs. Averaging in Distributed Primal-Dual Optimization}, url = {http://arxiv.org/abs/1502.03508}, year = 2015 }