%0 Conference Paper %1 10.1007/978-3-030-92511-6_5 %A Herrnleben, Stefan %A Leidinger, Maximilian %A Lesch, Veronika %A Prantl, Thomas %A Grohmann, Johannes %A Krupitzer, Christian %A Kounev, Samuel %B Performance Evaluation Methodologies and Tools %C Cham %D 2021 %E Zhao, Qianchuan %E Xia, Li %I Springer International Publishing %K descartes networking t_full %P 72--92 %R https://doi.org/10.1007/978-3-030-92511-6_5 %T ComBench: A Benchmarking Framework for Publish/Subscribe Communication Protocols Under Network Limitations %U https://link.springer.com/chapter/10.1007%2F978-3-030-92511-6_5 %X Efficient and dependable communication is a highly relevant aspect for Internet of Things (IoT) systems in which tiny sensors, actuators, wearables, or other smart devices exchange messages. Various publish/subscribe protocols address the challenges of communication in IoT systems. The selection process of a suitable protocol should consider the communication behavior of the application, the protocol's performance, the resource requirements on the end device, and the network connection quality, as IoT environments often rely on wireless networks. Benchmarking is a common approach to evaluate and compare systems, considering the performance and aspects like dependability or security. In this paper, we present our IoT communication benchmarking framework ComBench for publish/subscribe protocols focusing on constrained networks with varying quality conditions. The benchmarking framework supports system designers, software engineers, and application developers to select and investigate the behavior of communication protocols. Our benchmarking framework contributes to (i) show the impact of fluctuating network quality on communication, (ii) compare multiple protocols, protocol features, and protocol implementations, and (iii) analyze scalability, robustness, and dependability of clients, networks, and brokers in different scenarios. Our case study demonstrates the applicability of our framework to support the decision for the best-suited protocol in various scenarios.

@inproceedings{10.1007/978-3-030-92511-6_5, abstract = {Efficient and dependable communication is a highly relevant aspect for Internet of Things (IoT) systems in which tiny sensors, actuators, wearables, or other smart devices exchange messages. Various publish/subscribe protocols address the challenges of communication in IoT systems. The selection process of a suitable protocol should consider the communication behavior of the application, the protocol's performance, the resource requirements on the end device, and the network connection quality, as IoT environments often rely on wireless networks. Benchmarking is a common approach to evaluate and compare systems, considering the performance and aspects like dependability or security. In this paper, we present our IoT communication benchmarking framework ComBench for publish/subscribe protocols focusing on constrained networks with varying quality conditions. The benchmarking framework supports system designers, software engineers, and application developers to select and investigate the behavior of communication protocols. Our benchmarking framework contributes to (i) show the impact of fluctuating network quality on communication, (ii) compare multiple protocols, protocol features, and protocol implementations, and (iii) analyze scalability, robustness, and dependability of clients, networks, and brokers in different scenarios. Our case study demonstrates the applicability of our framework to support the decision for the best-suited protocol in various scenarios.}, added-at = {2021-12-08T03:00:23.000+0100}, address = {Cham}, author = {Herrnleben, Stefan and Leidinger, Maximilian and Lesch, Veronika and Prantl, Thomas and Grohmann, Johannes and Krupitzer, Christian and Kounev, Samuel}, biburl = {https://www.bibsonomy.org/bibtex/2327742a94308a26709f32b31c1177a75/se-group}, booktitle = {Performance Evaluation Methodologies and Tools}, doi = {https://doi.org/10.1007/978-3-030-92511-6_5}, editor = {Zhao, Qianchuan and Xia, Li}, interhash = {1df303e562e79d316977511cc95b88fd}, intrahash = {327742a94308a26709f32b31c1177a75}, keywords = {descartes networking t_full}, pages = {72--92}, publisher = {Springer International Publishing}, timestamp = {2021-12-09T12:38:58.000+0100}, title = {ComBench: A Benchmarking Framework for Publish/Subscribe Communication Protocols Under Network Limitations}, url = {https://link.springer.com/chapter/10.1007%2F978-3-030-92511-6_5}, year = 2021 }