%0 Journal Article %1 eismann2022study %A Eismann, Simon %A Costa, Diego %A Liao, Lizhi %A Bezemer, Cor-Paul %A Shang, Weiyi %A van Hoorn, André %A Kounev, Samuel %D 2022 %J Journal of Systems and Software (JSS) %K descartes t_journalmagazine myown %T A case study on the stability of performance tests for serverless applications %U https://www.sciencedirect.com/science/article/abs/pii/S0164121222000498 %V 189 %X Context: While in serverless computing, application resource management and operational concerns are generally delegated to the cloud provider, ensuring that serverless applications meet their performance requirements is still a responsibility of the developers. Performance testing is a commonly used performance assessment practice; however, it traditionally requires visibility of the resource environment. Objective: In this study, we investigate whether performance tests of serverless applications are stable, that is, if their results are reproducible, and what implications the serverless paradigm has for performance tests. Method: We conduct a case study where we collect two datasets of performance test results: (a) repetitions of performance tests for varying memory size and load intensities and (b) three repetitions of the same performance test every day for ten months. Results: We find that performance tests of serverless applications are comparatively stable if conducted on the same day. However, we also observe short-term performance variations and frequent long-term performance changes. Conclusion: Performance tests for serverless applications can be stable; however, the serverless model impacts the planning, execution, and analysis of performance tests.

@article{eismann2022study, abstract = {Context: While in serverless computing, application resource management and operational concerns are generally delegated to the cloud provider, ensuring that serverless applications meet their performance requirements is still a responsibility of the developers. Performance testing is a commonly used performance assessment practice; however, it traditionally requires visibility of the resource environment. Objective: In this study, we investigate whether performance tests of serverless applications are stable, that is, if their results are reproducible, and what implications the serverless paradigm has for performance tests. Method: We conduct a case study where we collect two datasets of performance test results: (a) repetitions of performance tests for varying memory size and load intensities and (b) three repetitions of the same performance test every day for ten months. Results: We find that performance tests of serverless applications are comparatively stable if conducted on the same day. However, we also observe short-term performance variations and frequent long-term performance changes. Conclusion: Performance tests for serverless applications can be stable; however, the serverless model impacts the planning, execution, and analysis of performance tests.}, added-at = {2022-04-01T15:39:23.000+0200}, author = {Eismann, Simon and Costa, Diego and Liao, Lizhi and Bezemer, Cor-Paul and Shang, Weiyi and van Hoorn, André and Kounev, Samuel}, biburl = {https://www.bibsonomy.org/bibtex/252bd0251fc1b38a607d86babbbd95577/simon.eismann}, interhash = {f772349d38e4fe33ca3fda9014b86a4d}, intrahash = {52bd0251fc1b38a607d86babbbd95577}, journal = {Journal of Systems and Software (JSS)}, keywords = {descartes t_journalmagazine myown}, timestamp = {2022-04-01T15:50:15.000+0200}, title = {A case study on the stability of performance tests for serverless applications}, url = {https://www.sciencedirect.com/science/article/abs/pii/S0164121222000498}, volume = 189, year = 2022 }