%0 Journal Article %1 Issa_Mattos_2019 %A Mattos, David Issa %A Bosch, Jan %A Olsson, Helena Holmström %D 2019 %I Elsevier BV %J Information & Software Technology %K product.strategy requirements %P 68-81 %R 10.1016/j.infsof.2019.05.004 %T Multi-armed bandits in the wild: Pitfalls and strategies in online experiments. %U https://doi.org/10.1016%2Fj.infsof.2019.05.004 %V 113 %X Context Delivering faster value to customers with online experimentation is an emerging practice in industry. Multi-Armed Bandit (MAB) based experiments have the potential to deliver even faster results with a better allocation of resources over traditional A/B experiments. However, the incorrect use of MAB-based experiments can lead to incorrect conclusions that can potentially hurt the company's business. Objective The objective of this study is to understand the pitfalls and restrictions of using MABs in online experiments, as well as the strategies that are used to overcome them. Method This research uses a multiple case study method with eleven experts across five software companies and simulations to triangulate the data of some of the identified limitations. Results This study analyzes some limitations faced by companies using MAB and discusses strategies used to overcome them. The results are summarized into practitioners’ guidelines with criteria to select an appropriated experimental design. Conclusion MAB algorithms have the potential to deliver even faster results with a better allocation of resources over traditional A/B experiments. However, potential mistakes can occur and hinder the potential benefits of such approach. Together with the provided guidelines, we aim for this paper to be used as reference material for practitioners during the design of an online experiment.

@article{Issa_Mattos_2019, abstract = {Context Delivering faster value to customers with online experimentation is an emerging practice in industry. Multi-Armed Bandit (MAB) based experiments have the potential to deliver even faster results with a better allocation of resources over traditional A/B experiments. However, the incorrect use of MAB-based experiments can lead to incorrect conclusions that can potentially hurt the company's business. Objective The objective of this study is to understand the pitfalls and restrictions of using MABs in online experiments, as well as the strategies that are used to overcome them. Method This research uses a multiple case study method with eleven experts across five software companies and simulations to triangulate the data of some of the identified limitations. Results This study analyzes some limitations faced by companies using MAB and discusses strategies used to overcome them. The results are summarized into practitioners’ guidelines with criteria to select an appropriated experimental design. Conclusion MAB algorithms have the potential to deliver even faster results with a better allocation of resources over traditional A/B experiments. However, potential mistakes can occur and hinder the potential benefits of such approach. Together with the provided guidelines, we aim for this paper to be used as reference material for practitioners during the design of an online experiment.}, added-at = {2019-09-26T14:07:26.000+0200}, author = {Mattos, David Issa and Bosch, Jan and Olsson, Helena Holmström}, biburl = {https://www.bibsonomy.org/bibtex/232b122229f78118052ed84143cc1a53b/ispma}, doi = {10.1016/j.infsof.2019.05.004}, interhash = {efd67f37c1dded6b9f6d1bf11cd14871}, intrahash = {32b122229f78118052ed84143cc1a53b}, journal = {Information & Software Technology}, keywords = {product.strategy requirements}, month = sep, pages = {68-81}, publisher = {Elsevier {BV}}, timestamp = {2019-09-26T14:07:26.000+0200}, title = {Multi-armed bandits in the wild: Pitfalls and strategies in online experiments.}, url = {https://doi.org/10.1016%2Fj.infsof.2019.05.004}, volume = 113, year = 2019 }