%0 Conference Paper %1 schroder2011setting %A Schröder, Gunnar %A Thiele, Maik %A Lehner, Wolfgang %B Proceedings of the Workshop on User-Centric Evaluation of Recommender Systems and Their Interfaces %C Chicago, USA %D 2011 %E Felfernig, Alexander %E Chen, Li %E Mandl, Monika %E Willemsen, Martijn %E Bollen, Dirk %E Ekstrand, Michael %I CEUR-WS %K evaluation goal metric metrics recommender %P 78--85 %T Setting Goals and Choosing Metrics for Recommender System Evaluations %U http://ceur-ws.org/Vol-811/paper12.pdf %X Recommender systems have become an important personalization technique on the web and are widely used especially in e-commerce applications. However, operators of web shops and other platforms are challenged by the large variety of available algorithms and the multitude of their possible parameterizations. Since the quality of the recommendations that are given can have a significant business impact, the selection of a recommender system should be made based on well-founded evaluation data. The literature on recommender system evaluation offers a large variety of evaluation metrics but provides little guidance on how to choose among them. This paper focuses on the often neglected aspect of clearly defining the goal of an evaluation and how this goal relates to the selection of an appropriate metric. We discuss several well-known accuracy metrics and analyze how these reflect different evaluation goals. Furthermore we present some less well-known metrics as well as a variation of the area under the curve measure that are particularly suitable for the evaluation of recommender systems in e-commerce applications.

@inproceedings{schroder2011setting, abstract = {Recommender systems have become an important personalization technique on the web and are widely used especially in e-commerce applications. However, operators of web shops and other platforms are challenged by the large variety of available algorithms and the multitude of their possible parameterizations. Since the quality of the recommendations that are given can have a significant business impact, the selection of a recommender system should be made based on well-founded evaluation data. The literature on recommender system evaluation offers a large variety of evaluation metrics but provides little guidance on how to choose among them. This paper focuses on the often neglected aspect of clearly defining the goal of an evaluation and how this goal relates to the selection of an appropriate metric. We discuss several well-known accuracy metrics and analyze how these reflect different evaluation goals. Furthermore we present some less well-known metrics as well as a variation of the area under the curve measure that are particularly suitable for the evaluation of recommender systems in e-commerce applications.}, added-at = {2021-05-28T09:55:53.000+0200}, address = {Chicago, USA}, author = {Schröder, Gunnar and Thiele, Maik and Lehner, Wolfgang}, biburl = {https://www.bibsonomy.org/bibtex/24c82b562f40ac822fbcd02affca14435/jaeschke}, booktitle = {Proceedings of the Workshop on User-Centric Evaluation of Recommender Systems and Their Interfaces}, editor = {Felfernig, Alexander and Chen, Li and Mandl, Monika and Willemsen, Martijn and Bollen, Dirk and Ekstrand, Michael}, interhash = {41a34f1bbe649b182e5a174f1fe62dcf}, intrahash = {4c82b562f40ac822fbcd02affca14435}, keywords = {evaluation goal metric metrics recommender}, month = oct, pages = {78--85}, publisher = {CEUR-WS}, timestamp = {2021-05-28T09:56:26.000+0200}, title = {Setting Goals and Choosing Metrics for Recommender System Evaluations}, url = {http://ceur-ws.org/Vol-811/paper12.pdf}, year = 2011 }