%0 Conference Paper %1 beel2013research %A Beel, Joeran %A Langer, Stefan %A Genzmehr, Marcel %A Gipp, Bela %A Breitinger, Corinna %A Nürnberger, Andreas %B Proceedings of the Workshop on Reproducibility and Replication in Recommender Systems Evaluation %D 2013 %K evaluation paper recommender survey %T Research Paper Recommender System Evaluation: A Quantitative Literature Survey %U http://gipp.com/wp-content/papercite-data/pdf/beel2013i.pdf %X Over 80 approaches for academic literature recommendation exist today. The approaches were introduced and evaluated in more than 170 research articles, as well as patents, presentations and blogs. We reviewed these approaches and found most evaluations to contain major shortcomings. Of the approaches proposed, 21% were not evaluated. Among the evaluated approaches, 19% were not evaluated against a baseline. Of the user studies performed, 60% had 15 or fewer participants or did not report on the number of participants. Information on runtime and coverage was rarely provided. Due to these and several other shortcomings described in this paper, we conclude that it is currently not possible to determine which recommendation approaches for academic literature are the most promising. However, there is little value in the existence of more than 80 approaches if the best performing approaches are unknown.

@inproceedings{beel2013research, abstract = {Over 80 approaches for academic literature recommendation exist today. The approaches were introduced and evaluated in more than 170 research articles, as well as patents, presentations and blogs. We reviewed these approaches and found most evaluations to contain major shortcomings. Of the approaches proposed, 21% were not evaluated. Among the evaluated approaches, 19% were not evaluated against a baseline. Of the user studies performed, 60% had 15 or fewer participants or did not report on the number of participants. Information on runtime and coverage was rarely provided. Due to these and several other shortcomings described in this paper, we conclude that it is currently not possible to determine which recommendation approaches for academic literature are the most promising. However, there is little value in the existence of more than 80 approaches if the best performing approaches are unknown. }, added-at = {2014-01-14T09:43:23.000+0100}, author = {Beel, Joeran and Langer, Stefan and Genzmehr, Marcel and Gipp, Bela and Breitinger, Corinna and Nürnberger, Andreas}, biburl = {https://www.bibsonomy.org/bibtex/271fa5d4e11753f2b1f6b79cb6beefb72/jaeschke}, booktitle = {Proceedings of the Workshop on Reproducibility and Replication in Recommender Systems Evaluation}, interhash = {544758b1fd737c010643f529c4f48ae6}, intrahash = {71fa5d4e11753f2b1f6b79cb6beefb72}, keywords = {evaluation paper recommender survey}, timestamp = {2014-07-28T15:57:31.000+0200}, title = {Research Paper Recommender System Evaluation: A Quantitative Literature Survey}, url = {http://gipp.com/wp-content/papercite-data/pdf/beel2013i.pdf}, year = 2013 }