The concept of recall has been one of the key elements of system measurement throughout the history of information retrieval, despite the fact that there are many unanswered questions as to its value. In this essay, we review those questions and explore several further issues that affect the usefulness of recall. In particular, we ask whether it is reasonable to expect to be able to measure recall; whether some researchers are conflating the concepts of recall and answer set cardinality; and whether it is plausible that a user would rely on a belief that a system is "high recall" to deeply explore an answer list. Combined with earlier observations about the unknowability of recall, and the lack of a plausible user model in which recall is a measure of satisfaction, we conclude that use of recall as a measure of the effectiveness of ranked querying is indefensible.
%0 Journal Article
%1 Zobel:2009:ARP:1670598.1670600
%A Zobel, Justin
%A Moffat, Alistair
%A Park, Laurence A.F.
%C New York, NY, USA
%D 2009
%I ACM
%J SIGIR Forum
%K evaluation informationretrieval recall
%P 3--8
%R 10.1145/1670598.1670600
%T Against recall: is it persistence, cardinality, density, coverage, or totality?
%U http://doi.acm.org/10.1145/1670598.1670600
%V 43
%X The concept of recall has been one of the key elements of system measurement throughout the history of information retrieval, despite the fact that there are many unanswered questions as to its value. In this essay, we review those questions and explore several further issues that affect the usefulness of recall. In particular, we ask whether it is reasonable to expect to be able to measure recall; whether some researchers are conflating the concepts of recall and answer set cardinality; and whether it is plausible that a user would rely on a belief that a system is "high recall" to deeply explore an answer list. Combined with earlier observations about the unknowability of recall, and the lack of a plausible user model in which recall is a measure of satisfaction, we conclude that use of recall as a measure of the effectiveness of ranked querying is indefensible.
@article{Zobel:2009:ARP:1670598.1670600,
abstract = {The concept of recall has been one of the key elements of system measurement throughout the history of information retrieval, despite the fact that there are many unanswered questions as to its value. In this essay, we review those questions and explore several further issues that affect the usefulness of recall. In particular, we ask whether it is reasonable to expect to be able to measure recall; whether some researchers are conflating the concepts of recall and answer set cardinality; and whether it is plausible that a user would rely on a belief that a system is "high recall" to deeply explore an answer list. Combined with earlier observations about the unknowability of recall, and the lack of a plausible user model in which recall is a measure of satisfaction, we conclude that use of recall as a measure of the effectiveness of ranked querying is indefensible.},
acmid = {1670600},
added-at = {2011-05-09T14:47:39.000+0200},
address = {New York, NY, USA},
author = {Zobel, Justin and Moffat, Alistair and Park, Laurence A.F.},
biburl = {https://www.bibsonomy.org/bibtex/212a90cdd960cebbd2bd2ddbe322d5b1b/cdevries},
description = {Against recall},
doi = {10.1145/1670598.1670600},
interhash = {4c909ac5b37dabb3eb4616be53cf8932},
intrahash = {12a90cdd960cebbd2bd2ddbe322d5b1b},
issn = {0163-5840},
issue = {1},
journal = {SIGIR Forum},
keywords = {evaluation informationretrieval recall},
month = {June},
numpages = {6},
pages = {3--8},
publisher = {ACM},
timestamp = {2011-05-09T14:47:40.000+0200},
title = {Against recall: is it persistence, cardinality, density, coverage, or totality?},
url = {http://doi.acm.org/10.1145/1670598.1670600},
volume = 43,
year = 2009
}