C. Buckley, and E. Voorhees. Proceedings of the 27th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2004), page 25--32. Sheffield, United Kingdom, ACM, (July 2004)
DOI: 10.1145/1008992.1009000
Abstract
This paper examines whether the Cranfield evaluation methodology is robust to gross violations of the completeness assumption (i.e., the assumption that all relevant documents within a test collection have been identified and are present in the collection). We show that current evaluation measures are not robust to substantially incomplete relevance judgments. A new measure is introduced that is both highly correlated with existing measures when complete judgments are available and more robust to incomplete judgment sets. This finding suggests that substantially larger or dynamic test collections built using current pooling practices should be viable laboratory tools, despite the fact that the relevance information will be incomplete and imperfect.
%0 Conference Paper
%1 buckley2004
%A Buckley, Chris
%A Voorhees, Ellen M.
%B Proceedings of the 27th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2004)
%C Sheffield, United Kingdom
%D 2004
%E Sanderson, Mark
%E Järvelin, Kalervo
%E Allan, James
%E Bruza, Peter
%I ACM
%K evaluation learning link-mining ontology
%P 25--32
%R 10.1145/1008992.1009000
%T Retrieval evaluation with incomplete information
%U http://doi.acm.org/10.1145/1008992.1009000
%X This paper examines whether the Cranfield evaluation methodology is robust to gross violations of the completeness assumption (i.e., the assumption that all relevant documents within a test collection have been identified and are present in the collection). We show that current evaluation measures are not robust to substantially incomplete relevance judgments. A new measure is introduced that is both highly correlated with existing measures when complete judgments are available and more robust to incomplete judgment sets. This finding suggests that substantially larger or dynamic test collections built using current pooling practices should be viable laboratory tools, despite the fact that the relevance information will be incomplete and imperfect.
%@ 1-58113-881-4
@inproceedings{buckley2004,
abstract = {This paper examines whether the Cranfield evaluation methodology is robust to gross violations of the completeness assumption (i.e., the assumption that all relevant documents within a test collection have been identified and are present in the collection). We show that current evaluation measures are not robust to substantially incomplete relevance judgments. A new measure is introduced that is both highly correlated with existing measures when complete judgments are available and more robust to incomplete judgment sets. This finding suggests that substantially larger or dynamic test collections built using current pooling practices should be viable laboratory tools, despite the fact that the relevance information will be incomplete and imperfect.},
acmid = {1009000},
added-at = {2011-10-03T01:07:55.000+0200},
address = {Sheffield, United Kingdom},
author = {Buckley, Chris and Voorhees, Ellen M.},
biburl = {https://www.bibsonomy.org/bibtex/29eca7f1402fe95d38c80a57f77b2f876/utahell},
booktitle = {Proceedings of the 27th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2004)},
description = {Retrieval evaluation with incomplete information},
doi = {10.1145/1008992.1009000},
editor = {Sanderson, Mark and J\"arvelin, Kalervo and Allan, James and Bruza, Peter},
interhash = {b68baf207b6b215c1f6c727ce12a79f3},
intrahash = {9eca7f1402fe95d38c80a57f77b2f876},
isbn = {1-58113-881-4},
keywords = {evaluation learning link-mining ontology},
location = {Sheffield, United Kingdom},
month = {July},
numpages = {8},
pages = {25--32},
publisher = {ACM},
timestamp = {2011-12-16T16:38:33.000+0100},
title = {Retrieval evaluation with incomplete information},
url = {http://doi.acm.org/10.1145/1008992.1009000},
year = 2004
}