Since the recent study (Krichene and Rendle 2020) done by Krichene and Rendle
on the sampling-based top-k evaluation metric for recommendation, there has
been a lot of debates on the validity of using sampling to evaluate
recommendation algorithms. Though their work and the recent work (Li et
al.2020) have proposed some basic approaches for mapping the sampling-based
metrics to their global counterparts which rank the entire set of items, there
is still a lack of understanding and consensus on how sampling should be used
for recommendation evaluation. The proposed approaches either are rather
uninformative (linking sampling to metric evaluation) or can only work on
simple metrics, such as Recall/Precision (Krichene and Rendle 2020; Li et al.
2020). In this paper, we introduce a new research problem on learning the
empirical rank distribution, and a new approach based on the estimated rank
distribution, to estimate the top-k metrics. Since this question is closely
related to the underlying mechanism of sampling for recommendation, tackling it
can help better understand the power of sampling and can help resolve the
questions of if and how should we use sampling for evaluating recommendation.
We introduce two approaches based on MLE (MaximalLikelihood Estimation) and its
weighted variants, and ME(Maximal Entropy) principals to recover the empirical
rank distribution, and then utilize them for metrics estimation. The
experimental results show the advantages of using the new approaches for
evaluating recommendation algorithms based on top-k metrics.
Description
On Estimating Recommendation Evaluation Metrics under Sampling
%0 Generic
%1 jin2021estimating
%A Jin, Ruoming
%A Li, Dong
%A Mudrak, Benjamin
%A Gao, Jing
%A Liu, Zhi
%D 2021
%K deep_learning recommendation sampling
%T On Estimating Recommendation Evaluation Metrics under Sampling
%U http://arxiv.org/abs/2103.01474
%X Since the recent study (Krichene and Rendle 2020) done by Krichene and Rendle
on the sampling-based top-k evaluation metric for recommendation, there has
been a lot of debates on the validity of using sampling to evaluate
recommendation algorithms. Though their work and the recent work (Li et
al.2020) have proposed some basic approaches for mapping the sampling-based
metrics to their global counterparts which rank the entire set of items, there
is still a lack of understanding and consensus on how sampling should be used
for recommendation evaluation. The proposed approaches either are rather
uninformative (linking sampling to metric evaluation) or can only work on
simple metrics, such as Recall/Precision (Krichene and Rendle 2020; Li et al.
2020). In this paper, we introduce a new research problem on learning the
empirical rank distribution, and a new approach based on the estimated rank
distribution, to estimate the top-k metrics. Since this question is closely
related to the underlying mechanism of sampling for recommendation, tackling it
can help better understand the power of sampling and can help resolve the
questions of if and how should we use sampling for evaluating recommendation.
We introduce two approaches based on MLE (MaximalLikelihood Estimation) and its
weighted variants, and ME(Maximal Entropy) principals to recover the empirical
rank distribution, and then utilize them for metrics estimation. The
experimental results show the advantages of using the new approaches for
evaluating recommendation algorithms based on top-k metrics.
@misc{jin2021estimating,
abstract = {Since the recent study (Krichene and Rendle 2020) done by Krichene and Rendle
on the sampling-based top-k evaluation metric for recommendation, there has
been a lot of debates on the validity of using sampling to evaluate
recommendation algorithms. Though their work and the recent work (Li et
al.2020) have proposed some basic approaches for mapping the sampling-based
metrics to their global counterparts which rank the entire set of items, there
is still a lack of understanding and consensus on how sampling should be used
for recommendation evaluation. The proposed approaches either are rather
uninformative (linking sampling to metric evaluation) or can only work on
simple metrics, such as Recall/Precision (Krichene and Rendle 2020; Li et al.
2020). In this paper, we introduce a new research problem on learning the
empirical rank distribution, and a new approach based on the estimated rank
distribution, to estimate the top-k metrics. Since this question is closely
related to the underlying mechanism of sampling for recommendation, tackling it
can help better understand the power of sampling and can help resolve the
questions of if and how should we use sampling for evaluating recommendation.
We introduce two approaches based on MLE (MaximalLikelihood Estimation) and its
weighted variants, and ME(Maximal Entropy) principals to recover the empirical
rank distribution, and then utilize them for metrics estimation. The
experimental results show the advantages of using the new approaches for
evaluating recommendation algorithms based on top-k metrics.},
added-at = {2021-04-28T16:27:06.000+0200},
author = {Jin, Ruoming and Li, Dong and Mudrak, Benjamin and Gao, Jing and Liu, Zhi},
biburl = {https://www.bibsonomy.org/bibtex/2531f7d13e45d7904ab404ead09662074/dallmann},
description = {On Estimating Recommendation Evaluation Metrics under Sampling},
interhash = {152b01d87e4d4c389d5b85b0161e3bfb},
intrahash = {531f7d13e45d7904ab404ead09662074},
keywords = {deep_learning recommendation sampling},
note = {cite arxiv:2103.01474},
timestamp = {2021-04-28T16:27:06.000+0200},
title = {On Estimating Recommendation Evaluation Metrics under Sampling},
url = {http://arxiv.org/abs/2103.01474},
year = 2021
}