Lacking standardized extrinsic evaluation methods for vector representations
of words, the NLP community has relied heavily on word similarity tasks as a
proxy for intrinsic evaluation of word vectors. Word similarity evaluation,
which correlates the distance between vectors and human judgments of semantic
similarity is attractive, because it is computationally inexpensive and fast.
In this paper we present several problems associated with the evaluation of
word vectors on word similarity datasets, and summarize existing solutions. Our
study suggests that the use of word similarity tasks for evaluation of word
vectors is not sustainable and calls for further research on evaluation
methods.
Description
[1605.02276v1] Problems With Evaluation of Word Embeddings Using Word Similarity Tasks
%0 Generic
%1 faruqui2016problems
%A Faruqui, Manaal
%A Tsvetkov, Yulia
%A Rastogi, Pushpendre
%A Dyer, Chris
%D 2016
%K embedding evaluation problem semantics
%T Problems With Evaluation of Word Embeddings Using Word Similarity Tasks
%U http://arxiv.org/abs/1605.02276
%X Lacking standardized extrinsic evaluation methods for vector representations
of words, the NLP community has relied heavily on word similarity tasks as a
proxy for intrinsic evaluation of word vectors. Word similarity evaluation,
which correlates the distance between vectors and human judgments of semantic
similarity is attractive, because it is computationally inexpensive and fast.
In this paper we present several problems associated with the evaluation of
word vectors on word similarity datasets, and summarize existing solutions. Our
study suggests that the use of word similarity tasks for evaluation of word
vectors is not sustainable and calls for further research on evaluation
methods.
@misc{faruqui2016problems,
abstract = {Lacking standardized extrinsic evaluation methods for vector representations
of words, the NLP community has relied heavily on word similarity tasks as a
proxy for intrinsic evaluation of word vectors. Word similarity evaluation,
which correlates the distance between vectors and human judgments of semantic
similarity is attractive, because it is computationally inexpensive and fast.
In this paper we present several problems associated with the evaluation of
word vectors on word similarity datasets, and summarize existing solutions. Our
study suggests that the use of word similarity tasks for evaluation of word
vectors is not sustainable and calls for further research on evaluation
methods.},
added-at = {2016-05-14T18:56:52.000+0200},
author = {Faruqui, Manaal and Tsvetkov, Yulia and Rastogi, Pushpendre and Dyer, Chris},
biburl = {https://www.bibsonomy.org/bibtex/240e09cab22c78717be5eaaee8b935b54/thoni},
description = {[1605.02276v1] Problems With Evaluation of Word Embeddings Using Word Similarity Tasks},
interhash = {36111f331d7ae81b4c10087c1518c336},
intrahash = {40e09cab22c78717be5eaaee8b935b54},
keywords = {embedding evaluation problem semantics},
note = {cite arxiv:1605.02276v1},
timestamp = {2016-09-06T08:23:07.000+0200},
title = {Problems With Evaluation of Word Embeddings Using Word Similarity Tasks},
url = {http://arxiv.org/abs/1605.02276},
year = 2016
}