Word embeddings generated by neural network methods such as word2vec (W2V)
are well known to exhibit seemingly linear behaviour, e.g. the embeddings of
analogy "woman is to queen as man is to king" approximately describe a
parallelogram. This property is particularly intriguing since the embeddings
are not trained to achieve it. Several explanations have been proposed, but
each introduces assumptions that do not hold in practice. We derive a
probabilistically grounded definition of paraphrasing and show it can be
re-interpreted as word transformation, a mathematical description of "$w_x$ is
to $w_y$". From these concepts we prove existence of the linear relationship
between W2V-type embeddings that underlies the analogical phenomenon, and
identify explicit error terms in the relationship.
Description
Analogies Explained: Towards Understanding Word Embeddings
%0 Generic
%1 allen2019analogies
%A Allen, Carl
%A Hospedales, Timothy
%D 2019
%K explained language model toread understanding word2vec
%T Analogies Explained: Towards Understanding Word Embeddings
%U http://arxiv.org/abs/1901.09813
%X Word embeddings generated by neural network methods such as word2vec (W2V)
are well known to exhibit seemingly linear behaviour, e.g. the embeddings of
analogy "woman is to queen as man is to king" approximately describe a
parallelogram. This property is particularly intriguing since the embeddings
are not trained to achieve it. Several explanations have been proposed, but
each introduces assumptions that do not hold in practice. We derive a
probabilistically grounded definition of paraphrasing and show it can be
re-interpreted as word transformation, a mathematical description of "$w_x$ is
to $w_y$". From these concepts we prove existence of the linear relationship
between W2V-type embeddings that underlies the analogical phenomenon, and
identify explicit error terms in the relationship.
@misc{allen2019analogies,
abstract = {Word embeddings generated by neural network methods such as word2vec (W2V)
are well known to exhibit seemingly linear behaviour, e.g. the embeddings of
analogy "woman is to queen as man is to king" approximately describe a
parallelogram. This property is particularly intriguing since the embeddings
are not trained to achieve it. Several explanations have been proposed, but
each introduces assumptions that do not hold in practice. We derive a
probabilistically grounded definition of paraphrasing and show it can be
re-interpreted as word transformation, a mathematical description of "$w_x$ is
to $w_y$". From these concepts we prove existence of the linear relationship
between W2V-type embeddings that underlies the analogical phenomenon, and
identify explicit error terms in the relationship.},
added-at = {2019-03-18T17:35:07.000+0100},
author = {Allen, Carl and Hospedales, Timothy},
biburl = {https://www.bibsonomy.org/bibtex/23dab227a78a265488954de1b05347121/hotho},
description = {Analogies Explained: Towards Understanding Word Embeddings},
interhash = {e665fecc2000ff3e6d7d047f71ee69a0},
intrahash = {3dab227a78a265488954de1b05347121},
keywords = {explained language model toread understanding word2vec},
note = {cite arxiv:1901.09813},
timestamp = {2019-03-18T17:35:07.000+0100},
title = {Analogies Explained: Towards Understanding Word Embeddings},
url = {http://arxiv.org/abs/1901.09813},
year = 2019
}