%0 Generic %1 wang2018multitask %A Wang, Alex %A Singh, Amapreet %A Michael, Julian %A Hill, Felix %A Levy, Omer %A Bowman, Samuel R. %D 2018 %K %T GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding %U http://arxiv.org/abs/1804.07461 %X For natural language understanding (NLU) technology to be maximally useful, both practically and as a scientific object of study, it must be general: it must be able to process language in a way that is not exclusively tailored to any one specific task or dataset. In pursuit of this objective, we introduce the General Language Understanding Evaluation benchmark (GLUE), a tool for evaluating and analyzing the performance of models across a diverse range of existing NLU tasks. GLUE is model-agnostic, but it incentivizes sharing knowledge across tasks because certain tasks have very limited training data. We further provide a hand-crafted diagnostic test suite that enables detailed linguistic analysis of NLU models. We evaluate baselines based on current methods for multi-task and transfer learning and find that they do not immediately give substantial improvements over the aggregate performance of training a separate model per task, indicating room for improvement in developing general and robust NLU systems.

@misc{wang2018multitask, abstract = {For natural language understanding (NLU) technology to be maximally useful, both practically and as a scientific object of study, it must be general: it must be able to process language in a way that is not exclusively tailored to any one specific task or dataset. In pursuit of this objective, we introduce the General Language Understanding Evaluation benchmark (GLUE), a tool for evaluating and analyzing the performance of models across a diverse range of existing NLU tasks. GLUE is model-agnostic, but it incentivizes sharing knowledge across tasks because certain tasks have very limited training data. We further provide a hand-crafted diagnostic test suite that enables detailed linguistic analysis of NLU models. We evaluate baselines based on current methods for multi-task and transfer learning and find that they do not immediately give substantial improvements over the aggregate performance of training a separate model per task, indicating room for improvement in developing general and robust NLU systems.}, added-at = {2021-05-21T12:32:03.000+0200}, author = {Wang, Alex and Singh, Amapreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R.}, biburl = {https://www.bibsonomy.org/bibtex/218baed31ae7b31a210bcac8ea1f78d3b/lea-w}, interhash = {dad3beefc98da499511538ec9c95e4af}, intrahash = {18baed31ae7b31a210bcac8ea1f78d3b}, keywords = {}, note = {cite arxiv:1804.07461Comment: https://gluebenchmark.com/}, timestamp = {2021-05-21T12:32:03.000+0200}, title = {GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding}, url = {http://arxiv.org/abs/1804.07461}, year = 2018 }