Labeling data is an important step in the supervised machine learning lifecycle. It is a laborious human activity comprised of repeated decision making: the human labeler decides which of several potential labels to apply to each example. Prior work has shown that providing AI assistance can improve the accuracy of binary decision tasks. However, the role of AI assistance in more complex data-labeling scenarios with a larger set of labels has not yet been explored. We designed an AI labeling assistant that uses a semi-supervised learning algorithm to predict the most probable labels for each example. We leverage these predictions to provide assistance in two ways: (i) providing a label recommendation and (ii) reducing the labeler’s decision space by focusing their attention on only the most probable labels. We conducted a user study (n=54) to evaluate an AI-assisted interface for data labeling in this context. Our results highlight that the AI assistance improves both labeler accuracy and speed, especially when the labeler finds the correct label in the reduced label space. We discuss findings related to the presentation of AI assistance and design implications for intelligent labeling interfaces.
Description
Increasing the Speed and Accuracy of Data Labeling Through an AI Assisted Interface | 26th International Conference on Intelligent User Interfaces
%0 Conference Paper
%1 Desmond_2021
%A Desmond, Michael
%A Muller, Michael
%A Ashktorab, Zahra
%A Dugan, Casey
%A Duesterwald, Evelyn
%A Brimijoin, Kristina
%A Finegan-Dollak, Catherine
%A Brachman, Michelle
%A Sharma, Aabhas
%A Joshi, Narendra Nath
%A Pan, Qian
%B 26th International Conference on Intelligent User Interfaces
%D 2021
%I ACM
%K adaptive-navigation-support annotation human-in-the-loop iui2021 labeling
%P 392-401
%R 10.1145/3397481.3450698
%T Increasing the Speed and Accuracy of Data Labeling Through an AI Assisted Interface
%U https://doi.org/10.1145%2F3397481.3450698
%X Labeling data is an important step in the supervised machine learning lifecycle. It is a laborious human activity comprised of repeated decision making: the human labeler decides which of several potential labels to apply to each example. Prior work has shown that providing AI assistance can improve the accuracy of binary decision tasks. However, the role of AI assistance in more complex data-labeling scenarios with a larger set of labels has not yet been explored. We designed an AI labeling assistant that uses a semi-supervised learning algorithm to predict the most probable labels for each example. We leverage these predictions to provide assistance in two ways: (i) providing a label recommendation and (ii) reducing the labeler’s decision space by focusing their attention on only the most probable labels. We conducted a user study (n=54) to evaluate an AI-assisted interface for data labeling in this context. Our results highlight that the AI assistance improves both labeler accuracy and speed, especially when the labeler finds the correct label in the reduced label space. We discuss findings related to the presentation of AI assistance and design implications for intelligent labeling interfaces.
@inproceedings{Desmond_2021,
abstract = {Labeling data is an important step in the supervised machine learning lifecycle. It is a laborious human activity comprised of repeated decision making: the human labeler decides which of several potential labels to apply to each example. Prior work has shown that providing AI assistance can improve the accuracy of binary decision tasks. However, the role of AI assistance in more complex data-labeling scenarios with a larger set of labels has not yet been explored. We designed an AI labeling assistant that uses a semi-supervised learning algorithm to predict the most probable labels for each example. We leverage these predictions to provide assistance in two ways: (i) providing a label recommendation and (ii) reducing the labeler’s decision space by focusing their attention on only the most probable labels. We conducted a user study (n=54) to evaluate an AI-assisted interface for data labeling in this context. Our results highlight that the AI assistance improves both labeler accuracy and speed, especially when the labeler finds the correct label in the reduced label space. We discuss findings related to the presentation of AI assistance and design implications for intelligent labeling interfaces.},
added-at = {2021-04-16T21:17:54.000+0200},
author = {Desmond, Michael and Muller, Michael and Ashktorab, Zahra and Dugan, Casey and Duesterwald, Evelyn and Brimijoin, Kristina and Finegan-Dollak, Catherine and Brachman, Michelle and Sharma, Aabhas and Joshi, Narendra Nath and Pan, Qian},
biburl = {https://www.bibsonomy.org/bibtex/2e4b5c9496c2460a647d5a3308123cecd/brusilovsky},
booktitle = {26th International Conference on Intelligent User Interfaces},
description = {Increasing the Speed and Accuracy of Data Labeling Through an AI Assisted Interface | 26th International Conference on Intelligent User Interfaces},
doi = {10.1145/3397481.3450698},
interhash = {646526bb87a52ada03e5ef231407d550},
intrahash = {e4b5c9496c2460a647d5a3308123cecd},
keywords = {adaptive-navigation-support annotation human-in-the-loop iui2021 labeling},
month = apr,
pages = {392-401},
publisher = {{ACM}},
timestamp = {2021-04-16T21:17:54.000+0200},
title = {Increasing the Speed and Accuracy of Data Labeling Through an {AI} Assisted Interface},
url = {https://doi.org/10.1145%2F3397481.3450698},
year = 2021
}