%0 Conference Paper %1 van_der_Stappen_2021 %A van der Stappen, Almar %A Funk, Mathias %B 26th International Conference on Intelligent User Interfaces %D 2021 %I ACM %K iui %P 514-519 %R 10.1145/3397481.3450668 %T Towards Guidelines for Designing Human-in-the-Loop Machine Training Interfaces %U https://doi.org/10.1145%2F3397481.3450668 %X Supervised machine learning approaches commonly require good availability and quality of training data. In applications that depend on human-labeled data, especially from experts, or that depend on contextual knowledge for training data sets, the human-in-the-loop presents a serious bottleneck to the scalability of training efforts. Even if human labeling is generally feasible, sustained human performance and high-quality labels in larger quantities are challenging. Interactive Machine Learning can help solve usability problems in traditional machine learning by giving users agency in deciding how systems learn from data. Yet, the field lacks clear design guidelines for such interfaces, specifically regarding the scaling of training processes. In this paper, we present results from a pilot study in which participants interacted with several interface variants of a recommender engine and evaluated them on interaction and efficiency parameters. Based on the performance of these different learning system implementations we propose design guidelines for the design of such systems and a score for comparative evaluation, in which we combine interaction experience and system learning efficiency into one relative scoring unit.

@inproceedings{van_der_Stappen_2021, abstract = {Supervised machine learning approaches commonly require good availability and quality of training data. In applications that depend on human-labeled data, especially from experts, or that depend on contextual knowledge for training data sets, the human-in-the-loop presents a serious bottleneck to the scalability of training efforts. Even if human labeling is generally feasible, sustained human performance and high-quality labels in larger quantities are challenging. Interactive Machine Learning can help solve usability problems in traditional machine learning by giving users agency in deciding how systems learn from data. Yet, the field lacks clear design guidelines for such interfaces, specifically regarding the scaling of training processes. In this paper, we present results from a pilot study in which participants interacted with several interface variants of a recommender engine and evaluated them on interaction and efficiency parameters. Based on the performance of these different learning system implementations we propose design guidelines for the design of such systems and a score for comparative evaluation, in which we combine interaction experience and system learning efficiency into one relative scoring unit.}, added-at = {2022-01-16T20:45:42.000+0100}, author = {van der Stappen, Almar and Funk, Mathias}, biburl = {https://www.bibsonomy.org/bibtex/2896dfa1e7b438580ddb9d568b2b1b51a/brusilovsky}, booktitle = {26th International Conference on Intelligent User Interfaces}, description = {Towards Guidelines for Designing Human-in-the-Loop Machine Training Interfaces | 26th International Conference on Intelligent User Interfaces}, doi = {10.1145/3397481.3450668}, interhash = {dd84d72f3091605a58e149aa36a04aa0}, intrahash = {896dfa1e7b438580ddb9d568b2b1b51a}, keywords = {iui}, month = apr, pages = {514-519}, publisher = {{ACM}}, timestamp = {2022-01-16T20:46:26.000+0100}, title = {Towards Guidelines for Designing Human-in-the-Loop Machine Training Interfaces}, url = {https://doi.org/10.1145%2F3397481.3450668}, year = 2021 }