Artificial intelligence (AI) possesses the potential to augment customer service employees e.g. via decision support or solution recommendations. Still, its underlying data for training and testing the AI systems is provided by human annotators through human-in-the-loop configurations. However, due to the high effort for annotators and lack of incentives, AI systems face low underlying data quality. That in turn results in low prediction performance and limited acceptance by the targeted user group. Faced with the enormous volume and increasing complexity of service requests, IT service management (ITSM) especially, relies on high data quality for AI systems and in-corporating domain-specific knowledge. By analyzing the existing labeling process in that specific case, we design a revised to-be process and develop a conceptual model from a value co-creation perspective. Finally, a functional prototype as an instantiation in the ITSM domain is implemented and evaluated through accuracy metrics and user evaluation. The results show that the new process increases the perceived value of both labeling quality and the perceived prediction quality. Thus, we contribute a conceptual model that supports the systematic design of efficient and interactive labeling processes in diverse applications of reinforcement learning systems.
%0 Conference Paper
%1 ls_leimeister
%A Reinhard, Philipp
%A Li, Mahei Manhai
%A Dickhaut, Ernestine
%A Reh, Cornelius
%A Peters, Christoph
%A Leimeister, Jan Marco
%B International Conference on Design Science Research (DESRIST)
%C Pretoria, South Africa
%D 2023
%E Gerber, Aurona
%E Baskerville, Richard
%K Artificial_intelligence Human-in-the-loop Interactive_labeling Value_co-creation dempub itegpub pub_cpe pub_cre pub_edi pub_jml pub_mli pub_pre
%T A Conceptual Model for Labeling in Reinforcement Learning Systems: A Value Co-Creation Perspective
%U https://pubs.wi-kassel.de/wp-content/uploads/2023/03/JML_923.pdf
%X Artificial intelligence (AI) possesses the potential to augment customer service employees e.g. via decision support or solution recommendations. Still, its underlying data for training and testing the AI systems is provided by human annotators through human-in-the-loop configurations. However, due to the high effort for annotators and lack of incentives, AI systems face low underlying data quality. That in turn results in low prediction performance and limited acceptance by the targeted user group. Faced with the enormous volume and increasing complexity of service requests, IT service management (ITSM) especially, relies on high data quality for AI systems and in-corporating domain-specific knowledge. By analyzing the existing labeling process in that specific case, we design a revised to-be process and develop a conceptual model from a value co-creation perspective. Finally, a functional prototype as an instantiation in the ITSM domain is implemented and evaluated through accuracy metrics and user evaluation. The results show that the new process increases the perceived value of both labeling quality and the perceived prediction quality. Thus, we contribute a conceptual model that supports the systematic design of efficient and interactive labeling processes in diverse applications of reinforcement learning systems.
@inproceedings{ls_leimeister,
abstract = {Artificial intelligence (AI) possesses the potential to augment customer service employees e.g. via decision support or solution recommendations. Still, its underlying data for training and testing the AI systems is provided by human annotators through human-in-the-loop configurations. However, due to the high effort for annotators and lack of incentives, AI systems face low underlying data quality. That in turn results in low prediction performance and limited acceptance by the targeted user group. Faced with the enormous volume and increasing complexity of service requests, IT service management (ITSM) especially, relies on high data quality for AI systems and in-corporating domain-specific knowledge. By analyzing the existing labeling process in that specific case, we design a revised to-be process and develop a conceptual model from a value co-creation perspective. Finally, a functional prototype as an instantiation in the ITSM domain is implemented and evaluated through accuracy metrics and user evaluation. The results show that the new process increases the perceived value of both labeling quality and the perceived prediction quality. Thus, we contribute a conceptual model that supports the systematic design of efficient and interactive labeling processes in diverse applications of reinforcement learning systems.},
added-at = {2023-03-29T12:35:56.000+0200},
address = {Pretoria, South Africa},
author = {Reinhard, Philipp and Li, Mahei Manhai and Dickhaut, Ernestine and Reh, Cornelius and Peters, Christoph and Leimeister, Jan Marco},
biburl = {https://www.bibsonomy.org/bibtex/273b4c87b566701f48703ad512b668b70/ls_leimeister},
booktitle = {International Conference on Design Science Research (DESRIST)},
editor = {Gerber, Aurona and Baskerville, Richard},
eventdate = {31 May - 02 Jun 2023},
eventtitle = {International Conference on Design Science Research in Information Systems and Technology (DESRIST)},
interhash = {010b2a7f252c8da333f6b7a124faf780},
intrahash = {73b4c87b566701f48703ad512b668b70},
keywords = {Artificial_intelligence Human-in-the-loop Interactive_labeling Value_co-creation dempub itegpub pub_cpe pub_cre pub_edi pub_jml pub_mli pub_pre},
language = {English},
timestamp = {2023-06-07T12:48:36.000+0200},
title = {A Conceptual Model for Labeling in Reinforcement Learning Systems: A Value Co-Creation Perspective},
url = {https://pubs.wi-kassel.de/wp-content/uploads/2023/03/JML_923.pdf},
venue = {Pretoria, South Africa},
year = 2023
}