Solving complex classification tasks using deep neural networks typically requires large amounts of annotated data. However, corresponding class labels are noisy when provided by error-prone annotators, e.g., crowdworkers. Training standard deep neural networks leads to subpar performances in such multi-annotator supervised learning settings. We address this issue by presenting a probabilistic training framework named multi-annotator deep learning (MaDL). A downstream ground truth and an annotator performance model are jointly trained in an end-to-end learning approach. The ground truth model learns to predict instances' true class labels, while the annotator performance model infers probabilistic estimates of annotators' performances. A modular network architecture enables us to make varying assumptions regarding annotators' performances, e.g., an optional class or instance dependency. Further, we learn annotator embeddings to estimate annotators' densities within a latent space as proxies of their potentially correlated annotations. Together with a weighted loss function, we improve the learning from correlated annotation patterns. In a comprehensive evaluation, we examine three research questions about multi-annotator supervised learning. Our findings show MaDL's state-of-the-art performance and robustness against many correlated, spamming annotators.
%0 Journal Article
%1 herde2023multi
%A Herde, Marek
%A Huseljic, Denis
%A Sick, Bernhard
%D 2023
%J Transactions on Machine Learning Research
%K imported itegpub isac-www NoisyLabels DeepLearning Crowdsourcing
%T Multi-annotator Deep Learning: A Probabilistic Framework for Classification
%U https://openreview.net/forum?id=MgdoxzImlK
%X Solving complex classification tasks using deep neural networks typically requires large amounts of annotated data. However, corresponding class labels are noisy when provided by error-prone annotators, e.g., crowdworkers. Training standard deep neural networks leads to subpar performances in such multi-annotator supervised learning settings. We address this issue by presenting a probabilistic training framework named multi-annotator deep learning (MaDL). A downstream ground truth and an annotator performance model are jointly trained in an end-to-end learning approach. The ground truth model learns to predict instances' true class labels, while the annotator performance model infers probabilistic estimates of annotators' performances. A modular network architecture enables us to make varying assumptions regarding annotators' performances, e.g., an optional class or instance dependency. Further, we learn annotator embeddings to estimate annotators' densities within a latent space as proxies of their potentially correlated annotations. Together with a weighted loss function, we improve the learning from correlated annotation patterns. In a comprehensive evaluation, we examine three research questions about multi-annotator supervised learning. Our findings show MaDL's state-of-the-art performance and robustness against many correlated, spamming annotators.
@article{herde2023multi,
abstract = {Solving complex classification tasks using deep neural networks typically requires large amounts of annotated data. However, corresponding class labels are noisy when provided by error-prone annotators, e.g., crowdworkers. Training standard deep neural networks leads to subpar performances in such multi-annotator supervised learning settings. We address this issue by presenting a probabilistic training framework named multi-annotator deep learning (MaDL). A downstream ground truth and an annotator performance model are jointly trained in an end-to-end learning approach. The ground truth model learns to predict instances' true class labels, while the annotator performance model infers probabilistic estimates of annotators' performances. A modular network architecture enables us to make varying assumptions regarding annotators' performances, e.g., an optional class or instance dependency. Further, we learn annotator embeddings to estimate annotators' densities within a latent space as proxies of their potentially correlated annotations. Together with a weighted loss function, we improve the learning from correlated annotation patterns. In a comprehensive evaluation, we examine three research questions about multi-annotator supervised learning. Our findings show MaDL's state-of-the-art performance and robustness against many correlated, spamming annotators.},
added-at = {2023-10-18T11:37:39.000+0200},
author = {Herde, Marek and Huseljic, Denis and Sick, Bernhard},
biburl = {https://www.bibsonomy.org/bibtex/2585c0c2c8eb52d717ffe6c603d01084a/ies},
codeurl = {https://github.com/ies-research/multi-annotator-deep-learning},
interhash = {f3f942451e0beb8358412bfe5ea5618a},
intrahash = {585c0c2c8eb52d717ffe6c603d01084a},
journal = {Transactions on Machine Learning Research},
keywords = {imported itegpub isac-www NoisyLabels DeepLearning Crowdsourcing},
timestamp = {2023-10-18T11:37:39.000+0200},
title = {Multi-annotator Deep Learning: A Probabilistic Framework for Classification},
url = {https://openreview.net/forum?id=MgdoxzImlK},
year = 2023
}