%0 Conference Paper %1 info3-inproceedings-2013-475 %A Redi, Judith %A Hoßfeld, Tobias %A Korshunov, Pavel %A Mazza, Filippo %A Povoa, Isabel %A Keimel, Christian %B ACM CrowdMM 2013 %C Barcelona, Spain %D 2013 %K myown %T Crowdsourcing-based multimedia subjective evaluations: a case study on image recognizability and aesthetic appeal %X Crowdsourcing is a popular approach for employing people online to perform short and simple tasks. Research on Quality of Experience (QoE), which relies on subjective evaluations of media, could especially benefit from it. One question related to QoE, which is as well important for image retrieval and summarization systems, concerns modelling and quantifying the subjective notions of `beauty' (aesthetic appeal) and `something well-known' (content recognizability). Since both notions may be subject to cultural and social effects, crowdsourcing, which allows to employ worldwide, can be a great tool for carrying on subjective studies in a time and cost-effective way. On the other hand, the crowdsourcing environment does not allow for the degree of excperimental control which is necessary to guarantee reliable subjective data. In this paper, we evaluate aesthetic appeal and recognizability of images using a crowdsourcing platform and compare the outcomes with more conventional evaluations conducted in a controlled lab environment. We find high correlation between crowdsourcing and lab scores for recognizability but not for aesthetic appeal. The analysis of the results indicates possible causes for this, suggesting that crowdsourcing can be used for QoE subjective assessments as long as the workers' tasks are designed with extreme care to avoid misinterpretations and unreliability.

@inproceedings{info3-inproceedings-2013-475, abstract = {Crowdsourcing is a popular approach for employing people online to perform short and simple tasks. Research on Quality of Experience (QoE), which relies on subjective evaluations of media, could especially benefit from it. One question related to QoE, which is as well important for image retrieval and summarization systems, concerns modelling and quantifying the subjective notions of `beauty' (aesthetic appeal) and `something well-known' (content recognizability). Since both notions may be subject to cultural and social effects, crowdsourcing, which allows to employ worldwide, can be a great tool for carrying on subjective studies in a time and cost-effective way. On the other hand, the crowdsourcing environment does not allow for the degree of excperimental control which is necessary to guarantee reliable subjective data. In this paper, we evaluate aesthetic appeal and recognizability of images using a crowdsourcing platform and compare the outcomes with more conventional evaluations conducted in a controlled lab environment. We find high correlation between crowdsourcing and lab scores for recognizability but not for aesthetic appeal. The analysis of the results indicates possible causes for this, suggesting that crowdsourcing can be used for QoE subjective assessments as long as the workers' tasks are designed with extreme care to avoid misinterpretations and unreliability. }, added-at = {2016-03-10T17:38:16.000+0100}, address = {Barcelona, Spain}, author = {Redi, Judith and Hoßfeld, Tobias and Korshunov, Pavel and Mazza, Filippo and Povoa, Isabel and Keimel, Christian}, biburl = {https://www.bibsonomy.org/bibtex/2d4d1397d2ec0ca9d009efb1e65187158/uniwue_info3}, booktitle = {ACM CrowdMM 2013}, interhash = {5fb0d5f561e87724d01c87144b0dfad8}, intrahash = {d4d1397d2ec0ca9d009efb1e65187158}, keywords = {myown}, month = {10}, timestamp = {2022-03-14T00:11:00.000+0100}, title = {Crowdsourcing-based multimedia subjective evaluations: a case study on image recognizability and aesthetic appeal}, year = 2013 }