%0 Conference Paper %1 hossfeld2014survey %A Hoßfeld, T. %A Hirth, M. %A Korshunov, P. %A Hanhart, P. %A Gardlo, B. %A Keimel, C. %A Timmerer, C. %B Multimedia Signal Processing (MMSP), 2014 IEEE 16th International Workshop on %D 2014 %K ACR testing methodology Amazon Mechanical Turk Crowdsourcing DCR testing methodology Internet Microworkers Multimedia communication PC testing methodology QoE assessment Quality assessment Reliability Streaming media Testing Training Web-based crowdsourcing frameworks business data processing multimedia computing myown online multimedia content assessment outsourcing quality of experience subjective multimedia evaluations subjective multimedia quality assessment %P 1-6 %R 10.1109/MMSP.2014.6958831 %T Survey of web-based crowdsourcing frameworks for subjective quality assessment %X The popularity of the crowdsourcing for performing various tasks online increased significantly in the past few years. The low cost and flexibility of crowdsourcing, in particular, attracted researchers in the field of subjective multimedia evaluations and Quality of Experience (QoE). Since online assessment of multimedia content is challenging, several dedicated frameworks were created to aid in the designing of the tests, including the support of the testing methodologies like ACR, DCR, and PC, setting up the tasks, training sessions, screening of the subjects, and storage of the resulted data. In this paper, we focus on the web-based frameworks for multimedia quality assessments that support commonly used crowdsourcing platforms such as Amazon Mechanical Turk and Microworkers. We provide a detailed overview of the crowdsourcing frameworks and evaluate them to aid researchers in the field of QoE assessment in the selection of frameworks and crowdsourcing platforms that are adequate for their experiments.

@inproceedings{hossfeld2014survey, abstract = {The popularity of the crowdsourcing for performing various tasks online increased significantly in the past few years. The low cost and flexibility of crowdsourcing, in particular, attracted researchers in the field of subjective multimedia evaluations and Quality of Experience (QoE). Since online assessment of multimedia content is challenging, several dedicated frameworks were created to aid in the designing of the tests, including the support of the testing methodologies like ACR, DCR, and PC, setting up the tasks, training sessions, screening of the subjects, and storage of the resulted data. In this paper, we focus on the web-based frameworks for multimedia quality assessments that support commonly used crowdsourcing platforms such as Amazon Mechanical Turk and Microworkers. We provide a detailed overview of the crowdsourcing frameworks and evaluate them to aid researchers in the field of QoE assessment in the selection of frameworks and crowdsourcing platforms that are adequate for their experiments.}, added-at = {2015-02-27T18:21:34.000+0100}, author = {Hoßfeld, T. and Hirth, M. and Korshunov, P. and Hanhart, P. and Gardlo, B. and Keimel, C. and Timmerer, C.}, biburl = {https://www.bibsonomy.org/bibtex/270d86ade740a5d24322e6eedd8394c9f/keimel}, booktitle = {Multimedia Signal Processing (MMSP), 2014 IEEE 16th International Workshop on}, doi = {10.1109/MMSP.2014.6958831}, interhash = {a0a5725fbb11a371d34f36262291a129}, intrahash = {70d86ade740a5d24322e6eedd8394c9f}, keywords = {ACR testing methodology Amazon Mechanical Turk Crowdsourcing DCR testing methodology Internet Microworkers Multimedia communication PC testing methodology QoE assessment Quality assessment Reliability Streaming media Testing Training Web-based crowdsourcing frameworks business data processing multimedia computing myown online multimedia content assessment outsourcing quality of experience subjective multimedia evaluations subjective multimedia quality assessment}, month = sep, pages = {1-6}, timestamp = {2015-08-05T13:26:03.000+0200}, title = {Survey of web-based crowdsourcing frameworks for subjective quality assessment}, year = 2014 }