A. Kittur, E. Chi, and B. Suh. Proceeding of the 16thth annual SIGCHI conference on Human Factors in Computing Systems (CHI-08), Florence, Italy, (2008)
Abstract
User studies are important for many aspects of the design process and involve techniques ranging from informal surveys to rigorous laboratory studies. However, the costs involved in engaging users often requires practitioners to trade off between sample size, time requirements, and monetary costs. Micro-task markets, such as Amazon's Mechanical Turk, offer a potential paradigm for engaging a large number of users for low time and monetary costs. Here we investigate the utility of a micro-task market for collecting user measurements, and discuss design considerations for developing remote micro user evaluation tasks. Although micro-task markets have great potential for rapidly collecting user measurements at low costs, we found that special care is needed in formulating tasks in order to harness the capabilities of the approach.
%0 Conference Paper
%1 Kittur:EtAl:08
%A Kittur, Aniket
%A Chi, Ed H.
%A Suh, Bongwon
%B Proceeding of the 16thth annual SIGCHI conference on Human Factors in Computing Systems (CHI-08)
%C Florence, Italy
%D 2008
%K 2008 mturk
%T Crowdsourcing user studies with Mechanical Turk
%U http://www-users.cs.umn.edu/~echi/papers/2008-CHI2008/2008-02-mech-turk-online-experiments-chi1049-kittur.pdf
%X User studies are important for many aspects of the design process and involve techniques ranging from informal surveys to rigorous laboratory studies. However, the costs involved in engaging users often requires practitioners to trade off between sample size, time requirements, and monetary costs. Micro-task markets, such as Amazon's Mechanical Turk, offer a potential paradigm for engaging a large number of users for low time and monetary costs. Here we investigate the utility of a micro-task market for collecting user measurements, and discuss design considerations for developing remote micro user evaluation tasks. Although micro-task markets have great potential for rapidly collecting user measurements at low costs, we found that special care is needed in formulating tasks in order to harness the capabilities of the approach.
@inproceedings{Kittur:EtAl:08,
abstract = {User studies are important for many aspects of the design process and involve techniques ranging from informal surveys to rigorous laboratory studies. However, the costs involved in engaging users often requires practitioners to trade off between sample size, time requirements, and monetary costs. Micro-task markets, such as Amazon's Mechanical Turk, offer a potential paradigm for engaging a large number of users for low time and monetary costs. Here we investigate the utility of a micro-task market for collecting user measurements, and discuss design considerations for developing remote micro user evaluation tasks. Although micro-task markets have great potential for rapidly collecting user measurements at low costs, we found that special care is needed in formulating tasks in order to harness the capabilities of the approach.},
added-at = {2009-05-27T01:12:25.000+0200},
address = {Florence, Italy},
author = {Kittur, Aniket and Chi, Ed H. and Suh, Bongwon},
biburl = {https://www.bibsonomy.org/bibtex/2d47ab4e0d9fa2837310d75432fdaa4b5/seandalai},
booktitle = {Proceeding of the 16thth annual SIGCHI conference on Human Factors in Computing Systems (CHI-08)},
interhash = {3e22d46ecdff57efe1057e653b1d64f3},
intrahash = {d47ab4e0d9fa2837310d75432fdaa4b5},
keywords = {2008 mturk},
timestamp = {2009-05-27T01:12:25.000+0200},
title = {Crowdsourcing user studies with Mechanical Turk},
url = {http://www-users.cs.umn.edu/~echi/papers/2008-CHI2008/2008-02-mech-turk-online-experiments-chi1049-kittur.pdf},
year = 2008
}