Crowdsourcing user studies with Mechanical Turk Export

@inproceedings{Kittur08, abstract = {User studies are important for many aspects of the design process and involve techniques ranging from informal surveys to rigorous laboratory studies. However, the costs involved in engaging users often requires practitioners to trade off between sample size, time requirements, and monetary costs. Micro-task markets, such as Amazon's Mechanical Turk, offer a potential paradigm for engaging a large number of users for low time and monetary costs. Here we investigate the utility of a micro-task market for collecting user measurements, and discuss design considerations for developing remote micro user evaluation tasks. Although micro-task markets have great potential for rapidly collecting user measurements at low costs, we found that special care is needed in formulating tasks in order to harness the capabilities of the approach.}, address = {New York, NY, USA}, author = {Kittur, Aniket and Chi, Ed H. and Suh, Bongwon}, booktitle = {CHI '08: Proceeding of the twenty-sixth annual SIGCHI conference on Human factors in computing systems}, citeulike-article-id = {2686454}, citeulike-linkout-0 = {http://portal.acm.org/citation.cfm?id=1357054.1357127}, citeulike-linkout-1 = {http://dx.doi.org/10.1145/1357054.1357127}, doi = {10.1145/1357054.1357127}, isbn = {978-1-60558-011-1}, keywords = {crowdsourcing, evaluation, hci, mechturk, user-study}, location = {Florence, Italy}, pages = {453--456}, posted-at = {2008-04-29 11:08:59}, priority = {0}, publisher = {ACM}, title = {Crowdsourcing user studies with Mechanical Turk}, url = {http://dx.doi.org/10.1145/1357054.1357127}, year = {2008} }

TY - CONF ID - Kittur08 L3 - citeulike-article-id:2686454 N2 - User studies are important for many aspects of the design process and involve techniques ranging from informal surveys to rigorous laboratory studies. However, the costs involved in engaging users often requires practitioners to trade off between sample size, time requirements, and monetary costs. Micro-task markets, such as Amazon's Mechanical Turk, offer a potential paradigm for engaging a large number of users for low time and monetary costs. Here we investigate the utility of a micro-task market for collecting user measurements, and discuss design considerations for developing remote micro user evaluation tasks. Although micro-task markets have great potential for rapidly collecting user measurements at low costs, we found that special care is needed in formulating tasks in order to harness the capabilities of the approach. CY - Florence, Italy SN - 978-1-60558-011-1 AD - New York, NY, USA EP - 456 TI - Crowdsourcing user studies with Mechanical Turk PB - ACM T2 - CHI '08: Proceeding of the twenty-sixth annual SIGCHI conference on Human factors in computing systems SP - 453 KW - crowdsourcing KW - evaluation KW - hci KW - mechturk KW - user-study AU - Kittur, Aniket AU - Chi, Ed AU - Suh, Bongwon PY - 2008/// UR - http://dx.doi.org/10.1145/1357054.1357127 ER -