%0 Conference Paper %1 Kochhar:2010:ALH:1837885.1837890 %A Kochhar, Shailesh %A Mazzocchi, Stefano %A Paritosh, Praveen %B Proceedings of the ACM SIGKDD Workshop on Human Computation %C New York, NY, USA %D 2010 %I ACM %K TOREAD crowdsourcing sociology %P 10--17 %R 10.1145/1837885.1837890 %T The anatomy of a large-scale human computation engine %U http://doi.acm.org/10.1145/1837885.1837890 %X In this paper we describe Rabj, an engine designed to simplify collecting human input. We have used Rabj to collect over 2.3 million human judgments to augment data mining, data entry, and curation tasks at Freebase over the course of a year. We illustrate several successful applications that have used Rabj to collect human judgment. We describe how the architecture and design decisions of Rabj are affected by the constraints of <i>content agnosticity, data freshness, latency</i> and <i>visibility</i>. We present work aimed at increasing the yield and reliability of human computation efforts. Finally, we discuss empirical observations and lessons learned in the course of a year of operating the service. %@ 978-1-4503-0222-7

@inproceedings{Kochhar:2010:ALH:1837885.1837890, abstract = {In this paper we describe Rabj, an engine designed to simplify collecting human input. We have used Rabj to collect over 2.3 million human judgments to augment data mining, data entry, and curation tasks at Freebase over the course of a year. We illustrate several successful applications that have used Rabj to collect human judgment. We describe how the architecture and design decisions of Rabj are affected by the constraints of <i>content agnosticity, data freshness, latency</i> and <i>visibility</i>. We present work aimed at increasing the yield and reliability of human computation efforts. Finally, we discuss empirical observations and lessons learned in the course of a year of operating the service.}, acmid = {1837890}, added-at = {2012-03-28T18:36:55.000+0200}, address = {New York, NY, USA}, author = {Kochhar, Shailesh and Mazzocchi, Stefano and Paritosh, Praveen}, biburl = {https://www.bibsonomy.org/bibtex/2a974ce4571651fd4e73cb023a9aa90c8/mstrohm}, booktitle = {Proceedings of the ACM SIGKDD Workshop on Human Computation}, description = {The anatomy of a large-scale human computation engine}, doi = {10.1145/1837885.1837890}, interhash = {3e5aaca09e0b777ffbc38886b9009ffb}, intrahash = {a974ce4571651fd4e73cb023a9aa90c8}, isbn = {978-1-4503-0222-7}, keywords = {TOREAD crowdsourcing sociology}, location = {Washington DC}, numpages = {8}, pages = {10--17}, publisher = {ACM}, series = {HCOMP '10}, timestamp = {2012-03-28T18:36:55.000+0200}, title = {The anatomy of a large-scale human computation engine}, url = {http://doi.acm.org/10.1145/1837885.1837890}, year = 2010 }