How to Get the Most out of Your Curation Effort Export

@article{citeulike:4537943, abstract = {Large-scale annotation efforts typically involve several experts who may disagree with each other. We propose an approach for modeling disagreements among experts that allows providing each annotation with a confidence value (i.e., the posterior probability that it is correct). Our approach allows computing certainty-level for individual annotations, given annotator-specific parameters estimated from data. We developed two probabilistic models for performing this analysis, compared these models using computer simulation, and tested each model's actual performance, based on a large data set generated by human annotators specifically for this study. We show that even in the worst-case scenario, when all annotators disagree, our approach allows us to significantly increase the probability of choosing the correct annotation. Along with this publication we make publicly available a corpus of 10,000 sentences annotated according to several cardinal dimensions that we have introduced in earlier work. The 10,000 sentences were all 3-fold annotated by a group of eight experts, while a 1,000-sentence subset was further 5-fold annotated by five new experts. While the presented data represent a specialized curation task, our modeling approach is general; most data annotation studies could benefit from our methodology.}, author = {Rzhetsky, Andrey and Shatkay, Hagit and Wilbur, W. John}, citeulike-article-id = {4537943}, citeulike-linkout-0 = {http://dx.doi.org/10.1371/journal.pcbi.1000391}, citeulike-linkout-1 = {http://view.ncbi.nlm.nih.gov/pubmed/19440548}, citeulike-linkout-2 = {http://www.hubmed.org/display.cgi?uids=19440548}, day = {22}, doi = {10.1371/journal.pcbi.1000391}, issn = {1553-7358}, journal = {PLoS Comput Biol}, keywords = {biocad, text\_mining}, month = {May}, number = {5}, pages = {e1000391+}, posted-at = {2009-06-14 16:03:27}, priority = {5}, publisher = {Public Library of Science}, title = {How to Get the Most out of Your Curation Effort}, url = {http://dx.doi.org/10.1371/journal.pcbi.1000391}, volume = {5}, year = {2009} }

TY - JOUR ID - citeulike:4537943 L3 - citeulike-article-id:4537943 N2 - Large-scale annotation efforts typically involve several experts who may disagree with each other. We propose an approach for modeling disagreements among experts that allows providing each annotation with a confidence value (i.e., the posterior probability that it is correct). Our approach allows computing certainty-level for individual annotations, given annotator-specific parameters estimated from data. We developed two probabilistic models for performing this analysis, compared these models using computer simulation, and tested each model's actual performance, based on a large data set generated by human annotators specifically for this study. We show that even in the worst-case scenario, when all annotators disagree, our approach allows us to significantly increase the probability of choosing the correct annotation. Along with this publication we make publicly available a corpus of 10,000 sentences annotated according to several cardinal dimensions that we have introduced in earlier work. The 10,000 sentences were all 3-fold annotated by a group of eight experts, while a 1,000-sentence subset was further 5-fold annotated by five new experts. While the presented data represent a specialized curation task, our modeling approach is general; most data annotation studies could benefit from our methodology. IS - 5 JF - PLoS Comput Biol SN - 1553-7358 TI - How to Get the Most out of Your Curation Effort PB - Public Library of Science VL - 5 SP - e1000391 KW - biocad KW - text_mining AU - Rzhetsky, Andrey AU - Shatkay, Hagit AU - Wilbur, John PY - 2009/05/22/ UR - http://dx.doi.org/10.1371/journal.pcbi.1000391 ER -