Where to stop reading a ranked list?: threshold optimization using truncated score distributions Export

@inproceedings{citeulike:5939977, abstract = {Ranked retrieval has a particular disadvantage in comparison with traditional Boolean retrieval: there is no clear cut-off point where to stop consulting results. This is a serious problem in some setups. We investigate and further develop methods to select the rank cut-off value which optimizes a given effectiveness measure. Assuming no other input than a system's output for a query--document scores and their distribution--the task is essentially a score-distributional threshold optimization problem. The recent trend in modeling score distributions is to use a normal-exponential mixture: normal for relevant, and exponential for non-relevant document scores. We discuss the two main theoretical problems with the current model, support incompatibility and non-convexity, and develop new models that address them. The main contributions of the paper are two truncated normal-exponential models, varying in the way the out-truncated score ranges are handled. We conduct a range of experiments using the TREC 2007 and 2008 Legal Track data, and show that the truncated models lead to significantly better results.}, address = {New York, NY, USA}, author = {Arampatzis, Avi and Kamps, Jaap and Robertson, Stephen}, booktitle = {SIGIR '09: Proceedings of the 32nd international ACM SIGIR conference on Research and development in information retrieval}, citeulike-article-id = {5939977}, citeulike-linkout-0 = {http://portal.acm.org/citation.cfm?id=1572031}, citeulike-linkout-1 = {http://dx.doi.org/10.1145/1571941.1572031}, doi = {10.1145/1571941.1572031}, isbn = {978-1-60558-483-6}, keywords = {normal-exponential, score-distributions, tfm}, location = {Boston, MA, USA}, pages = {524--531}, posted-at = {2009-10-14 17:08:01}, priority = {5}, publisher = {ACM}, title = {Where to stop reading a ranked list?: threshold optimization using truncated score distributions}, url = {http://dx.doi.org/10.1145/1571941.1572031}, year = {2009} }

TY - CONF ID - citeulike:5939977 L3 - citeulike-article-id:5939977 N2 - Ranked retrieval has a particular disadvantage in comparison with traditional Boolean retrieval: there is no clear cut-off point where to stop consulting results. This is a serious problem in some setups. We investigate and further develop methods to select the rank cut-off value which optimizes a given effectiveness measure. Assuming no other input than a system's output for a query--document scores and their distribution--the task is essentially a score-distributional threshold optimization problem. The recent trend in modeling score distributions is to use a normal-exponential mixture: normal for relevant, and exponential for non-relevant document scores. We discuss the two main theoretical problems with the current model, support incompatibility and non-convexity, and develop new models that address them. The main contributions of the paper are two truncated normal-exponential models, varying in the way the out-truncated score ranges are handled. We conduct a range of experiments using the TREC 2007 and 2008 Legal Track data, and show that the truncated models lead to significantly better results. CY - Boston, MA, USA SN - 978-1-60558-483-6 AD - New York, NY, USA EP - 531 TI - Where to stop reading a ranked list?: threshold optimization using truncated score distributions PB - ACM T2 - SIGIR '09: Proceedings of the 32nd international ACM SIGIR conference on Research and development in information retrieval SP - 524 KW - normal-exponential KW - score-distributions KW - tfm AU - Arampatzis, Avi AU - Kamps, Jaap AU - Robertson, Stephen PY - 2009/// UR - http://dx.doi.org/10.1145/1571941.1572031 ER -