PSkip: estimating relevance ranking quality from web search clickthrough data Export

@inproceedings{citeulike:5044005, abstract = {In this article, we report our efforts in mining the information encoded as clickthrough data in the server logs to evaluate and monitor the relevance ranking quality of a commercial web search engine. We describe a metric called pSkip that aims to quantify the ranking quality by estimating the probability of users encountering non relevant results that cost them the efforts to read and skip. A search engine with a lower pSkip is regarded as having a better ranking quality. A key design goal of pSkip is to integrate the findings from two sets of user studies that utilize eye-tracking devices to track users' browsing patterns on the search result pages, and that use specially instrumented browsers to actively solicit users' explicit judgments on their search activities. We present the derivation of the maximum likelihood estimation of pSkip and demonstrate its efficacy in describing the user study data. The mathematical properties of pSkip are further analyzed and compared with several objective metrics as well as the cumulated gain method that uses subjective judgments. Experimental data show that pSkip can measure aspects of the search quality that these existing metrics are not designed or fail to address, such as identifying the real search intents expressed in the ambiguous queries. Although effective and superior in many ways, we also report a series of experiments that show pSkip may be influenced by system issues that are not directly related to relevance ranking, suggesting that measurements complementary to pSkip are still needed in order to form a holistic and accurate characterization of the ranking quality.}, address = {New York, NY, USA}, author = {Wang, Kuansan and Walker, Toby and Zheng, Zijian}, booktitle = {KDD '09: Proceedings of the 15th ACM SIGKDD international conference on Knowledge discovery and data mining}, citeulike-article-id = {5044005}, citeulike-linkout-0 = {http://portal.acm.org/citation.cfm?id=1557019.1557164}, citeulike-linkout-1 = {http://dx.doi.org/10.1145/1557019.1557164}, doi = {10.1145/1557019.1557164}, isbn = {978-1-60558-495-9}, keywords = {ranking, search}, location = {Paris, France}, pages = {1355--1364}, posted-at = {2009-07-03 09:27:45}, priority = {2}, publisher = {ACM}, title = {PSkip: estimating relevance ranking quality from web search clickthrough data}, url = {http://dx.doi.org/10.1145/1557019.1557164}, year = {2009} }

TY - CONF ID - citeulike:5044005 L3 - citeulike-article-id:5044005 N2 - In this article, we report our efforts in mining the information encoded as clickthrough data in the server logs to evaluate and monitor the relevance ranking quality of a commercial web search engine. We describe a metric called pSkip that aims to quantify the ranking quality by estimating the probability of users encountering non relevant results that cost them the efforts to read and skip. A search engine with a lower pSkip is regarded as having a better ranking quality. A key design goal of pSkip is to integrate the findings from two sets of user studies that utilize eye-tracking devices to track users' browsing patterns on the search result pages, and that use specially instrumented browsers to actively solicit users' explicit judgments on their search activities. We present the derivation of the maximum likelihood estimation of pSkip and demonstrate its efficacy in describing the user study data. The mathematical properties of pSkip are further analyzed and compared with several objective metrics as well as the cumulated gain method that uses subjective judgments. Experimental data show that pSkip can measure aspects of the search quality that these existing metrics are not designed or fail to address, such as identifying the real search intents expressed in the ambiguous queries. Although effective and superior in many ways, we also report a series of experiments that show pSkip may be influenced by system issues that are not directly related to relevance ranking, suggesting that measurements complementary to pSkip are still needed in order to form a holistic and accurate characterization of the ranking quality. CY - Paris, France SN - 978-1-60558-495-9 AD - New York, NY, USA EP - 1364 TI - PSkip: estimating relevance ranking quality from web search clickthrough data PB - ACM T2 - KDD '09: Proceedings of the 15th ACM SIGKDD international conference on Knowledge discovery and data mining SP - 1355 KW - ranking KW - search AU - Wang, Kuansan AU - Walker, Toby AU - Zheng, Zijian PY - 2009/// UR - http://dx.doi.org/10.1145/1557019.1557164 ER -