On test collections for adaptive information retrieval Export

@article{citeulike:2563854, abstract = {Traditional Cranfield test collections represent an abstraction of a retrieval task that Sparck Jones calls the ''core competency'' of retrieval: a task that is necessary, but not sufficient, for user retrieval tasks. The abstraction facilitates research by controlling for (some) sources of variability, thus increasing the power of experiments that compare system effectiveness while reducing their cost. However, even within the highly-abstracted case of the Cranfield paradigm, meta-analysis demonstrates that the user/topic effect is greater than the system effect, so experiments must include a relatively large number of topics to distinguish systems' effectiveness. The evidence further suggests that changing the abstraction slightly to include just a bit more characterization of the user will result in a dramatic loss of power or increase in cost of retrieval experiments. Defining a new, feasible abstraction for supporting adaptive IR research will require winnowing the list of all possible factors that can affect retrieval behavior to a minimum number of essential factors.}, address = {Tarrytown, NY, USA}, author = {Voorhees, Ellen M.}, citeulike-article-id = {2563854}, citeulike-linkout-0 = {http://portal.acm.org/citation.cfm?id=1453386}, citeulike-linkout-1 = {http://dx.doi.org/10.1016/j.ipm.2007.12.011}, citeulike-linkout-2 = {http://linkinghub.elsevier.com/retrieve/pii/S0306457308000253}, citeulike-linkout-3 = {http://www.sciencedirect.com/science/article/B6VC8-4S2VG3F-1/2/fb8e860b7f80cc4e93df31548413da6a}, doi = {10.1016/j.ipm.2007.12.011}, issn = {0306-4573}, journal = {Inf. Process. Manage.}, keywords = {adaptive, collection, evaluation, ir, test}, month = {October}, number = {6}, pages = {1879--1885}, posted-at = {2008-10-24 11:40:44}, priority = {2}, publisher = {Pergamon Press, Inc.}, title = {On test collections for adaptive information retrieval}, url = {http://dx.doi.org/10.1016/j.ipm.2007.12.011}, volume = {44}, year = {2008} }

TY - JOUR ID - citeulike:2563854 L3 - citeulike-article-id:2563854 N2 - Traditional Cranfield test collections represent an abstraction of a retrieval task that Sparck Jones calls the ''core competency'' of retrieval: a task that is necessary, but not sufficient, for user retrieval tasks. The abstraction facilitates research by controlling for (some) sources of variability, thus increasing the power of experiments that compare system effectiveness while reducing their cost. However, even within the highly-abstracted case of the Cranfield paradigm, meta-analysis demonstrates that the user/topic effect is greater than the system effect, so experiments must include a relatively large number of topics to distinguish systems' effectiveness. The evidence further suggests that changing the abstraction slightly to include just a bit more characterization of the user will result in a dramatic loss of power or increase in cost of retrieval experiments. Defining a new, feasible abstraction for supporting adaptive IR research will require winnowing the list of all possible factors that can affect retrieval behavior to a minimum number of essential factors. IS - 6 JF - Inf. Process. Manage. SN - 0306-4573 AD - Tarrytown, NY, USA EP - 1885 TI - On test collections for adaptive information retrieval PB - Pergamon Press, Inc. VL - 44 SP - 1879 KW - adaptive KW - collection KW - evaluation KW - ir KW - test AU - Voorhees, Ellen PY - 2008/10// UR - http://dx.doi.org/10.1016/j.ipm.2007.12.011 ER -