On single-pass indexing with MapReduce Export

@inproceedings{McCreadie2009mapred, abstract = {Indexing is an important Information Retrieval (IR) operation, which must be parallelised to support large-scale document corpora. We propose a novel adaptation of the state-of-the-art single-pass indexing algorithm in terms of the MapReduce programming model. We then experiment with this adaptation, in the context of the Hadoop MapReduce implementation. In particular, we explore the scale of improvements that can be achieved when using firstly more processing hardware and secondly larger corpora. Our results show that indexing speed increases in a close to linear fashion when scaling corpus size or number of processing machines. This suggests that the proposed indexing implementation is viable to support upcoming large-scale corpora.}, address = {New York, NY, USA}, author = {McCreadie, Richard M. C. and Macdonald, Craig and Ounis, Iadh}, booktitle = {SIGIR '09: Proceedings of the 32nd international ACM SIGIR conference on Research and development in information retrieval}, citeulike-article-id = {5486538}, citeulike-linkout-0 = {http://portal.acm.org/citation.cfm?id=1571941.1572106}, citeulike-linkout-1 = {http://dx.doi.org/10.1145/1571941.1572106}, doi = {10.1145/1571941.1572106}, isbn = {978-1-60558-483-6}, keywords = {index, map, reduce}, location = {Boston, MA, USA}, pages = {742--743}, posted-at = {2009-11-10 04:32:00}, priority = {2}, publisher = {ACM}, title = {On single-pass indexing with MapReduce}, url = {http://dx.doi.org/10.1145/1571941.1572106}, year = {2009} }

TY - CONF ID - McCreadie2009mapred L3 - citeulike-article-id:5486538 N2 - Indexing is an important Information Retrieval (IR) operation, which must be parallelised to support large-scale document corpora. We propose a novel adaptation of the state-of-the-art single-pass indexing algorithm in terms of the MapReduce programming model. We then experiment with this adaptation, in the context of the Hadoop MapReduce implementation. In particular, we explore the scale of improvements that can be achieved when using firstly more processing hardware and secondly larger corpora. Our results show that indexing speed increases in a close to linear fashion when scaling corpus size or number of processing machines. This suggests that the proposed indexing implementation is viable to support upcoming large-scale corpora. CY - Boston, MA, USA SN - 978-1-60558-483-6 AD - New York, NY, USA EP - 743 TI - On single-pass indexing with MapReduce PB - ACM T2 - SIGIR '09: Proceedings of the 32nd international ACM SIGIR conference on Research and development in information retrieval SP - 742 KW - index KW - map KW - reduce AU - McCreadie, Richard AU - Macdonald, Craig AU - Ounis, Iadh PY - 2009/// UR - http://dx.doi.org/10.1145/1571941.1572106 ER -