Crawling a country: better strategies than breadth-first for web page ordering Export

@inproceedings{citeulike:4491445, abstract = {This article compares several page ordering strategies for Web crawling under several metrics. The objective of these strategies is to download the most "important" pages "early" during the crawl. As the coverage of modern search engines is small compared to the size of the Web, and it is impossible to index all of the Web for both theoretical and practical reasons, it is relevant to index at least the most important pages.We use data from actual Web pages to build Web graphs and execute a crawler simulator on those graphs. As the Web is very dynamic, crawling simulation is the only way to ensure that all the strategies considered are compared under the same conditions. We propose several page ordering strategies that are more efficient than breadth- first search and strategies based on partial Pagerank calculations.}, address = {New York, NY, USA}, author = {Yates, Ricardo B. and Castillo, Carlos and Marin, Mauricio and Rodriguez, Andrea}, booktitle = {WWW '05: Special interest tracks and posters of the 14th international conference on World Wide Web}, citeulike-article-id = {4491445}, citeulike-linkout-0 = {http://portal.acm.org/citation.cfm?id=1062745.1062768}, citeulike-linkout-1 = {http://dx.doi.org/10.1145/1062745.1062768}, doi = {10.1145/1062745.1062768}, isbn = {1-59593-051-5}, keywords = {ir}, location = {Chiba, Japan}, pages = {864--872}, posted-at = {2009-05-22 11:22:35}, priority = {2}, publisher = {ACM}, title = {Crawling a country: better strategies than breadth-first for web page ordering}, url = {http://dx.doi.org/10.1145/1062745.1062768}, year = {2005} }

TY - CONF ID - citeulike:4491445 L3 - citeulike-article-id:4491445 N2 - This article compares several page ordering strategies for Web crawling under several metrics. The objective of these strategies is to download the most "important" pages "early" during the crawl. As the coverage of modern search engines is small compared to the size of the Web, and it is impossible to index all of the Web for both theoretical and practical reasons, it is relevant to index at least the most important pages.We use data from actual Web pages to build Web graphs and execute a crawler simulator on those graphs. As the Web is very dynamic, crawling simulation is the only way to ensure that all the strategies considered are compared under the same conditions. We propose several page ordering strategies that are more efficient than breadth- first search and strategies based on partial Pagerank calculations. CY - Chiba, Japan SN - 1-59593-051-5 AD - New York, NY, USA EP - 872 TI - Crawling a country: better strategies than breadth-first for web page ordering PB - ACM T2 - WWW '05: Special interest tracks and posters of the 14th international conference on World Wide Web SP - 864 KW - ir AU - Yates, Ricardo AU - Castillo, Carlos AU - Marin, Mauricio AU - Rodriguez, Andrea PY - 2005/// UR - http://dx.doi.org/10.1145/1062745.1062768 ER -