IRLbot: scaling to 6 billion pages and beyond TeX Export

@inproceedings{citeulike:3115612, abstract = {This paper shares our experience in designing a web crawler that can download billions of pages using a single-server implementation and models its performance. We show that with the quadratically increasing complexity of verifying URL uniqueness, BFS crawl order, and fixed per-host rate-limiting, current crawling algorithms cannot effectively cope with the sheer volume of URLs generated in large crawls, highly-branching spam, legitimate multi-million-page blog sites, and infinite loops created by server-side scripts. We offer a set of techniques for dealing with these issues and test their performance in an implementation we call IRLbot. In our recent experiment that lasted 41 days, IRLbot running on a single server successfully crawled 6.3 billion valid HTML pages (\$7.6\$ billion connection requests) and sustained an average download rate of 319 mb/s (1,789 pages/s). Unlike our prior experiments with algorithms proposed in related work, this version of IRLbot did not experience any bottlenecks and successfully handled content from over 117 million hosts, parsed out 394 billion links, and discovered a subset of the web graph with 41 billion unique nodes.}, address = {New York, NY, USA}, author = {Lee, Hsin T. and Leonard, Derek and Wang, Xiaoming and Loguinov, Dmitri}, booktitle = {WWW '08: Proceeding of the 17th international conference on World Wide Web}, citeulike-article-id = {3115612}, citeulike-linkout-0 = {http://portal.acm.org/citation.cfm?id=1367497.1367556}, citeulike-linkout-1 = {http://dx.doi.org/10.1145/1367497.1367556}, doi = {10.1145/1367497.1367556}, isbn = {978-1-60558-085-2}, keywords = {web-crawling}, location = {Beijing, China}, pages = {427--436}, posted-at = {2009-06-02 16:52:46}, priority = {2}, publisher = {ACM}, title = {IRLbot: scaling to 6 billion pages and beyond}, url = {http://dx.doi.org/10.1145/1367497.1367556}, year = {2008} }

TY - CONF ID - citeulike:3115612 L3 - citeulike-article-id:3115612 N2 - This paper shares our experience in designing a web crawler that can download billions of pages using a single-server implementation and models its performance. We show that with the quadratically increasing complexity of verifying URL uniqueness, BFS crawl order, and fixed per-host rate-limiting, current crawling algorithms cannot effectively cope with the sheer volume of URLs generated in large crawls, highly-branching spam, legitimate multi-million-page blog sites, and infinite loops created by server-side scripts. We offer a set of techniques for dealing with these issues and test their performance in an implementation we call IRLbot. In our recent experiment that lasted 41 days, IRLbot running on a single server successfully crawled 6.3 billion valid HTML pages ($7.6$ billion connection requests) and sustained an average download rate of 319 mb/s (1,789 pages/s). Unlike our prior experiments with algorithms proposed in related work, this version of IRLbot did not experience any bottlenecks and successfully handled content from over 117 million hosts, parsed out 394 billion links, and discovered a subset of the web graph with 41 billion unique nodes. CY - Beijing, China SN - 978-1-60558-085-2 AD - New York, NY, USA EP - 436 TI - IRLbot: scaling to 6 billion pages and beyond PB - ACM T2 - WWW '08: Proceeding of the 17th international conference on World Wide Web SP - 427 KW - web-crawling AU - Lee, Hsin AU - Leonard, Derek AU - Wang, Xiaoming AU - Loguinov, Dmitri PY - 2008/// UR - http://dx.doi.org/10.1145/1367497.1367556 ER -