Sitemaps: above and beyond the crawl of duty Export

@inproceedings{citeulike:5776568, abstract = {Comprehensive coverage of the public web is crucial to web search engines. Search engines use crawlers to retrieve pages and then discover new ones by extracting the pages' outgoing links. However, the set of pages reachable from the publicly linked web is estimated to be significantly smaller than the invisible web, the set of documents that have no incoming links and can only be retrieved through web applications and web forms. The Sitemaps protocol is a fast-growing web protocol supported jointly by major search engines to help content creators and search engines unlock this hidden data by making it available to search engines. In this paper, we perform a detailed study of how "classic" discovery crawling compares with Sitemaps, in key measures such as coverage and freshness over key representative websites as well as over billions of URLs seen at Google. We observe that Sitemaps and discovery crawling complement each other very well, and offer different tradeoffs.}, address = {New York, NY, USA}, author = {Schonfeld, Uri and Shivakumar, Narayanan}, booktitle = {WWW '09: Proceedings of the 18th international conference on World wide web}, citeulike-article-id = {5776568}, citeulike-linkout-0 = {http://portal.acm.org/citation.cfm?id=1526842}, citeulike-linkout-1 = {http://dx.doi.org/10.1145/1526709.1526842}, doi = {10.1145/1526709.1526842}, isbn = {978-1-60558-487-4}, keywords = {sitemaps, web-crawling}, location = {Madrid, Spain}, pages = {991--1000}, posted-at = {2009-09-11 20:54:34}, priority = {0}, publisher = {ACM}, title = {Sitemaps: above and beyond the crawl of duty}, url = {http://dx.doi.org/10.1145/1526709.1526842}, year = {2009} }

TY - CONF ID - citeulike:5776568 L3 - citeulike-article-id:5776568 N2 - Comprehensive coverage of the public web is crucial to web search engines. Search engines use crawlers to retrieve pages and then discover new ones by extracting the pages' outgoing links. However, the set of pages reachable from the publicly linked web is estimated to be significantly smaller than the invisible web, the set of documents that have no incoming links and can only be retrieved through web applications and web forms. The Sitemaps protocol is a fast-growing web protocol supported jointly by major search engines to help content creators and search engines unlock this hidden data by making it available to search engines. In this paper, we perform a detailed study of how "classic" discovery crawling compares with Sitemaps, in key measures such as coverage and freshness over key representative websites as well as over billions of URLs seen at Google. We observe that Sitemaps and discovery crawling complement each other very well, and offer different tradeoffs. CY - Madrid, Spain SN - 978-1-60558-487-4 AD - New York, NY, USA EP - 1000 TI - Sitemaps: above and beyond the crawl of duty PB - ACM T2 - WWW '09: Proceedings of the 18th international conference on World wide web SP - 991 KW - sitemaps KW - web-crawling AU - Schonfeld, Uri AU - Shivakumar, Narayanan PY - 2009/// UR - http://dx.doi.org/10.1145/1526709.1526842 ER -