Managing duplicates across sequential crawls Export

@misc{managingduplicates, abstract = {Dealing with documents that remain unchanged between harvesting rounds is an important topic for many organizations archiving the World Wide Web. This paper discusses some of the key problems in dealing with this and then outlines a simple, yet effective way of managing at least a part of it. This is done in form of an add-on module for the popular web crawler Heritrix. The paper contains the results of crawls using this new software. Finally, there is a discussion on the limitations and some of the future work needed to improve duplicate handling.}, author = {Sigurðsson, Kristinn}, booktitle = {Papers from the 6th International Web Archiving Workshop (IWAW'06)}, citeulike-article-id = {5861909}, citeulike-linkout-0 = {http://vefsofnun.bok.hi.is/upload/3/ManagingDuplicatesAcrossSequentialCrawls.pdf}, day = {33}, keywords = {indexing, robots, search-engines}, month = {September}, posted-at = {2009-09-30 21:47:37}, priority = {2}, title = {Managing duplicates across sequential crawls}, url = {http://vefsofnun.bok.hi.is/upload/3/ManagingDuplicatesAcrossSequentialCrawls.pdf}, year = {2006} }

TY - ELEC ID - managingduplicates L3 - citeulike-article-id:5861909 N2 - Dealing with documents that remain unchanged between harvesting rounds is an important topic for many organizations archiving the World Wide Web. This paper discusses some of the key problems in dealing with this and then outlines a simple, yet effective way of managing at least a part of it. This is done in form of an add-on module for the popular web crawler Heritrix. The paper contains the results of crawls using this new software. Finally, there is a discussion on the limitations and some of the future work needed to improve duplicate handling. TI - Managing duplicates across sequential crawls T2 - Papers from the 6th International Web Archiving Workshop (IWAW'06) KW - indexing KW - robots KW - search-engines AU - Sigurðsson, Kristinn PY - 2006/09/33/ UR - http://vefsofnun.bok.hi.is/upload/3/ManagingDuplicatesAcrossSequentialCrawls.pdf ER -