Crawling the Hidden Web Export

@inproceedings{citeulike:1764402, abstract = {Current-day crawlers retrieve content only from the publicly indexableWeb, i.e., the set of web pages reachable purely by following hypertext links, ignoring search forms and pages that require authorization or prior registration. In particular, they ignore the tremendous amount of high quality content ” hidden” behind search forms, in large searchable electronic databases. In this paper, we provide a framework for addressing the problem of extracting content from this hidden Web. At Stanford, we have built a task-specific hidden Web crawler called the Hidden Web Exposer (HiWE). We describe the architecture of HiWE and present a number of novel techniques that went into its design and implementation. We also present results from experiments we conducted to test and validate our techniques.}, address = {San Francisco, CA, USA}, author = {Raghavan, Sriram and Garcia-Molina, Hector}, booktitle = {VLDB '01: Proceedings of the 27th International Conference on Very Large Data Bases}, citeulike-article-id = {1764402}, citeulike-linkout-0 = {http://portal.acm.org/citation.cfm?id=672025}, citeulike-linkout-1 = {http://citeseer.ist.psu.edu/raghavan01crawling.html}, citeulike-linkout-2 = {http://dbpubs.stanford.edu:8090/pub/2001-19}, isbn = {1558608044}, keywords = {2001, crawler, hidden\_web, stanford}, pages = {129--138}, posted-at = {2007-10-13 17:38:13}, priority = {3}, publisher = {Morgan Kaufmann Publishers Inc.}, title = {Crawling the Hidden Web}, url = {http://portal.acm.org/citation.cfm?id=672025}, year = {2001} }

TY - CONF ID - citeulike:1764402 L3 - citeulike-article-id:1764402 N2 - Current-day crawlers retrieve content only from the publicly indexableWeb, i.e., the set of web pages reachable purely by following hypertext links, ignoring search forms and pages that require authorization or prior registration. In particular, they ignore the tremendous amount of high quality content “hidden” behind search forms, in large searchable electronic databases. In this paper, we provide a framework for addressing the problem of extracting content from this hidden Web. At Stanford, we have built a task-specific hidden Web crawler called the Hidden Web Exposer (HiWE). We describe the architecture of HiWE and present a number of novel techniques that went into its design and implementation. We also present results from experiments we conducted to test and validate our techniques. SN - 1558608044 AD - San Francisco, CA, USA EP - 138 TI - Crawling the Hidden Web PB - Morgan Kaufmann Publishers Inc. T2 - VLDB '01: Proceedings of the 27th International Conference on Very Large Data Bases SP - 129 KW - 2001 KW - crawler KW - hidden_web KW - stanford AU - Raghavan, Sriram AU - Garcia-Molina, Hector PY - 2001/// UR - http://portal.acm.org/citation.cfm?id=672025 ER -