Sitemap: http://www.citeulike.org/sitemap.xml

# Useless/Evil bots #
User-agent: LinkWalker
User-agent: NPBot
User-agent: sna-0
User-agent: ConveraCrawler
User-agent: MJ12bot
User-agent: Yahoo Pipes 1.0
User-agent: Yahoo Pipes 2.0
Disallow: /

# Google supports wildcards
# Without this restriction we just end up with too many pages
User-agent: Googlebot
Disallow: /copy
Disallow: /delete
Disallow: /deletenote
Disallow: /bibtex/
Disallow: /endnote/
Disallow: /delicious/
Disallow: /bibtex_options/
Disallow: /pdf_options/
Disallow: /login
Disallow: /login?
Disallow: /login?from
Disallow: /logout
Disallow: /posturl
Disallow: /add
Disallow: /post_url.adp
Disallow: /posturl2
Disallow: /rss/
Disallow: /search/
Disallow: /search/all
Disallow: /nocrawl/
Disallow: /login.do
Disallow: /post_known.do
Disallow: /post_unknown.do
Disallow: /register.do

User-agent: MJ12bot
Disallow: /copy
Disallow: /delete
Disallow: /deletenote
Disallow: /bibtex/
Disallow: /endnote/
Disallow: /delicious/
Disallow: /bibtex_options/
Disallow: /pdf_options/
Disallow: /login
Disallow: /login?
Disallow: /login?from
Disallow: /logout
Disallow: /posturl
Disallow: /add
Disallow: /post_url.adp
Disallow: /posturl2
Disallow: /rss/
Disallow: /search/
Disallow: /search/all
Disallow: /nocrawl/
Disallow: /login.do
Disallow: /post_known.do
Disallow: /post_unknown.do
Disallow: /register.do
Crawl-delay: 1

User-agent: *
Disallow: /copy
Disallow: /delete
Disallow: /deletenote
Disallow: /bibtex/
Disallow: /endnote/
Disallow: /delicious/
Disallow: /bibtex_options/
Disallow: /pdf_options/
Disallow: /login
Disallow: /login?
Disallow: /login?from
Disallow: /logout
Disallow: /posturl
Disallow: /add
Disallow: /post_url.adp
Disallow: /posturl2
Disallow: /rss/
Disallow: /search/
Disallow: /search/all
Disallow: /nocrawl/
Disallow: /login.do
Disallow: /post_known.do
Disallow: /post_unknown.do
Disallow: /register.do
Crawl-delay: 2