# We don't want the data downloads being indexed by search engines because it
# would just cause confusion in search results

User-agent: *
Disallow: /perl/wikipedia_cities.cgi
Disallow: /cgi-bin/ubb/search.cgi
Disallow: /cgi-bin/ubb/postings.cgi

User-agent: msnbot
Disallow: /perl/wikipedia_cities.cgi
Disallow: /cgi-bin/ubb/search.cgi
Disallow: /cgi-bin/ubb/postings.cgi
Crawl-Delay: 2