# robots.txt for DMM Web Sites

User-agent: slurp
User-agent: googlebot
User-agent: msnbot
Disallow: /_private/
Disallow: /_vti_bin/
Disallow: /_vti_cnf/
Disallow: /_vti_inf.html
Disallow: /_vti_pvt/
Disallow: /_vti_txt/
Disallow: /abandon/i/
Disallow: /abandon/images/
Disallow: /archives/i/
Disallow: /archives/images/
Disallow: /archives/i/
Disallow: /articles/t/
Disallow: /banners/i/
Disallow: /books/i/
Disallow: /books/images/
Disallow: /certs/images/
Disallow: /cgi-bin/
Disallow: /colliery/banners/
Disallow: /colliery/i/
Disallow: /colliery/images/
Disallow: /collnear/i/
Disallow: /company/i/
Disallow: /company/images/
Disallow: /css/
Disallow: /daysout/i/
Disallow: /daysout/images/
Disallow: /daysout/t/
Disallow: /displays/i/
Disallow: /displays/images/
Disallow: /educate/i/
Disallow: /educate/images/
Disallow: /errors/
Disallow: /galantry/i/
Disallow: /history/i/
Disallow: /history/images/
Disallow: /history/t/
Disallow: /images/
Disallow: /incident/i/
Disallow: /localrec/i/
Disallow: /inquests/i/
Disallow: /lowsrc/
Disallow: /managers/i/
Disallow: /minerals/i/
Disallow: /minerals/t/
Disallow: /misc/i/
Disallow: /names/i/
Disallow: /names/images/
Disallow: /ncb/i/
Disallow: /postinfo.html
Disallow: /prosecut/i/
Disallow: /reports/i/
Disallow: /reports/images/
Disallow: /shafts/i/
Disallow: /shaftsf/i/
Disallow: /shaftsm/i/
Disallow: /sitemap/i/
Disallow: /spam/
Disallow: /stats/i/
Disallow: /stats/images/
Disallow: /temp/
Disallow: /whoswho/i/
Disallow: /whoswho/images/

# Block crawler: Mozilla/4.0 (compatible;  Vagabondo/4.0; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/; http://www.wise-guys.nl/)
User-agent: Bilbo
Disallow: /

# Block crawler: Mozilla/4.0 (compatible;  Vagabondo/4.0; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/; http://www.wise-guys.nl/)
User-agent: Vagabondo
Disallow: /

# Block crawler: Mozilla/5.0 (compatible; WebDataCentreBot/1.0; +http://WebDataCentre.com/)
User-agent: WebDataCentreBot
Disallow: /

# Block crawler: Nokia6680/1.0 (4.04.07) SymbianOS/8.0 Series60/2.6 Profile/MIDP-2.0 Configuration/CLDC-1.1 (botmobi find.mobi/bot.html find@mtld.mobi)
User-agent: botmobi
Disallow: /

# Block crawler: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; MSIECrawler)
User-agent: MSIECrawler
Disallow: /

# Block crawler: CatchBot/1.0; +http://www.catchbot.com
User-agent: CatchBot
Disallow: /

# Block crawler: Mozilla/5.0 (compatible; FirstSearchBot/1.0; +http://www.first-search.com/dmm-pitwork.org.uk.htm)
User-agent: FirstSearchBot
Disallow: /

# Block crawler: Gaisbot/3.0+(robot06@gais.cs.ccu.edu.tw;+http://gais.cs.ccu.edu.tw/robot.php)
User-agent: Gaisbot
Disallow: /

# Block crawler: KeywenBot/4.1  http://www.keywen.com/Encyclopedia/Links
User-agent: KeywenBot
Disallow: /

# Block crawler: Sosospider+(+http://help.soso.com/webspider.htm)
User-agent: Sosospider
Disallow: /

# Block crawler: compatible; Mozilla 4.0; MSIE 5.5; (SqwidgeBot b1 - http://www.sqwidge.com/bot/)
User-agent: SqwidgeBot
Disallow: /

# Block crawler: TurnitinBot/2.1 (http://www.turnitin.com/robot/crawlerinfo.html)
User-agent: TurnitinBot
Disallow: /

# 06/08/09
# Block crawler: Mozilla/5.0 (compatible; FirstSearchBot/1.0; +http://www.first-search.com/dmm-pitwork.org.uk.htm)
User-agent: FirstSearchBot/1.0
User-agent: FirstSearchBot
Disallow: /

# Block crawler: Mozilla/4.0 (compatible; BOTW Spider; +http://botw.org)
User-agent: BOTW Spider
Disallow: /

# Block crawler: DataPatrol/Nutch-1.0 (DataPatrol indexer from Garlik; http://www.garlik.com/products.php; crawler at garlik dot com)
User-agent: DataPatrol/Nutch-1.0
User-agent: DataPatrol
Disallow: /

# Block crawler: GingerCrawler/1.0 (Language Assistant for Dyslexics; www.gingersoftware.com/crawler_agent.htm; support at ginger software dot com)
User-agent: GingerCrawler/1.0
User-agent: GingerCrawler
Disallow: /

# Block crawler: smart.apnoti.com Robot/v1.34 (http://smart.apnoti.com/bot)
User-agent: smart.apnoti.com Robot/v1.34 (http://smart.apnoti.com/bot)
Disallow: /

# 13/08/09
# Block crawler: WebAlta Crawler/1.3.23 (http://www.webalta.net/ru/about_webmaster.html) (Windows; U; Windows NT 5.1; ru-RU)
User-agent: WebAlta Crawler/1.3.23
User-agent: WebAlta
Disallow: /

# Block crawler: Mozilla/5.0 (compatible; VDBot/1.0; +http://www.bvdep.com/)
User-agent: VDBot/1.0
User-agent: VDBot
Disallow: /

# Block crawler: Servage Robot ( http://www.servage.net/page/robot/ )
User-agent: Servage Robot
User-agent: Servage
Disallow: /

# Block crawler: Mozilla/5.0 (compatible; Purebot/1.1; +http://www.puritysearch.net/)
User-agent: Purebot
User-agent: Purebot/1.1
Disallow: /

User-agent: *
Disallow: /