# try to block wget's standard setting, but they can 
# still use 'wget -e robots=off'
# slow down robots for bandwidth savings
User-agent: *
Crawl-delay: 10
Disallow: /images/  

# block bad robots
User-agent: wget
Disallow: /  
User-agent: ShopWiki
Disallow: /
User-agent: Googlebot-Image
Disallow: /
User-agent: yahoo-mmcrawler
Disallow: /
User-agent: psbot
Disallow: /
User-agent: Exalead
Disallow: /
User-agent: Exabot
Disallow: /
User-agent: crawl
Disallow: /

# Dis-Allow all, only use for stage2
#User-agent:  *
#Disallow: /