# please do not crawl


User-agent: *
Disallow: /admin/
Disallow: /includes/
Disallow: /download/
Disallow: /images/
Disallow: /product_info.php
Disallow: /image_search.php

# seconds to wait between successive requests = 30
User-agent: *
Crawl-delay: 30

# maximum rate is one page every 20 seconds
User-agent: *
Request-rate: 1/20 

# only visit between 1:00 AM and 8:00 AM UT (GMT)
# Visit-time: 0100-0900 

# these agents is not allowed to crawl pages
User-Agent: MJ12bot
Disallow:   /

User-Agent: Yanga
Disallow:   /

User-Agent: Yanga WorldSearch Bot
Disallow:   /

User-Agent: YodaoBot
Disallow:   /