# Robots.txt file

User-Agent: Direct Hit Grabber
User-Agent: Mozilla/4.06 [en] (Direct Hit Grabber)
Disallow: /

User-Agent: Slurp
Disallow: /

User-Agent: Scooter
Disallow: /

User-Agent: AhrefsBot
Disallow: /

# Let all other crawlers do their stuff unhindered
User-Agent: *