12 lines
206 B
Plaintext
12 lines
206 B
Plaintext
|
# Directions for web crawlers.
|
||
|
# See http://www.robotstxt.org/wc/norobots.html.
|
||
|
|
||
|
User-agent: HTTrack
|
||
|
User-agent: puf
|
||
|
User-agent: MSIECrawler
|
||
|
User-agent: Nutch
|
||
|
Disallow: /
|
||
|
|
||
|
User-agent: msnbot
|
||
|
Crawl-delay: 1
|