block all spiders by default, allow major ones
This commit is contained in:
parent
ed512cc065
commit
b1e9e359a3
|
@ -1,5 +1,16 @@
|
||||||
# no caching
|
# block all spiders by default
|
||||||
User-agent: ia_archiver
|
User-agent: *
|
||||||
Disallow: /
|
Disallow: /
|
||||||
User-agent: archive.org_bot
|
|
||||||
|
# but allow major ones
|
||||||
|
User-agent: Googlebot
|
||||||
|
Allow: /
|
||||||
|
|
||||||
|
User-agent: Slurp
|
||||||
|
Allow: /
|
||||||
|
|
||||||
|
User-Agent: msnbot
|
||||||
|
Disallow:
|
||||||
|
|
||||||
|
User-agent: Baiduspider
|
||||||
Disallow: /
|
Disallow: /
|
||||||
|
|
Loading…
Reference in New Issue